Merge branch 'develop' into features/shared

lib/spack/llnl/util/cpu/detect.py

@@ -223,3 +223,15 @@ def compatibility_check_for_x86_64(info, target):
     return (target == arch_root or arch_root in target.ancestors) \
         and (target.vendor == vendor or target.vendor == 'generic') \
         and target.features.issubset(features)
+
+
+@compatibility_check(architecture_family='aarch64')
+def compatibility_check_for_aarch64(info, target):
+    basename = 'aarch64'
+    features = set(info.get('Features', '').split())
+    vendor = info.get('CPU implementer', 'generic')
+
+    arch_root = targets[basename]
+    return (target == arch_root or arch_root in target.ancestors) \
+        and (target.vendor == vendor or target.vendor == 'generic') \
+        and target.features.issubset(features)

lib/spack/llnl/util/cpu/microarchitectures.json

@@ -1161,6 +1161,97 @@
         }
       }
     },
+    "thunderx2": {
+      "from": "aarch64",
+      "vendor": "0x43",
+      "features": [
+        "fp",
+        "asimd",
+        "evtstrm",
+        "aes",
+        "pmull",
+        "sha1",
+        "sha2",
+        "crc32",
+        "atomics",
+        "cpuid",
+        "asimdrdm"
+      ],
+      "compilers": {
+        "gcc": [
+          {
+            "versions": "4.8:4.8.9",
+            "flags": "-march=armv8-a"
+          },
+          {
+            "versions": "4.9:5.9",
+            "flags": "-march=armv8-a+crc+crypto"
+          },
+          {
+            "versions": "6:6.9",
+            "flags": "-march=armv8.1-a+crc+crypto"
+          },
+          {
+            "versions": "7:",
+            "flags": "-mcpu=thunderx2t99"
+          }
+        ],
+        "clang": {
+          "versions": ":",
+          "flags": "-march=armv8-a -mcpu=generic"
+        }
+      }
+    },
+    "a64fx": {
+      "from": "aarch64",
+      "vendor": "0x46",
+      "features": [
+        "fp",
+        "asimd",
+        "evtstrm",
+        "aes",
+        "pmull",
+        "sha1",
+        "sha2",
+        "crc32",
+        "atomics",
+        "cpuid",
+        "asimdrdm",
+        "fphp",
+        "asimdhp",
+        "fcma",
+        "dcpop",
+        "sve"
+      ],
+      "compilers": {
+        "gcc": [
+          {
+            "versions": "4.8:4.8.9",
+            "flags": "-march=armv8-a"
+          },
+          {
+            "versions": "4.9:5.9",
+            "flags": "-march=armv8-a+crc+crypto"
+          },
+          {
+            "versions": "6:6.9",
+            "flags": "-march=armv8.1-a+crc+crypto"
+          },
+          {
+            "versions": "7:7.9",
+            "flags": "-arch=armv8.2a+crc+crypt+fp16"
+          },
+          {
+            "versions": "8:",
+            "flags": "-arch=armv8.2a+crc+aes+sh2+fp16+sve -msve-vector-bits=512"
+          }
+        ],
+        "clang": {
+          "versions": ":",
+          "flags": "-march=armv8-a -mcpu=generic"
+        }
+      }
+    },
     "arm": {
       "from": null,
       "vendor": "generic",

lib/spack/spack/binary_distribution.py

@@ -521,8 +521,6 @@ def relocate_package(workdir, spec, allow_root):
     old_prefix = str(buildinfo.get('spackprefix',
                                    '/not/in/buildinfo/dictionary'))
     rel = buildinfo.get('relative_rpaths', False)
-    if rel:
-        return
 
     tty.msg("Relocating package from",
             "%s to %s." % (old_path, new_path))

lib/spack/spack/caches.py

@@ -54,20 +54,23 @@ class MirrorCache(object):
     def __init__(self, root):
         self.root = os.path.abspath(root)
 
-    def store(self, fetcher, relative_dest, cosmetic_path=None):
+    def store(self, fetcher, relative_dest):
+        """Fetch and relocate the fetcher's target into our mirror cache."""
+
         # Note this will archive package sources even if they would not
         # normally be cached (e.g. the current tip of an hg/git branch)
         dst = os.path.join(self.root, relative_dest)
         mkdirp(os.path.dirname(dst))
         fetcher.archive(dst)
 
-        # Add a symlink path that a human can read to understand what resource
-        # the archive path refers to
-        if not cosmetic_path:
-            return
-        cosmetic_path = os.path.join(self.root, cosmetic_path)
+    def symlink(self, mirror_ref):
+        """Symlink a human readible path in our mirror to the actual
+        storage location."""
+
+        cosmetic_path = os.path.join(self.root, mirror_ref.cosmetic_path)
         relative_dst = os.path.relpath(
-            dst, start=os.path.dirname(cosmetic_path))
+            mirror_ref.storage_path,
+            start=os.path.dirname(cosmetic_path))
         if not os.path.exists(cosmetic_path):
             mkdirp(os.path.dirname(cosmetic_path))
             os.symlink(relative_dst, cosmetic_path)

lib/spack/spack/mirror.py

@@ -522,7 +522,7 @@ def add_single_spec(spec, mirror_root, mirror_stats):
         else:
             tty.warn(
                 "Error while fetching %s" % spec.cformat('{name}{@version}'),
-                exception.message)
+                getattr(exception, 'message', exception))
         mirror_stats.error()
 
 

lib/spack/spack/patch.py

@@ -199,11 +199,15 @@ def fetch(self, stage):
         fetcher = fs.URLFetchStrategy(self.url, fetch_digest,
                                       expand=bool(self.archive_sha256))
 
-        per_package_ref = os.path.join(
-            self.owner.split('.')[-1], os.path.basename(self.url))
+        # The same package can have multiple patches with the same name but
+        # with different contents, therefore apply a subset of the hash.
+        name = '{0}-{1}'.format(os.path.basename(self.url), fetch_digest[:7])
+
+        per_package_ref = os.path.join(self.owner.split('.')[-1], name)
         # Reference starting with "spack." is required to avoid cyclic imports
         mirror_ref = spack.mirror.mirror_archive_paths(
-            fetcher, per_package_ref)
+            fetcher,
+            per_package_ref)
 
         self.stage = spack.stage.Stage(fetcher, mirror_paths=mirror_ref)
         self.stage.create()

lib/spack/spack/relocate.py

@@ -378,17 +378,21 @@ def replace_prefix_text(path_name, old_dir, new_dir):
     Replace old install prefix with new install prefix
     in text files using utf-8 encoded strings.
     """
-
-    def replace(match):
-        return match.group().replace(old_dir.encode('utf-8'),
-                                     new_dir.encode('utf-8'))
     with open(path_name, 'rb+') as f:
         data = f.read()
         f.seek(0)
-        pat = re.compile(old_dir.encode('utf-8'))
-        if not pat.search(data):
-            return
-        ndata = pat.sub(replace, data)
+        # Replace old_dir with new_dir if it appears at the beginning of a path
+        # Negative lookbehind for a character legal in a path
+        # Then a match group for any characters legal in a compiler flag
+        # Then old_dir
+        # Then characters legal in a path
+        # Ensures we only match the old_dir if it's precedeed by a flag or by
+        # characters not legal in a path, but not if it's preceeded by other
+        # components of a path.
+        old_bytes = old_dir.encode('utf-8')
+        pat = b'(?<![\\w\\-_/])([\\w\\-_]*?)%s([\\w\\-_/]*)' % old_bytes
+        repl = b'\\1%s\\2' % new_dir.encode('utf-8')
+        ndata = re.sub(pat, repl, data)
         f.write(ndata)
         f.truncate()
 

lib/spack/spack/stage.py

@@ -501,13 +501,13 @@ def cache_mirror(self, stats):
 
         if os.path.exists(absolute_storage_path):
             stats.already_existed(absolute_storage_path)
-            return
+        else:
+            self.fetch()
+            spack.caches.mirror_cache.store(
+                self.fetcher, self.mirror_paths.storage_path)
+            stats.added(absolute_storage_path)
 
-        self.fetch()
-        spack.caches.mirror_cache.store(
-            self.fetcher, self.mirror_paths.storage_path,
-            self.mirror_paths.cosmetic_path)
-        stats.added(absolute_storage_path)
+        spack.caches.mirror_cache.symlink(self.mirror_paths)
 
     def expand_archive(self):
         """Changes to the stage directory and attempt to expand the downloaded

lib/spack/spack/test/data/targets/linux-centos7-thunderx2

@@ -0,0 +1,8 @@
+processor       : 0
+BogoMIPS        : 400.00
+Features        : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics cpuid asimdrdm
+CPU implementer : 0x43
+CPU architecture: 8
+CPU variant     : 0x1
+CPU part        : 0x0af
+CPU revision    : 1

lib/spack/spack/test/llnl/util/cpu.py

@@ -32,6 +32,7 @@
     'linux-scientific7-piledriver',
     'linux-rhel6-piledriver',
     'linux-centos7-power8le',
+    'linux-centos7-thunderx2',
     'darwin-mojave-ivybridge',
     'darwin-mojave-haswell',
     'darwin-mojave-skylake',
@@ -121,6 +122,8 @@ def test_equality(supported_target):
     ('piledriver <= steamroller', True),
     ('zen2 >= zen', True),
     ('zen >= zen', True),
+    ('aarch64 <= thunderx2', True),
+    ('aarch64 <= a64fx', True),
     # Test unrelated microarchitectures
     ('power8 < skylake', False),
     ('power8 <= skylake', False),
@@ -205,12 +208,15 @@ def test_target_json_schema():
     ('nehalem', 'gcc', '4.9.3', '-march=nehalem -mtune=nehalem'),
     ('nehalem', 'gcc', '4.8.5', '-march=corei7 -mtune=corei7'),
     ('sandybridge', 'gcc', '4.8.5', '-march=corei7-avx -mtune=corei7-avx'),
+    ('thunderx2', 'gcc', '4.8.5', '-march=armv8-a'),
+    ('thunderx2', 'gcc', '4.9.3', '-march=armv8-a+crc+crypto'),
     # Test Clang / LLVM
     ('sandybridge', 'clang', '3.9.0', '-march=x86-64 -mcpu=sandybridge'),
     ('icelake', 'clang', '6.0.0', '-march=x86-64 -mcpu=icelake'),
     ('icelake', 'clang', '8.0.0', '-march=x86-64 -mcpu=icelake-client'),
     ('zen2', 'clang', '9.0.0', '-march=x86-64 -mcpu=znver2'),
     ('power9le', 'clang', '8.0.0', '-march=ppc64le -mcpu=pwr9'),
+    ('thunderx2', 'clang', '6.0.0', '-march=armv8-a -mcpu=generic'),
     # Test Intel on Intel CPUs
     ('sandybridge', 'intel', '17.0.2', '-march=corei7-avx -mtune=corei7-avx'),
     ('sandybridge', 'intel', '18.0.5',