Make targets an attribute of compilers (#2500)

* Make targets an attribute of compilers, similar to OS. Allows users to use `spack compiler find` to add compilers for the same platform/os but different targets when spack is installed on a file system mounted on machines with different targets.

* Changed get_compilers() method to treat old compilers with no target as target='any'

* flake8 changes

* Address change requests for comments/code clarity

lib/spack/spack/architecture.py

@@ -77,6 +77,7 @@
 """
 import os
 import inspect
+import platform as py_platform
 
 from llnl.util.lang import memoized, list_modules, key_ordering
 from llnl.util.filesystem import join_path
@@ -334,7 +335,7 @@ def find_compiler(self, cmp_cls, *path):
                 if newcount <= prevcount:
                     continue
 
-            compilers[ver] = cmp_cls(spec, self, paths)
+            compilers[ver] = cmp_cls(spec, self, py_platform.machine(), paths)
 
         return list(compilers.values())
 

lib/spack/spack/cmd/compiler.py

@@ -32,7 +32,7 @@
 from llnl.util.lang import index_by
 from llnl.util.tty.colify import colify
 from llnl.util.tty.color import colorize
-from spack.spec import CompilerSpec
+from spack.spec import CompilerSpec, ArchSpec
 from spack.util.environment import get_path
 
 description = "Manage compilers"
@@ -91,17 +91,26 @@ def compiler_find(args):
     # Just let compiler_find do the
     # entire process and return an empty config from all_compilers
     # Default for any other process is init_config=True
-    compilers = [c for c in spack.compilers.find_compilers(*paths)
+    compilers = [c for c in spack.compilers.find_compilers(*paths)]
-                 if c.spec not in spack.compilers.all_compilers(
+    new_compilers = []
-                     scope=args.scope, init_config=False)]
+    for c in compilers:
-    if compilers:
+        arch_spec = ArchSpec(None, c.operating_system, c.target)
-        spack.compilers.add_compilers_to_config(compilers, scope=args.scope,
+        same_specs = spack.compilers.compilers_for_spec(c.spec,
+                                                        arch_spec,
+                                                        args.scope)
+
+        if not same_specs:
+            new_compilers.append(c)
+
+    if new_compilers:
+        spack.compilers.add_compilers_to_config(new_compilers,
+                                                scope=args.scope,
                                                 init_config=False)
-        n = len(compilers)
+        n = len(new_compilers)
         s = 's' if n > 1 else ''
         filename = spack.config.get_config_filename(args.scope, 'compilers')
         tty.msg("Added %d new compiler%s to %s" % (n, s, filename))
-        colify(reversed(sorted(c.spec for c in compilers)), indent=4)
+        colify(reversed(sorted(c.spec for c in new_compilers)), indent=4)
     else:
         tty.msg("Found no new compilers")
 

lib/spack/spack/compiler.py

@@ -112,11 +112,12 @@ def fc_rpath_arg(self):
     # Name of module used to switch versions of this compiler
     PrgEnv_compiler = None
 
-    def __init__(self, cspec, operating_system,
+    def __init__(self, cspec, operating_system, target,
                  paths, modules=[], alias=None, environment=None,
                  extra_rpaths=None, **kwargs):
         self.spec = cspec
         self.operating_system = str(operating_system)
+        self.target = str(target)
         self.modules = modules
         self.alias = alias
 

lib/spack/spack/compilers/__init__.py

@@ -61,6 +61,7 @@ def _to_dict(compiler):
                       for attr in _path_instance_vars)
     d['flags'] = dict((fname, fvals) for fname, fvals in compiler.flags)
     d['operating_system'] = str(compiler.operating_system)
+    d['target'] = str(compiler.target)
     d['modules'] = compiler.modules if compiler.modules else []
     d['environment'] = compiler.environment if compiler.environment else {}
     d['extra_rpaths'] = compiler.extra_rpaths if compiler.extra_rpaths else []
@@ -216,10 +217,21 @@ def get_compilers(cspec):
             if items['spec'] != str(cspec):
                 continue
 
+            # If an arch spec is given, confirm that this compiler
+            # is for the given operating system
             os = items.get('operating_system', None)
             if arch_spec and os != arch_spec.platform_os:
                 continue
 
+            # If an arch spec is given, confirm that this compiler
+            # is for the given target. If the target is 'any', match
+            # any given arch spec. If the compiler has no assigned
+            # target this is an old compiler config file, skip this logic.
+            target = items.get('target', None)
+            if arch_spec and target and (target != arch_spec.target and
+                                         target != 'any'):
+                continue
+
             if not ('paths' in items and
                     all(n in items['paths'] for n in _path_instance_vars)):
                 raise InvalidCompilerConfigurationError(cspec)
@@ -244,8 +256,8 @@ def get_compilers(cspec):
             extra_rpaths = items.get('extra_rpaths', [])
 
             compilers.append(
-                cls(cspec, os, compiler_paths, mods, alias, environment,
+                cls(cspec, os, target, compiler_paths, mods, alias,
-                    extra_rpaths, **compiler_flags))
+                    environment, extra_rpaths, **compiler_flags))
 
         return compilers
 

lib/spack/spack/concretize.py

@@ -305,10 +305,12 @@ def concretize_compiler(self, spec):
            build with the compiler that will be used by libraries that
            link to this one, to maximize compatibility.
         """
-        # Pass on concretizing the compiler if the target is not yet determined
+        # Pass on concretizing the compiler if the target or operating system
-        if not spec.architecture.platform_os:
+        # is not yet determined
-            # Although this usually means changed, this means awaiting other
+        if not (spec.architecture.platform_os and spec.architecture.target):
-            # changes
+            # We haven't changed, but other changes need to happen before we
+            # continue. `return True` here to force concretization to keep
+            # running.
             return True
 
         # Only use a matching compiler if it is of the proper style
@@ -356,7 +358,8 @@ def _proper_compiler_style(cspec, aspec):
         if not matches:
             arch = spec.architecture
             raise UnavailableCompilerVersionError(other_compiler,
-                                                  arch.platform_os)
+                                                  arch.platform_os,
+                                                  arch.target)
 
         # copy concrete version into other_compiler
         try:
@@ -365,7 +368,8 @@ def _proper_compiler_style(cspec, aspec):
                 if _proper_compiler_style(c, spec.architecture)).copy()
         except StopIteration:
             raise UnavailableCompilerVersionError(
-                spec.compiler, spec.architecture.platform_os
+                spec.compiler, spec.architecture.platform_os,
+                spec.architecture.target
             )
 
         assert(spec.compiler.concrete)
@@ -377,10 +381,12 @@ def concretize_compiler_flags(self, spec):
         compiler is used, defaulting to no compiler flags in the spec.
         Default specs set at the compiler level will still be added later.
         """
-
+        # Pass on concretizing the compiler flags if the target or operating
-        if not spec.architecture.platform_os:
+        # system is not set.
-            # Although this usually means changed, this means awaiting other
+        if not (spec.architecture.platform_os and spec.architecture.target):
-            # changes
+            # We haven't changed, but other changes need to happen before we
+            # continue. `return True` here to force concretization to keep
+            # running.
             return True
 
         def compiler_match(spec1, spec2):
@@ -515,10 +521,11 @@ class UnavailableCompilerVersionError(spack.error.SpackError):
     """Raised when there is no available compiler that satisfies a
        compiler spec."""
 
-    def __init__(self, compiler_spec, operating_system):
+    def __init__(self, compiler_spec, operating_system, target):
         super(UnavailableCompilerVersionError, self).__init__(
             "No available compiler version matches '%s' on operating_system %s"
-            % (compiler_spec, operating_system),
+            "for target %s"
+            % (compiler_spec, operating_system, target),
             "Run 'spack compilers' to see available compiler Options.")
 
 

lib/spack/spack/operating_systems/cnl.py

@@ -57,7 +57,8 @@ def find_compiler(self, cmp_cls, *paths):
             for name, version in matches:
                 v = version
                 comp = cmp_cls(
-                    spack.spec.CompilerSpec(name + '@' + v), self,
+                    spack.spec.CompilerSpec(name + '@' + v),
+                    self, any,
                     ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name + '/' + v])
 
                 compilers.append(comp)