Create option to build compilers as needed (#10761)

* Create option to build missing compilers and add them to config before installing packages that use them
* Clean up kwarg passing for do_install, put compiler bootstrapping in separate method

etc/spack/defaults/config.yaml

@@ -66,6 +66,12 @@ config:
   verify_ssl: true
 
 
+  # If set to true, Spack will attempt to build any compiler on the spec
+  # that is not already available. If set to False, Spack will only use
+  # compilers already configured in compilers.yaml
+  install_missing_compilers: False
+
+
   # If set to true, Spack will always check checksums after downloading
   # archives. If false, Spack skips the checksum step.
   checksum: true

lib/spack/spack/compilers/__init__.py

@@ -28,6 +28,17 @@
 #: cache of compilers constructed from config data, keyed by config entry id.
 _compiler_cache = {}
 
+_compiler_to_pkg = {
+    'clang': 'llvm+clang'
+}
+
+
+def pkg_spec_for_compiler(cspec):
+    """Return the spec of the package that provides the compiler."""
+    spec_str = '%s@%s' % (_compiler_to_pkg.get(cspec.name, cspec.name),
+                          cspec.versions)
+    return spack.spec.Spec(spec_str)
+
 
 def _auto_compiler_spec(function):
     def converter(cspec_like, *args, **kwargs):
@@ -203,6 +214,17 @@ def find(compiler_spec, scope=None, init_config=True):
             if c.satisfies(compiler_spec)]
 
 
+@_auto_compiler_spec
+def find_specs_by_arch(compiler_spec, arch_spec, scope=None, init_config=True):
+    """Return specs of available compilers that match the supplied
+       compiler spec.  Return an empty list if nothing found."""
+    return [c.spec for c in compilers_for_spec(compiler_spec,
+                                               arch_spec,
+                                               scope,
+                                               True,
+                                               init_config)]
+
+
 def all_compilers(scope=None):
     config = get_compiler_config(scope)
     compilers = list()

lib/spack/spack/concretize.py

@@ -28,6 +28,7 @@
 import spack.compilers
 import spack.architecture
 import spack.error
+from spack.config import config
 from spack.version import ver, Version, VersionList, VersionRange
 from spack.package_prefs import PackagePrefs, spec_externals, is_spec_buildable
 
@@ -47,7 +48,8 @@ class Concretizer(object):
     def __init__(self):
         # controls whether we check that compiler versions actually exist
         # during concretization. Used for testing and for mirror creation
-        self.check_for_compiler_existence = True
+        self.check_for_compiler_existence = not config.get(
+            'config:install_missing_compilers', False)
 
     @contextmanager
     def disable_compiler_existence_check(self):
@@ -56,6 +58,13 @@ def disable_compiler_existence_check(self):
         yield
         self.check_for_compiler_existence = saved
 
+    @contextmanager
+    def enable_compiler_existence_check(self):
+        saved = self.check_for_compiler_existence
+        self.check_for_compiler_existence = True
+        yield
+        self.check_for_compiler_existence = saved
+
     def _valid_virtuals_and_externals(self, spec):
         """Returns a list of candidate virtual dep providers and external
            packages that coiuld be used to concretize a spec.
@@ -303,36 +312,37 @@ def _proper_compiler_style(cspec, aspec):
         assert(other_spec)
 
         # Check if the compiler is already fully specified
-        if (other_compiler and other_compiler.concrete and
+        if other_compiler and other_compiler.concrete:
-                not self.check_for_compiler_existence):
+            if (self.check_for_compiler_existence and not
-            spec.compiler = other_compiler.copy()
+                    _proper_compiler_style(other_compiler, spec.architecture)):
-            return True
-
-        all_compiler_specs = spack.compilers.all_compiler_specs()
-        if not all_compiler_specs:
-            # If compiler existence checking is disabled, then we would have
-            # exited by now if there were sufficient hints to form a full
-            # compiler spec. Therefore even if compiler existence checking is
-            # disabled, compilers must be available at this point because the
-            # available compilers are used to choose a compiler. If compiler
-            # existence checking is enabled then some compiler must exist in
-            # order to complete the spec.
-            raise spack.compilers.NoCompilersError()
-
-        if other_compiler in all_compiler_specs:
-            spec.compiler = other_compiler.copy()
-            if not _proper_compiler_style(spec.compiler, spec.architecture):
                 _compiler_concretization_failure(
-                    spec.compiler, spec.architecture)
+                    other_compiler, spec.architecture)
+            spec.compiler = other_compiler
             return True
 
-        # Filter the compilers into a sorted list based on the compiler_order
+        if other_compiler:  # Another node has abstract compiler information
-        # from spackconfig
+            compiler_list = spack.compilers.find_specs_by_arch(
-        compiler_list = all_compiler_specs if not other_compiler else \
+                other_compiler, spec.architecture
-            spack.compilers.find(other_compiler)
+            )
-        if not compiler_list:
+            if not compiler_list:
-            # No compiler with a satisfactory spec was found
+                # We don't have a matching compiler installed
-            raise UnavailableCompilerVersionError(other_compiler)
+                if not self.check_for_compiler_existence:
+                    # Concretize compiler spec versions as a package to build
+                    cpkg_spec = spack.compilers.pkg_spec_for_compiler(
+                        other_compiler
+                    )
+                    self.concretize_version(cpkg_spec)
+                    spec.compiler.versions = cpkg_spec.versions
+                    return True
+                else:
+                    # No compiler with a satisfactory spec was found
+                    raise UnavailableCompilerVersionError(other_compiler)
+        else:
+            # We have no hints to go by, grab any compiler
+            compiler_list = spack.compilers.all_compiler_specs()
+            if not compiler_list:
+                # Spack has no compilers.
+                raise spack.compilers.NoCompilersError()
 
         # By default, prefer later versions of compilers
         compiler_list = sorted(

lib/spack/spack/package.py

@@ -61,6 +61,7 @@
 from spack.version import Version
 from spack.package_prefs import get_package_dir_permissions, get_package_group
 
+
 """Allowed URL schemes for spack packages."""
 _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
 
@@ -1324,19 +1325,29 @@ def try_install_from_binary_cache(self, explicit):
             self.spec, spack.store.layout, explicit=explicit)
         return True
 
-    def do_install(self,
+    def bootstrap_compiler(self, **kwargs):
-                   keep_prefix=False,
+        """Called by do_install to setup ensure Spack has the right compiler.
-                   keep_stage=False,
+
-                   install_source=False,
+        Checks Spack's compiler configuration for a compiler that
-                   install_deps=True,
+        matches the package spec. If none are configured, installs and
-                   skip_patch=False,
+        adds to the compiler configuration the compiler matching the
-                   verbose=False,
+        CompilerSpec object."""
-                   make_jobs=None,
+        compilers = spack.compilers.compilers_for_spec(
-                   fake=False,
+            self.spec.compiler,
-                   explicit=False,
+            arch_spec=self.spec.architecture
-                   tests=False,
+        )
-                   dirty=None,
+        if not compilers:
-                   **kwargs):
+            dep = spack.compilers.pkg_spec_for_compiler(self.spec.compiler)
+            # concrete CompilerSpec has less info than concrete Spec
+            # concretize as Spec to add that information
+            dep.concretize()
+            dep.package.do_install(**kwargs)
+            spack.compilers.add_compilers_to_config(
+                spack.compilers.find_compilers(dep.prefix)
+            )
+
+    def do_install(self, **kwargs):
+
         """Called by commands to install a package and its dependencies.
 
         Package implementations should override install() to describe
@@ -1363,18 +1374,34 @@ def do_install(self,
             tests (bool or list or set): False to run no tests, True to test
                 all packages, or a list of package names to run tests for some
             dirty (bool): Don't clean the build environment before installing.
+            restage (bool): Force spack to restage the package source.
             force (bool): Install again, even if already installed.
+            use_cache (bool): Install from binary package, if available.
+            stop_at (InstallPhase): last installation phase to be executed
+                (or None)
         """
         if not self.spec.concrete:
             raise ValueError("Can only install concrete packages: %s."
                              % self.spec.name)
 
+        keep_prefix = kwargs.get('keep_prefix', False)
+        keep_stage = kwargs.get('keep_stage', False)
+        install_source = kwargs.get('install_source', False)
+        install_deps = kwargs.get('install_deps', True)
+        skip_patch = kwargs.get('skip_patch', False)
+        verbose = kwargs.get('verbose', False)
+        make_jobs = kwargs.get('make_jobs', None)
+        fake = kwargs.get('fake', False)
+        explicit = kwargs.get('explicit', False)
+        tests = kwargs.get('tests', False)
+        dirty = kwargs.get('dirty', False)
+        restage = kwargs.get('restage', False)
+
         # For external packages the workflow is simplified, and basically
         # consists in module file generation and registration in the DB
         if self.spec.external:
             return self._process_external_package(explicit)
 
-        restage = kwargs.get('restage', False)
         partial = self.check_for_unfinished_installation(keep_prefix, restage)
 
         # Ensure package is not already installed
@@ -1399,21 +1426,22 @@ def do_install(self,
         # First, install dependencies recursively.
         if install_deps:
             tty.debug('Installing {0} dependencies'.format(self.name))
+            dep_kwargs = kwargs.copy()
+            dep_kwargs['explicit'] = False
+            dep_kwargs['install_deps'] = False
             for dep in self.spec.traverse(order='post', root=False):
-                dep.package.do_install(
+                dep.package.do_install(**dep_kwargs)
-                    install_deps=False,
-                    explicit=False,
-                    keep_prefix=keep_prefix,
-                    keep_stage=keep_stage,
-                    install_source=install_source,
-                    fake=fake,
-                    skip_patch=skip_patch,
-                    verbose=verbose,
-                    make_jobs=make_jobs,
-                    tests=tests,
-                    dirty=dirty,
-                    **kwargs)
 
+        # Then, install the compiler if it is not already installed.
+        if install_deps:
+            tty.debug('Boostrapping {0} compiler for {1}'.format(
+                self.spec.compiler, self.name
+            ))
+            comp_kwargs = kwargs.copy()
+            comp_kwargs['explicit'] = False
+            self.bootstrap_compiler(**comp_kwargs)
+
+        # Then, install the package proper
         tty.msg(colorize('@*{Installing} @*g{%s}' % self.name))
 
         if kwargs.get('use_cache', True):

lib/spack/spack/package_prefs.py

@@ -115,7 +115,7 @@ def _packages_config(cls):
         return cls._packages_config_cache
 
     @classmethod
-    def _order_for_package(cls, pkgname, component, vpkg=None, all=True):
+    def order_for_package(cls, pkgname, component, vpkg=None, all=True):
         """Given a package name, sort component (e.g, version, compiler, ...),
            and an optional vpkg, return the list from the packages config.
         """
@@ -151,7 +151,7 @@ def _specs_for_pkg(cls, pkgname, component, vpkg=None):
 
         specs = cls._spec_cache.get(key)
         if specs is None:
-            pkglist = cls._order_for_package(pkgname, component, vpkg)
+            pkglist = cls.order_for_package(pkgname, component, vpkg)
             spec_type = _spec_type(component)
             specs = [spec_type(s) for s in pkglist]
             cls._spec_cache[key] = specs
@@ -166,7 +166,7 @@ def clear_caches(cls):
     @classmethod
     def has_preferred_providers(cls, pkgname, vpkg):
         """Whether specific package has a preferred vpkg providers."""
-        return bool(cls._order_for_package(pkgname, 'providers', vpkg, False))
+        return bool(cls.order_for_package(pkgname, 'providers', vpkg, False))
 
     @classmethod
     def preferred_variants(cls, pkg_name):

lib/spack/spack/schema/config.py

@@ -41,6 +41,7 @@
             'source_cache': {'type': 'string'},
             'misc_cache': {'type': 'string'},
             'verify_ssl': {'type': 'boolean'},
+            'install_missing_compilers': {'type': 'boolean'},
             'debug': {'type': 'boolean'},
             'checksum': {'type': 'boolean'},
             'locks': {'type': 'boolean'},

lib/spack/spack/test/concretize.py

@@ -130,9 +130,11 @@ def test_concretize_with_restricted_virtual(self):
         concrete = check_concretize('mpileaks   ^mpich2@1.3.1:1.4')
         assert concrete['mpich2'].satisfies('mpich2@1.3.1:1.4')
 
-    def test_concretize_disable_compiler_existence_check(self):
+    def test_concretize_enable_disable_compiler_existence_check(self):
-        with pytest.raises(spack.concretize.UnavailableCompilerVersionError):
+        with spack.concretize.concretizer.enable_compiler_existence_check():
-            check_concretize('dttop %gcc@100.100')
+            with pytest.raises(
+                    spack.concretize.UnavailableCompilerVersionError):
+                check_concretize('dttop %gcc@100.100')
 
         with spack.concretize.concretizer.disable_compiler_existence_check():
             spec = check_concretize('dttop %gcc@100.100')
@@ -267,10 +269,13 @@ def test_concretize_two_virtuals_with_dual_provider_and_a_conflict(
         with pytest.raises(spack.spec.MultipleProviderError):
             s.concretize()
 
-    def test_no_matching_compiler_specs(self):
+    def test_no_matching_compiler_specs(self, mock_config):
-        s = Spec('a %gcc@0.0.0')
+        # only relevant when not building compilers as needed
-        with pytest.raises(spack.concretize.UnavailableCompilerVersionError):
+        with spack.concretize.concretizer.enable_compiler_existence_check():
-            s.concretize()
+            s = Spec('a %gcc@0.0.0')
+            with pytest.raises(
+                    spack.concretize.UnavailableCompilerVersionError):
+                s.concretize()
 
     def test_no_compilers_for_arch(self):
         s = Spec('a arch=linux-rhel0-x86_64')