zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create option to build compilers as needed (#10761)

Browse Source 
* Create option to build missing compilers and add them to config before installing packages that use them
* Clean up kwarg passing for do_install, put compiler bootstrapping in separate method
This commit is contained in:
Greg Becker
2019-03-07 17:30:48 -08:00
committed by GitHub
parent 0d07dea2eb
commit f4d4322a41
7 changed files with 137 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
etc/spack/defaults/config.yaml
View File

@@ -66,6 +66,12 @@ config:
verify_ssl: true
# If set to true, Spack will attempt to build any compiler on the spec
# that is not already available. If set to False, Spack will only use
# compilers already configured in compilers.yaml
install_missing_compilers: False
# If set to true, Spack will always check checksums after downloading
# archives. If false, Spack skips the checksum step.
checksum: true

22
lib/spack/spack/compilers/__init__.py
View File

@@ -28,6 +28,17 @@
#: cache of compilers constructed from config data, keyed by config entry id.
_compiler_cache = {}
_compiler_to_pkg = {
'clang': 'llvm+clang'
}
def pkg_spec_for_compiler(cspec):
"""Return the spec of the package that provides the compiler."""
spec_str = '%s@%s' % (_compiler_to_pkg.get(cspec.name, cspec.name),
cspec.versions)
return spack.spec.Spec(spec_str)
def _auto_compiler_spec(function):
def converter(cspec_like, *args, **kwargs):
@@ -203,6 +214,17 @@ def find(compiler_spec, scope=None, init_config=True):
if c.satisfies(compiler_spec)]
@_auto_compiler_spec
def find_specs_by_arch(compiler_spec, arch_spec, scope=None, init_config=True):
"""Return specs of available compilers that match the supplied
compiler spec. Return an empty list if nothing found."""
return [c.spec for c in compilers_for_spec(compiler_spec,
arch_spec,
scope,
True,
init_config)]
def all_compilers(scope=None):
config = get_compiler_config(scope)
compilers = list()

66
lib/spack/spack/concretize.py
View File

@@ -28,6 +28,7 @@
import spack.compilers
import spack.architecture
import spack.error
from spack.config import config
from spack.version import ver, Version, VersionList, VersionRange
from spack.package_prefs import PackagePrefs, spec_externals, is_spec_buildable
@@ -47,7 +48,8 @@ class Concretizer(object):
def __init__(self):
# controls whether we check that compiler versions actually exist
# during concretization. Used for testing and for mirror creation
self.check_for_compiler_existence = True
self.check_for_compiler_existence = not config.get(
'config:install_missing_compilers', False)
@contextmanager
def disable_compiler_existence_check(self):
@@ -56,6 +58,13 @@ def disable_compiler_existence_check(self):
yield
self.check_for_compiler_existence = saved
@contextmanager
def enable_compiler_existence_check(self):
saved = self.check_for_compiler_existence
self.check_for_compiler_existence = True
yield
self.check_for_compiler_existence = saved
def _valid_virtuals_and_externals(self, spec):
"""Returns a list of candidate virtual dep providers and external
packages that coiuld be used to concretize a spec.
@@ -303,36 +312,37 @@ def _proper_compiler_style(cspec, aspec):
assert(other_spec)
# Check if the compiler is already fully specified
if (other_compiler and other_compiler.concrete and
not self.check_for_compiler_existence):
spec.compiler = other_compiler.copy()
return True
all_compiler_specs = spack.compilers.all_compiler_specs()
if not all_compiler_specs:
# If compiler existence checking is disabled, then we would have
# exited by now if there were sufficient hints to form a full
# compiler spec. Therefore even if compiler existence checking is
# disabled, compilers must be available at this point because the
# available compilers are used to choose a compiler. If compiler
# existence checking is enabled then some compiler must exist in
# order to complete the spec.
raise spack.compilers.NoCompilersError()
if other_compiler in all_compiler_specs:
spec.compiler = other_compiler.copy()
if not _proper_compiler_style(spec.compiler, spec.architecture):
if other_compiler and other_compiler.concrete:
if (self.check_for_compiler_existence and not
_proper_compiler_style(other_compiler, spec.architecture)):
_compiler_concretization_failure(
spec.compiler, spec.architecture)
other_compiler, spec.architecture)
spec.compiler = other_compiler
return True
# Filter the compilers into a sorted list based on the compiler_order
# from spackconfig
compiler_list = all_compiler_specs if not other_compiler else \
spack.compilers.find(other_compiler)
if not compiler_list:
# No compiler with a satisfactory spec was found
raise UnavailableCompilerVersionError(other_compiler)
if other_compiler: # Another node has abstract compiler information
compiler_list = spack.compilers.find_specs_by_arch(
other_compiler, spec.architecture
)
if not compiler_list:
# We don't have a matching compiler installed
if not self.check_for_compiler_existence:
# Concretize compiler spec versions as a package to build
cpkg_spec = spack.compilers.pkg_spec_for_compiler(
other_compiler
)
self.concretize_version(cpkg_spec)
spec.compiler.versions = cpkg_spec.versions
return True
else:
# No compiler with a satisfactory spec was found
raise UnavailableCompilerVersionError(other_compiler)
else:
# We have no hints to go by, grab any compiler
compiler_list = spack.compilers.all_compiler_specs()
if not compiler_list:
# Spack has no compilers.
raise spack.compilers.NoCompilersError()
# By default, prefer later versions of compilers
compiler_list = sorted(

82
lib/spack/spack/package.py
View File

@@ -61,6 +61,7 @@
from spack.version import Version
from spack.package_prefs import get_package_dir_permissions, get_package_group
"""Allowed URL schemes for spack packages."""
_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
@@ -1324,19 +1325,29 @@ def try_install_from_binary_cache(self, explicit):
self.spec, spack.store.layout, explicit=explicit)
return True
def do_install(self,
keep_prefix=False,
keep_stage=False,
install_source=False,
install_deps=True,
skip_patch=False,
verbose=False,
make_jobs=None,
fake=False,
explicit=False,
tests=False,
dirty=None,
**kwargs):
def bootstrap_compiler(self, **kwargs):
"""Called by do_install to setup ensure Spack has the right compiler.
Checks Spack's compiler configuration for a compiler that
matches the package spec. If none are configured, installs and
adds to the compiler configuration the compiler matching the
CompilerSpec object."""
compilers = spack.compilers.compilers_for_spec(
self.spec.compiler,
arch_spec=self.spec.architecture
)
if not compilers:
dep = spack.compilers.pkg_spec_for_compiler(self.spec.compiler)
# concrete CompilerSpec has less info than concrete Spec
# concretize as Spec to add that information
dep.concretize()
dep.package.do_install(**kwargs)
spack.compilers.add_compilers_to_config(
spack.compilers.find_compilers(dep.prefix)
)
def do_install(self, **kwargs):
"""Called by commands to install a package and its dependencies.
Package implementations should override install() to describe
@@ -1363,18 +1374,34 @@ def do_install(self,
tests (bool or list or set): False to run no tests, True to test
all packages, or a list of package names to run tests for some
dirty (bool): Don't clean the build environment before installing.
restage (bool): Force spack to restage the package source.
force (bool): Install again, even if already installed.
use_cache (bool): Install from binary package, if available.
stop_at (InstallPhase): last installation phase to be executed
(or None)
"""
if not self.spec.concrete:
raise ValueError("Can only install concrete packages: %s."
% self.spec.name)
keep_prefix = kwargs.get('keep_prefix', False)
keep_stage = kwargs.get('keep_stage', False)
install_source = kwargs.get('install_source', False)
install_deps = kwargs.get('install_deps', True)
skip_patch = kwargs.get('skip_patch', False)
verbose = kwargs.get('verbose', False)
make_jobs = kwargs.get('make_jobs', None)
fake = kwargs.get('fake', False)
explicit = kwargs.get('explicit', False)
tests = kwargs.get('tests', False)
dirty = kwargs.get('dirty', False)
restage = kwargs.get('restage', False)
# For external packages the workflow is simplified, and basically
# consists in module file generation and registration in the DB
if self.spec.external:
return self._process_external_package(explicit)
restage = kwargs.get('restage', False)
partial = self.check_for_unfinished_installation(keep_prefix, restage)
# Ensure package is not already installed
@@ -1399,21 +1426,22 @@ def do_install(self,
# First, install dependencies recursively.
if install_deps:
tty.debug('Installing {0} dependencies'.format(self.name))
dep_kwargs = kwargs.copy()
dep_kwargs['explicit'] = False
dep_kwargs['install_deps'] = False
for dep in self.spec.traverse(order='post', root=False):
dep.package.do_install(
install_deps=False,
explicit=False,
keep_prefix=keep_prefix,
keep_stage=keep_stage,
install_source=install_source,
fake=fake,
skip_patch=skip_patch,
verbose=verbose,
make_jobs=make_jobs,
tests=tests,
dirty=dirty,
**kwargs)
dep.package.do_install(**dep_kwargs)
# Then, install the compiler if it is not already installed.
if install_deps:
tty.debug('Boostrapping {0} compiler for {1}'.format(
self.spec.compiler, self.name
))
comp_kwargs = kwargs.copy()
comp_kwargs['explicit'] = False
self.bootstrap_compiler(**comp_kwargs)
# Then, install the package proper
tty.msg(colorize('@*{Installing} @*g{%s}' % self.name))
if kwargs.get('use_cache', True):

6
lib/spack/spack/package_prefs.py
View File

@@ -115,7 +115,7 @@ def _packages_config(cls):
return cls._packages_config_cache
@classmethod
def _order_for_package(cls, pkgname, component, vpkg=None, all=True):
def order_for_package(cls, pkgname, component, vpkg=None, all=True):
"""Given a package name, sort component (e.g, version, compiler, ...),
and an optional vpkg, return the list from the packages config.
"""
@@ -151,7 +151,7 @@ def _specs_for_pkg(cls, pkgname, component, vpkg=None):
specs = cls._spec_cache.get(key)
if specs is None:
pkglist = cls._order_for_package(pkgname, component, vpkg)
pkglist = cls.order_for_package(pkgname, component, vpkg)
spec_type = _spec_type(component)
specs = [spec_type(s) for s in pkglist]
cls._spec_cache[key] = specs
@@ -166,7 +166,7 @@ def clear_caches(cls):
@classmethod
def has_preferred_providers(cls, pkgname, vpkg):
"""Whether specific package has a preferred vpkg providers."""
return bool(cls._order_for_package(pkgname, 'providers', vpkg, False))
return bool(cls.order_for_package(pkgname, 'providers', vpkg, False))
@classmethod
def preferred_variants(cls, pkg_name):

1
lib/spack/spack/schema/config.py
View File

@@ -41,6 +41,7 @@
'source_cache': {'type': 'string'},
'misc_cache': {'type': 'string'},
'verify_ssl': {'type': 'boolean'},
'install_missing_compilers': {'type': 'boolean'},
'debug': {'type': 'boolean'},
'checksum': {'type': 'boolean'},
'locks': {'type': 'boolean'},

19
lib/spack/spack/test/concretize.py
View File

@@ -130,9 +130,11 @@ def test_concretize_with_restricted_virtual(self):
concrete = check_concretize('mpileaks ^mpich2@1.3.1:1.4')
assert concrete['mpich2'].satisfies('mpich2@1.3.1:1.4')
def test_concretize_disable_compiler_existence_check(self):
with pytest.raises(spack.concretize.UnavailableCompilerVersionError):
check_concretize('dttop %gcc@100.100')
def test_concretize_enable_disable_compiler_existence_check(self):
with spack.concretize.concretizer.enable_compiler_existence_check():
with pytest.raises(
spack.concretize.UnavailableCompilerVersionError):
check_concretize('dttop %gcc@100.100')
with spack.concretize.concretizer.disable_compiler_existence_check():
spec = check_concretize('dttop %gcc@100.100')
@@ -267,10 +269,13 @@ def test_concretize_two_virtuals_with_dual_provider_and_a_conflict(
with pytest.raises(spack.spec.MultipleProviderError):
s.concretize()
def test_no_matching_compiler_specs(self):
s = Spec('a %gcc@0.0.0')
with pytest.raises(spack.concretize.UnavailableCompilerVersionError):
s.concretize()
def test_no_matching_compiler_specs(self, mock_config):
# only relevant when not building compilers as needed
with spack.concretize.concretizer.enable_compiler_existence_check():
s = Spec('a %gcc@0.0.0')
with pytest.raises(
spack.concretize.UnavailableCompilerVersionError):
s.concretize()
def test_no_compilers_for_arch(self):
s = Spec('a arch=linux-rhel0-x86_64')