From 285926cb69b2858e3e2dde4bf5200e8ada111a4c Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Mon, 11 Nov 2024 12:42:41 +0100 Subject: [PATCH] Overhaul the `spack compiler` command This reverts commit 2c47dddbc16e02c7027f26206874eede33e124f5. Now, `spack compiler` writes by default in packages.yaml. Entries in old `compilers.yaml` are converted to external specs as a way to support legacy configuration. Since this operation is expensive, an environment variable can be used to enforce the deprecation of `compiler.yaml`. The --mixed-toolchain option has been deprecated, since it stops to make sense once compiler are treated as nodes. --- lib/spack/spack/cmd/compiler.py | 113 ++-- lib/spack/spack/compilers/__init__.py | 940 ++++++++------------------ lib/spack/spack/config.py | 3 + lib/spack/spack/cray_manifest.py | 4 +- lib/spack/spack/package_base.py | 3 +- 5 files changed, 337 insertions(+), 726 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 8b83542f74e..fb9aa11eb25 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -4,6 +4,7 @@ import argparse import sys +import warnings import llnl.util.tty as tty from llnl.util.lang import index_by @@ -34,13 +35,13 @@ def setup_parser(subparser): "--mixed-toolchain", action="store_true", default=sys.platform == "darwin", - help="Allow mixed toolchains (for example: clang, clang++, gfortran)", + help="(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)", ) mixed_toolchain_group.add_argument( "--no-mixed-toolchain", action="store_false", dest="mixed_toolchain", - help="Do not allow mixed toolchains (for example: clang, clang++, gfortran)", + help="(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)", ) find_parser.add_argument("add_paths", nargs=argparse.REMAINDER) find_parser.add_argument( @@ -79,19 +80,22 @@ def compiler_find(args): """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration. """ + if args.mixed_toolchain: + warnings.warn( + "The '--mixed-toolchain' option has been deprecated in Spack v0.23, and currently " + "has no effect. The option will be removed in Spack v0.25" + ) + paths = args.add_paths or None new_compilers = spack.compilers.find_compilers( - path_hints=paths, - scope=args.scope, - mixed_toolchain=args.mixed_toolchain, - max_workers=args.jobs, + path_hints=paths, scope=args.scope, max_workers=args.jobs ) if new_compilers: n = len(new_compilers) s = "s" if n > 1 else "" - filename = spack.config.CONFIG.get_config_filename(args.scope, "compilers") + filename = spack.config.CONFIG.get_config_filename(args.scope, "packages") tty.msg(f"Added {n:d} new compiler{s} to {filename}") - compiler_strs = sorted(f"{c.spec.name}@{c.spec.version}" for c in new_compilers) + compiler_strs = sorted(f"{spec.name}@{spec.versions}" for spec in new_compilers) colify(reversed(compiler_strs), indent=4) else: tty.msg("Found no new compilers") @@ -100,52 +104,69 @@ def compiler_find(args): def compiler_remove(args): - compiler_spec = spack.spec.CompilerSpec(args.compiler_spec) - candidate_compilers = spack.compilers.compilers_for_spec(compiler_spec, scope=args.scope) + remover = spack.compilers.CompilerRemover(spack.config.CONFIG) + candidates = remover.mark_compilers(match=args.compiler_spec, scope=args.scope) + if not candidates: + tty.die(f"No compiler matches '{args.compiler_spec}'") - if not candidate_compilers: - tty.die("No compilers match spec %s" % compiler_spec) + compiler_strs = reversed(sorted(f"{spec.name}@{spec.versions}" for spec in candidates)) - if not args.all and len(candidate_compilers) > 1: - tty.error(f"Multiple compilers match spec {compiler_spec}. Choose one:") - colify(reversed(sorted([c.spec.display_str for c in candidate_compilers])), indent=4) - tty.msg("Or, use `spack compiler remove -a` to remove all of them.") + if not args.all and len(candidates) > 1: + tty.error(f"multiple compilers match the spec '{args.compiler_spec}':") + print() + colify(compiler_strs, indent=4) + print() + print( + "Either use a stricter spec to select only one, or use `spack compiler remove -a`" + " to remove all of them." + ) sys.exit(1) - for current_compiler in candidate_compilers: - spack.compilers.remove_compiler_from_config(current_compiler.spec, scope=args.scope) - tty.msg(f"{current_compiler.spec.display_str} has been removed") + remover.flush() + tty.msg("The following compilers have been removed:") + print() + colify(compiler_strs, indent=4) + print() def compiler_info(args): """Print info about all compilers matching a spec.""" - cspec = spack.spec.CompilerSpec(args.compiler_spec) - compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) + query = spack.spec.Spec(args.compiler_spec) + all_compilers = spack.compilers.all_compilers(scope=args.scope, init_config=False) + + compilers = [x for x in all_compilers if x.satisfies(query)] if not compilers: - tty.die("No compilers match spec %s" % cspec) + tty.die(f"No compilers match spec {query.cformat()}") else: for c in compilers: - print(c.spec.display_str + ":") - print("\tpaths:") - for cpath in ["cc", "cxx", "f77", "fc"]: - print("\t\t%s = %s" % (cpath, getattr(c, cpath, None))) - if c.flags: - print("\tflags:") - for flag, flag_value in c.flags.items(): - print("\t\t%s = %s" % (flag, flag_value)) - if len(c.environment) != 0: - if len(c.environment.get("set", {})) != 0: - print("\tenvironment:") - print("\t set:") - for key, value in c.environment["set"].items(): - print("\t %s = %s" % (key, value)) - if c.extra_rpaths: - print("\tExtra rpaths:") - for extra_rpath in c.extra_rpaths: - print("\t\t%s" % extra_rpath) - print("\tmodules = %s" % c.modules) - print("\toperating system = %s" % c.operating_system) + print(f"{c.cformat()}:") + print(f" prefix: {c.external_path}") + extra_attributes = getattr(c, "extra_attributes", {}) + if "compilers" in extra_attributes: + print(" compilers:") + for language, exe in extra_attributes.get("compilers", {}).items(): + print(f" {language}: {exe}") + if "flags" in extra_attributes: + print(" flags:") + for flag, flag_value in extra_attributes["flags"].items(): + print(f" {flag} = {flag_value}") + # FIXME (compiler as nodes): recover this printing + # if "environment" in extra_attributes: + # if len(c.environment.get("set", {})) != 0: + # print("\tenvironment:") + # print("\t set:") + # for key, value in c.environment["set"].items(): + # print("\t %s = %s" % (key, value)) + if "extra_rpaths" in extra_attributes: + print(" extra rpaths:") + for extra_rpath in extra_attributes["extra_rpaths"]: + print(f" {extra_rpath}") + if getattr(c, "external_modules", []): + print(" modules: ") + for module in c.external_modules: + print(f" {module}") + print() def compiler_list(args): @@ -162,7 +183,7 @@ def compiler_list(args): tty.msg(msg) return - index = index_by(compilers, lambda c: (c.spec.name, c.operating_system, c.target)) + index = index_by(compilers, spack.compilers.name_os_target) tty.msg("Available compilers") @@ -181,10 +202,10 @@ def compiler_list(args): name, os, target = key os_str = os if target: - os_str += "-%s" % target - cname = "%s{%s} %s" % (spack.spec.COMPILER_COLOR, name, os_str) + os_str += f"-{target}" + cname = f"{spack.spec.COMPILER_COLOR}{{{name}}} {os_str}" tty.hline(colorize(cname), char="-") - colify(reversed(sorted(c.spec.display_str for c in compilers))) + colify(reversed(sorted(c.format("{name}@{version}") for c in compilers))) def compiler(parser, args): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 44c084ac8ab..8c6959bbfb1 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -1,16 +1,14 @@ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - -"""This module contains functions related to finding compilers on the -system and configuring Spack to use multiple compilers. +"""This module contains functions related to finding compilers on the system, +and configuring Spack to use multiple compilers. """ -import importlib import os import re import sys import warnings -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple import archspec.cpu @@ -18,7 +16,6 @@ import llnl.util.lang import llnl.util.tty as tty -import spack.compiler import spack.config import spack.error import spack.paths @@ -27,30 +24,7 @@ import spack.spec from spack.operating_systems import windows_os from spack.util.environment import get_path -from spack.util.naming import mod_to_class -_other_instance_vars = [ - "modules", - "operating_system", - "environment", - "implicit_rpaths", - "extra_rpaths", -] - -# TODO: Caches at module level make it difficult to mock configurations in -# TODO: unit tests. It might be worth reworking their implementation. -#: cache of compilers constructed from config data, keyed by config entry id. -_compiler_cache: Dict[str, "spack.compiler.Compiler"] = {} - -_compiler_to_pkg = { - "clang": "llvm+clang", - "oneapi": "intel-oneapi-compilers", - "rocmcc": "llvm-amdgpu", - "intel@2020:": "intel-oneapi-compilers-classic", - "arm": "acfl", -} - -# TODO: generating this from the previous dict causes docs errors package_name_to_compiler_name = { "llvm": "clang", "intel-oneapi-compilers": "oneapi", @@ -64,186 +38,35 @@ COMPILER_TAG = "compiler" -def pkg_spec_for_compiler(cspec): - """Return the spec of the package that provides the compiler.""" - for spec, package in _compiler_to_pkg.items(): - if cspec.satisfies(spec): - spec_str = "%s@%s" % (package, cspec.versions) - break - else: - spec_str = str(cspec) - return spack.spec.parse_with_version_concrete(spec_str) - - -def _auto_compiler_spec(function): - def converter(cspec_like, *args, **kwargs): - if not isinstance(cspec_like, spack.spec.Spec): - cspec_like = spack.spec.Spec(cspec_like) - return function(cspec_like, *args, **kwargs) - - return converter - - -def _to_dict(compiler): - """Return a dict version of compiler suitable to insert in YAML.""" - return {"compiler": compiler.to_dict()} - - -def get_compiler_config( - configuration: "spack.config.Configuration", - *, - scope: Optional[str] = None, - init_config: bool = False, -) -> List[Dict]: - """Return the compiler configuration for the specified architecture.""" - config = configuration.get("compilers", scope=scope) or [] - if config or not init_config: - return config - - merged_config = configuration.get("compilers") - if merged_config: - # Config is empty for this scope - # Do not init config because there is a non-empty scope - return config - - find_compilers(scope=scope) - config = configuration.get("compilers", scope=scope) - return config - - -def get_compiler_config_from_packages( - configuration: "spack.config.Configuration", *, scope: Optional[str] = None -) -> List[Dict]: - """Return the compiler configuration from packages.yaml""" - packages_yaml = configuration.get("packages", scope=scope) - return CompilerConfigFactory.from_packages_yaml(packages_yaml) - - def compiler_config_files(): - config_files = list() - config = spack.config.CONFIG - for scope in config.writable_scopes: + config_files = [] + configuration = spack.config.CONFIG + for scope in configuration.writable_scopes: name = scope.name - compiler_config = config.get("compilers", scope=name) + + from_packages_yaml = CompilerFactory.from_packages_yaml(configuration, scope=name) + if from_packages_yaml: + config_files.append(configuration.get_config_filename(name, "packages")) + + compiler_config = configuration.get("compilers", scope=name) if compiler_config: - config_files.append(config.get_config_filename(name, "compilers")) - compiler_config_from_packages = get_compiler_config_from_packages(config, scope=name) - if compiler_config_from_packages: - config_files.append(config.get_config_filename(name, "packages")) + config_files.append(configuration.get_config_filename(name, "compilers")) + return config_files -def add_compilers_to_config(compilers, scope=None): - """Add compilers to the config for the specified architecture. - - Arguments: - compilers: a list of Compiler objects. - scope: configuration scope to modify. - """ - compiler_config = get_compiler_config(configuration=spack.config.CONFIG, scope=scope) - for compiler in compilers: - if not compiler.cc: - tty.debug(f"{compiler.spec} does not have a C compiler") - if not compiler.cxx: - tty.debug(f"{compiler.spec} does not have a C++ compiler") - if not compiler.f77: - tty.debug(f"{compiler.spec} does not have a Fortran77 compiler") - if not compiler.fc: - tty.debug(f"{compiler.spec} does not have a Fortran compiler") - compiler_config.append(_to_dict(compiler)) - spack.config.set("compilers", compiler_config, scope=scope) - - -@_auto_compiler_spec -def remove_compiler_from_config(compiler_spec, scope=None): - """Remove compilers from configuration by spec. - - If scope is None, all the scopes are searched for removal. - - Arguments: - compiler_spec: compiler to be removed - scope: configuration scope to modify - """ - candidate_scopes = [scope] - if scope is None: - candidate_scopes = spack.config.CONFIG.scopes.keys() - - removal_happened = False - for current_scope in candidate_scopes: - removal_happened |= _remove_compiler_from_scope(compiler_spec, scope=current_scope) - - msg = "`spack compiler remove` will not remove compilers defined in packages.yaml" - msg += "\nTo remove these compilers, either edit the config or use `spack external remove`" - tty.debug(msg) - return removal_happened - - -def _remove_compiler_from_scope(compiler_spec, scope): - """Removes a compiler from a specific configuration scope. - - Args: - compiler_spec: compiler to be removed - scope: configuration scope under consideration - - Returns: - True if one or more compiler entries were actually removed, False otherwise - """ - assert scope is not None, "a specific scope is needed when calling this function" - compiler_config = get_compiler_config(configuration=spack.config.CONFIG, scope=scope) - filtered_compiler_config = [ - compiler_entry - for compiler_entry in compiler_config - if not spack.spec.parse_with_version_concrete( - compiler_entry["compiler"]["spec"], compiler=True - ).satisfies(compiler_spec) - ] - - if len(filtered_compiler_config) == len(compiler_config): - return False - - # We need to preserve the YAML type for comments, hence we are copying the - # items in the list that has just been retrieved - compiler_config[:] = filtered_compiler_config - spack.config.CONFIG.set("compilers", compiler_config, scope=scope) - return True - - -def all_compilers_config( - configuration: "spack.config.Configuration", - *, - scope: Optional[str] = None, - init_config: bool = True, -) -> List["spack.compiler.Compiler"]: - """Return a set of specs for all the compiler versions currently - available to build with. These are instances of CompilerSpec. - """ - from_packages_yaml = get_compiler_config_from_packages(configuration, scope=scope) - if from_packages_yaml: - init_config = False - from_compilers_yaml = get_compiler_config(configuration, scope=scope, init_config=init_config) - - result = from_compilers_yaml + from_packages_yaml - # Dedupe entries by the compiler they represent - # If the entry is invalid, treat it as unique for deduplication - key = lambda c: _compiler_from_config_entry(c["compiler"] or id(c)) - return list(llnl.util.lang.dedupe(result, key=key)) - - -def all_compiler_specs(scope=None, init_config=True): - # Return compiler specs from the merged config. - return [ - spack.spec.parse_with_version_concrete(s["compiler"]["spec"]) - for s in all_compilers_config(spack.config.CONFIG, scope=scope, init_config=init_config) - ] +def add_compiler_to_config(compiler, scope=None) -> None: + """Add a Compiler object to the configuration, at the required scope.""" + # FIXME (compiler as nodes): still needed to read Cray manifest + raise NotImplementedError("'add_compiler_to_config' node implemented yet.") def find_compilers( path_hints: Optional[List[str]] = None, *, scope: Optional[str] = None, - mixed_toolchain: bool = False, max_workers: Optional[int] = None, -) -> List["spack.compiler.Compiler"]: +) -> List["spack.spec.Spec"]: """Searches for compiler in the paths given as argument. If any new compiler is found, the configuration is updated, and the list of new compiler objects is returned. @@ -251,14 +74,8 @@ def find_compilers( path_hints: list of path hints where to look for. A sensible default based on the ``PATH`` environment variable will be used if the value is None scope: configuration scope to modify - mixed_toolchain: allow mixing compilers from different toolchains if otherwise missing for - a certain language max_workers: number of processes used to search for compilers """ - import spack.detection - - known_compilers = set(all_compilers(init_config=False)) - if path_hints is None: path_hints = get_path("PATH") default_paths = fs.search_paths_for_executables(*path_hints) @@ -270,44 +87,9 @@ def find_compilers( compiler_pkgs, path_hints=default_paths, max_workers=max_workers ) - valid_compilers = {} - for name, detected in detected_packages.items(): - compilers = [x for x in detected if CompilerConfigFactory.from_external_spec(x)] - if not compilers: - continue - valid_compilers[name] = compilers - - def _has_fortran_compilers(x): - if "compilers" not in x.extra_attributes: - return False - - return "fortran" in x.extra_attributes["compilers"] - - if mixed_toolchain: - gccs = [x for x in valid_compilers.get("gcc", []) if _has_fortran_compilers(x)] - if gccs: - best_gcc = sorted( - gccs, key=lambda x: spack.spec.parse_with_version_concrete(x).version - )[-1] - gfortran = best_gcc.extra_attributes["compilers"]["fortran"] - for name in ("llvm", "apple-clang"): - if name not in valid_compilers: - continue - candidates = valid_compilers[name] - for candidate in candidates: - if _has_fortran_compilers(candidate): - continue - candidate.extra_attributes["compilers"]["fortran"] = gfortran - - new_compilers = [] - for name, detected in valid_compilers.items(): - for config in CompilerConfigFactory.from_specs(detected): - c = _compiler_from_config_entry(config["compiler"]) - if c in known_compilers: - continue - new_compilers.append(c) - - add_compilers_to_config(new_compilers, scope=scope) + new_compilers = spack.detection.update_configuration( + detected_packages, buildable=True, scope=scope + ) return new_compilers @@ -315,6 +97,7 @@ def select_new_compilers(compilers, scope=None): """Given a list of compilers, remove those that are already defined in the configuration. """ + # FIXME (compiler as nodes): still needed to read Cray manifest compilers_not_in_config = [] for c in compilers: arch_spec = spack.spec.ArchSpec((None, c.operating_system, c.target)) @@ -328,343 +111,186 @@ def select_new_compilers(compilers, scope=None): def supported_compilers() -> List[str]: - """Return a set of names of compilers supported by Spack. - - See available_compilers() to get a list of all the available - versions of supported compilers. - """ - # Hack to be able to call the compiler `apple-clang` while still - # using a valid python name for the module - return sorted(all_compiler_names()) + """Returns all the currently supported compiler packages""" + return sorted(spack.repo.PATH.packages_with_tags(COMPILER_TAG)) -def supported_compilers_for_host_platform() -> List[str]: - """Return a set of compiler class objects supported by Spack - that are also supported by the current host platform - """ - host_plat = spack.platforms.real_host() - return supported_compilers_for_platform(host_plat) - - -def supported_compilers_for_platform(platform: "spack.platforms.Platform") -> List[str]: - """Return a set of compiler class objects supported by Spack - that are also supported by the provided platform +def all_compilers( + scope: Optional[str] = None, init_config: bool = True +) -> List["spack.spec.Spec"]: + """Returns all the compilers from the current global configuration. Args: - platform (str): string representation of platform - for which compiler compatability should be determined + scope: configuration scope from which to extract the compilers. If None, the merged + configuration is used. + init_config: if True, search for compilers if none is found in configuration. """ - return [ - name - for name in supported_compilers() - if class_for_compiler_name(name).is_supported_on_platform(platform) - ] + compilers = all_compilers_from(configuration=spack.config.CONFIG, scope=scope) + if not compilers and init_config: + find_compilers(scope=scope) + compilers = all_compilers_from(configuration=spack.config.CONFIG, scope=scope) -def all_compiler_names() -> List[str]: - def replace_apple_clang(name): - return name if name != "apple_clang" else "apple-clang" - - return [replace_apple_clang(name) for name in all_compiler_module_names()] - - -@llnl.util.lang.memoized -def all_compiler_module_names() -> List[str]: - return list(llnl.util.lang.list_modules(spack.paths.compilers_path)) - - -@_auto_compiler_spec -def supported(compiler_spec): - """Test if a particular compiler is supported.""" - return compiler_spec.name in supported_compilers() - - -@_auto_compiler_spec -def find(compiler_spec, scope=None, init_config=True): - """Return specs of available compilers that match the supplied - compiler spec. Return an empty list if nothing found.""" - return [c for c in all_compiler_specs(scope, init_config) if c.satisfies(compiler_spec)] - - -@_auto_compiler_spec -def find_specs_by_arch(compiler_spec, arch_spec, scope=None, init_config=True): - """Return specs of available compilers that match the supplied - compiler spec. Return an empty list if nothing found.""" - return [ - c.spec - for c in compilers_for_spec( - compiler_spec, arch_spec=arch_spec, scope=scope, init_config=init_config - ) - ] - - -def all_compilers(scope=None, init_config=True): - return all_compilers_from( - configuration=spack.config.CONFIG, scope=scope, init_config=init_config - ) - - -def all_compilers_from(configuration, scope=None, init_config=True): - compilers = [] - for items in all_compilers_config( - configuration=configuration, scope=scope, init_config=init_config - ): - items = items["compiler"] - compiler = _compiler_from_config_entry(items) # can be None in error case - if compiler: - compilers.append(compiler) return compilers -@_auto_compiler_spec +def all_compilers_from( + configuration: "spack.config.ConfigurationType", scope: Optional[str] = None +) -> List["spack.spec.Spec"]: + """Returns all the compilers from the current global configuration. + + Args: + configuration: configuration to be queried + scope: configuration scope from which to extract the compilers. If None, the merged + configuration is used. + """ + compilers = CompilerFactory.from_packages_yaml(configuration, scope=scope) + + if os.environ.get("SPACK_EXPERIMENTAL_DEPRECATE_COMPILERS_YAML") != "1": + legacy_compilers = CompilerFactory.from_compilers_yaml(configuration, scope=scope) + if legacy_compilers: + # FIXME (compiler as nodes): write how to update the file. Maybe an ad-hoc command + warnings.warn( + "Some compilers are still defined in 'compilers.yaml', which has been deprecated " + "in v0.23. Those configuration files will be ignored from Spack v0.25.\n" + ) + for legacy in legacy_compilers: + if not any(c.satisfies(f"{legacy.name}@{legacy.versions}") for c in compilers): + compilers.append(legacy) + + return compilers + + +class CompilerRemover: + """Removes compiler from configuration.""" + + def __init__(self, configuration: "spack.config.ConfigurationType") -> None: + self.configuration = configuration + self.marked_packages_yaml: List[Tuple[str, Any]] = [] + self.marked_compilers_yaml: List[Tuple[str, Any]] = [] + + def mark_compilers( + self, *, match: str, scope: Optional[str] = None + ) -> List["spack.spec.Spec"]: + """Marks compilers to be removed in configuration, and returns a corresponding list + of specs. + + Args: + match: constraint that the compiler must match to be removed. + scope: scope where to remove the compiler. If None, all writeable scopes are checked. + """ + self.marked_packages_yaml = [] + self.marked_compilers_yaml = [] + candidate_scopes = [scope] + if scope is None: + candidate_scopes = [x.name for x in self.configuration.writable_scopes] + + all_removals = self._mark_in_packages_yaml(match, candidate_scopes) + all_removals.extend(self._mark_in_compilers_yaml(match, candidate_scopes)) + + return all_removals + + def _mark_in_packages_yaml(self, match, candidate_scopes): + compiler_package_names = supported_compilers() + all_removals = [] + for current_scope in candidate_scopes: + packages_yaml = self.configuration.get("packages", scope=current_scope) + if not packages_yaml: + continue + + removed_from_scope = [] + for name, entry in packages_yaml.items(): + if name not in compiler_package_names: + continue + + externals_config = entry.get("externals", None) + if not externals_config: + continue + + def _partition_match(external_yaml): + s = CompilerFactory.from_external_yaml(external_yaml) + return not s.satisfies(match) + + to_keep, to_remove = llnl.util.lang.stable_partition( + externals_config, _partition_match + ) + if not to_remove: + continue + + removed_from_scope.extend(to_remove) + entry["externals"] = to_keep + + if not removed_from_scope: + continue + + self.marked_packages_yaml.append((current_scope, packages_yaml)) + all_removals.extend( + [CompilerFactory.from_external_yaml(x) for x in removed_from_scope] + ) + return all_removals + + def _mark_in_compilers_yaml(self, match, candidate_scopes): + if os.environ.get("SPACK_EXPERIMENTAL_DEPRECATE_COMPILERS_YAML") == "1": + return [] + + all_removals = [] + for current_scope in candidate_scopes: + compilers_yaml = self.configuration.get("compilers", scope=current_scope) + if not compilers_yaml: + continue + + def _partition_match(entry): + external_specs = CompilerFactory.from_legacy_yaml(entry["compiler"]) + return not any(x.satisfies(match) for x in external_specs) + + to_keep, to_remove = llnl.util.lang.stable_partition(compilers_yaml, _partition_match) + if not to_remove: + continue + + compilers_yaml[:] = to_keep + self.marked_compilers_yaml.append((current_scope, compilers_yaml)) + for entry in to_remove: + all_removals.extend(CompilerFactory.from_legacy_yaml(entry["compiler"])) + + return all_removals + + def flush(self): + """Removes from configuration the specs that have been marked by the previous call + of ``remove_compilers``. + """ + for scope, packages_yaml in self.marked_packages_yaml: + self.configuration.set("packages", packages_yaml, scope=scope) + + for scope, compilers_yaml in self.marked_compilers_yaml: + self.configuration.set("compilers", compilers_yaml, scope=scope) + + def compilers_for_spec(compiler_spec, *, arch_spec=None, scope=None, init_config=True): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - config = all_compilers_config(spack.config.CONFIG, scope=scope, init_config=init_config) - matches = set(find(compiler_spec, scope, init_config)) - compilers = [] - for cspec in matches: - compilers.extend(get_compilers(config, cspec, arch_spec)) - return compilers + # FIXME (compiler as nodes): to be removed, or reimplemented + raise NotImplementedError("still to be implemented") def compilers_for_arch(arch_spec, scope=None): - config = all_compilers_config(spack.config.CONFIG, scope=scope, init_config=False) - return list(get_compilers(config, arch_spec=arch_spec)) - - -def compiler_specs_for_arch(arch_spec, scope=None): - return [c.spec for c in compilers_for_arch(arch_spec, scope)] - - -class CacheReference: - """This acts as a hashable reference to any object (regardless of whether - the object itself is hashable) and also prevents the object from being - garbage-collected (so if two CacheReference objects are equal, they - will refer to the same object, since it will not have been gc'ed since - the creation of the first CacheReference). - """ - - def __init__(self, val): - self.val = val - self.id = id(val) - - def __hash__(self): - return self.id - - def __eq__(self, other): - return isinstance(other, CacheReference) and self.id == other.id - - -def compiler_from_dict(items): - cspec = spack.spec.parse_with_version_concrete(items["spec"]) - os = items.get("operating_system", None) - target = items.get("target", None) - - if not ( - "paths" in items and all(n in items["paths"] for n in spack.compiler.PATH_INSTANCE_VARS) - ): - raise InvalidCompilerConfigurationError(cspec) - - cls = class_for_compiler_name(cspec.name) - - compiler_paths = [] - for c in spack.compiler.PATH_INSTANCE_VARS: - compiler_path = items["paths"][c] - if compiler_path != "None": - compiler_paths.append(compiler_path) - else: - compiler_paths.append(None) - - mods = items.get("modules") - if mods == "None": - mods = [] - - alias = items.get("alias", None) - compiler_flags = items.get("flags", {}) - environment = items.get("environment", {}) - extra_rpaths = items.get("extra_rpaths", []) - implicit_rpaths = items.get("implicit_rpaths", None) - - # Starting with c22a145, 'implicit_rpaths' was a list. Now it is a - # boolean which can be set by the user to disable all automatic - # RPATH insertion of compiler libraries - if implicit_rpaths is not None and not isinstance(implicit_rpaths, bool): - implicit_rpaths = None - - return cls( - cspec, - os, - target, - compiler_paths, - mods, - alias, - environment, - extra_rpaths, - enable_implicit_rpaths=implicit_rpaths, - **compiler_flags, - ) - - -def _compiler_from_config_entry(items): - """Note this is intended for internal use only. To avoid re-parsing - the same config dictionary this keeps track of its location in - memory. If you provide the same dictionary twice it will return - the same Compiler object (regardless of whether the dictionary - entries have changed). - """ - config_id = CacheReference(items) - compiler = _compiler_cache.get(config_id, None) - - if compiler is None: - try: - compiler = compiler_from_dict(items) - except UnknownCompilerError as e: - warnings.warn(e.message) - _compiler_cache[config_id] = compiler - - return compiler - - -def get_compilers(config, cspec=None, arch_spec=None): - compilers = [] - - for items in config: - items = items["compiler"] - - # We might use equality here. - if cspec and not spack.spec.parse_with_version_concrete( - items["spec"], compiler=True - ).satisfies(cspec): + # FIXME (compiler as nodes): this needs a better implementation + compilers = all_compilers_from(spack.config.CONFIG, scope=scope) + result = [] + for candidate in compilers: + _, operating_system, target = name_os_target(candidate) + same_os = operating_system == str(arch_spec.os) + same_target = str(archspec.cpu.TARGETS.get(target)) == str(arch_spec.target) + if not same_os or not same_target: continue - - # If an arch spec is given, confirm that this compiler - # is for the given operating system - os = items.get("operating_system", None) - if arch_spec and os != arch_spec.os: - continue - - # If an arch spec is given, confirm that this compiler - # is for the given target. If the target is 'any', match - # any given arch spec. If the compiler has no assigned - # target this is an old compiler config file, skip this logic. - target = items.get("target", None) - - try: - current_target = archspec.cpu.TARGETS[str(arch_spec.target)] - family = str(current_target.family) - except KeyError: - # TODO: Check if this exception handling makes sense, or if we - # TODO: need to change / refactor tests - family = str(arch_spec.target) - except AttributeError: - assert arch_spec is None - - if arch_spec and target and (target != family and target != "any"): - # If the family of the target is the family we are seeking, - # there's an error in the underlying configuration - if archspec.cpu.TARGETS[target].family == family: - msg = ( - 'the "target" field in compilers.yaml accepts only ' - 'target families [replace "{0}" with "{1}"' - ' in "{2}" specification]' - ) - msg = msg.format(str(target), family, items.get("spec", "??")) - raise ValueError(msg) - continue - - compiler = _compiler_from_config_entry(items) - if compiler: - compilers.append(compiler) - - return compilers + result.append(candidate) + return result -@_auto_compiler_spec -def compiler_for_spec(compiler_spec, arch_spec): - """Get the compiler that satisfies compiler_spec. compiler_spec must - be concrete.""" - assert compiler_spec.concrete - assert arch_spec.concrete - - compilers = compilers_for_spec(compiler_spec, arch_spec=arch_spec) - if len(compilers) < 1: - raise NoCompilerForSpecError(compiler_spec, arch_spec.os) - if len(compilers) > 1: - msg = "Multiple definitions of compiler %s " % compiler_spec - msg += "for architecture %s:\n %s" % (arch_spec, compilers) - tty.debug(msg) - return compilers[0] - - -@llnl.util.lang.memoized def class_for_compiler_name(compiler_name): """Given a compiler module name, get the corresponding Compiler class.""" - if not supported(compiler_name): - raise UnknownCompilerError(compiler_name) - - # Hack to be able to call the compiler `apple-clang` while still - # using a valid python name for the module - submodule_name = compiler_name - if compiler_name == "apple-clang": - submodule_name = compiler_name.replace("-", "_") - - module_name = ".".join(["spack", "compilers", submodule_name]) - module_obj = importlib.import_module(module_name) - cls = getattr(module_obj, mod_to_class(compiler_name)) - - # make a note of the name in the module so we can get to it easily. - cls.name = compiler_name - - return cls - - -def all_compiler_types(): - return [class_for_compiler_name(c) for c in supported_compilers()] - - -def is_mixed_toolchain(compiler): - """Returns True if the current compiler is a mixed toolchain, - False otherwise. - - Args: - compiler (spack.compiler.Compiler): a valid compiler object - """ - import spack.detection.path - - executables = [ - os.path.basename(compiler.cc or ""), - os.path.basename(compiler.cxx or ""), - os.path.basename(compiler.f77 or ""), - os.path.basename(compiler.fc or ""), - ] - - toolchains = set() - finder = spack.detection.path.ExecutablesFinder() - - for pkg_name in spack.repo.PATH.packages_with_tags(COMPILER_TAG): - pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) - patterns = finder.search_patterns(pkg=pkg_cls) - if not patterns: - continue - joined_pattern = re.compile(r"|".join(patterns)) - - if any(joined_pattern.search(exe) for exe in executables): - tty.debug(f"[TOOLCHAIN] MATCH {pkg_name}") - toolchains.add(pkg_name) - - if len(toolchains) > 1: - if ( - toolchains == {"llvm", "apple-clang", "aocc"} - # Msvc toolchain uses Intel ifx - or toolchains == {"msvc", "intel-oneapi-compilers"} - ): - return False - tty.debug("[TOOLCHAINS] {0}".format(toolchains)) - return True - - return False + # FIXME (compiler as nodes): to be removed, or reimplemented + raise NotImplementedError("still to be implemented") _EXTRA_ATTRIBUTES_KEY = "extra_attributes" @@ -673,28 +299,39 @@ def is_mixed_toolchain(compiler): _CXX_KEY, _FORTRAN_KEY = "cxx", "fortran" -class CompilerConfigFactory: - """Class aggregating all ways of constructing a list of compiler config entries.""" +def name_os_target(spec: "spack.spec.Spec") -> Tuple[str, str, str]: + if not spec.architecture: + host_platform = spack.platforms.host() + operating_system = host_platform.operating_system("default_os") + target = host_platform.target("default_target") + else: + target = spec.architecture.target + if not target: + target = spack.platforms.host().target("default_target") + target = target + + operating_system = spec.os + if not operating_system: + host_platform = spack.platforms.host() + operating_system = host_platform.operating_system("default_os") + + return spec.name, str(operating_system), str(target) + + +class CompilerFactory: + """Class aggregating all ways of constructing a list of compiler specs from config entries.""" + + _PACKAGES_YAML_CACHE = {} + _COMPILERS_YAML_CACHE = {} @staticmethod - def from_specs(specs: List["spack.spec.Spec"]) -> List[dict]: - result = [] - compiler_package_names = supported_compilers() + list(package_name_to_compiler_name.keys()) - for s in specs: - if s.name not in compiler_package_names: - continue - - candidate = CompilerConfigFactory.from_external_spec(s) - if candidate is None: - continue - - result.append(candidate) - return result - - @staticmethod - def from_packages_yaml(packages_yaml) -> List[dict]: - compiler_specs = [] - compiler_package_names = supported_compilers() + list(package_name_to_compiler_name.keys()) + def from_packages_yaml( + configuration: "spack.config.ConfigurationType", *, scope: Optional[str] = None + ) -> List["spack.spec.Spec"]: + """Returns the compiler specs defined in the "packages" section of the configuration""" + compilers = [] + compiler_package_names = supported_compilers() + packages_yaml = configuration.get("packages", scope=scope) for name, entry in packages_yaml.items(): if name not in compiler_package_names: continue @@ -703,17 +340,24 @@ def from_packages_yaml(packages_yaml) -> List[dict]: if not externals_config: continue - current_specs = [] + compiler_specs = [] for current_external in externals_config: - compiler = CompilerConfigFactory._spec_from_external_config(current_external) - if compiler: - current_specs.append(compiler) - compiler_specs.extend(current_specs) + key = str(current_external) + if key not in CompilerFactory._PACKAGES_YAML_CACHE: + CompilerFactory._PACKAGES_YAML_CACHE[key] = CompilerFactory.from_external_yaml( + current_external + ) - return CompilerConfigFactory.from_specs(compiler_specs) + compiler = CompilerFactory._PACKAGES_YAML_CACHE[key] + if compiler: + compiler_specs.append(compiler) + + compilers.extend(compiler_specs) + return compilers @staticmethod - def _spec_from_external_config(config): + def from_external_yaml(config: Dict[str, Any]) -> Optional["spack.spec.Spec"]: + """Returns a compiler spec from an external definition from packages.yaml.""" # Allow `@x.y.z` instead of `@=x.y.z` err_header = f"The external spec '{config['spec']}' cannot be used as a compiler" # If extra_attributes is not there I might not want to use this entry as a compiler, @@ -724,113 +368,57 @@ def _spec_from_external_config(config): extra_attributes = config[_EXTRA_ATTRIBUTES_KEY] result = spack.spec.Spec( str(spack.spec.parse_with_version_concrete(config["spec"])), + external_path=config.get("prefix"), external_modules=config.get("modules"), ) result.extra_attributes = extra_attributes + if result.architecture: + result.architecture.complete_with_defaults() + result._finalize_concretization() return result @staticmethod - def from_external_spec(spec: "spack.spec.Spec") -> Optional[dict]: - spec = spack.spec.parse_with_version_concrete(spec) - extra_attributes = getattr(spec, _EXTRA_ATTRIBUTES_KEY, None) - if extra_attributes is None: - return None + def from_legacy_yaml(compiler_dict: Dict[str, Any]) -> List["spack.spec.Spec"]: + """Returns a list of external specs, corresponding to a compiler entry + from compilers.yaml. + """ + from spack.detection.path import ExecutablesFinder - paths = CompilerConfigFactory._extract_compiler_paths(spec) - if paths is None: - return None + # FIXME (compiler as nodes): should we look at targets too? + result = [] + candidate_paths = [x for x in compiler_dict["paths"].values() if x is not None] + finder = ExecutablesFinder() - compiler_spec = spack.spec.CompilerSpec( - package_name_to_compiler_name.get(spec.name, spec.name), spec.version - ) + for pkg_name in spack.repo.PATH.packages_with_tags("compiler"): + pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) + pattern = re.compile(r"|".join(finder.search_patterns(pkg=pkg_cls))) + filtered_paths = [x for x in candidate_paths if pattern.search(os.path.basename(x))] + detected = finder.detect_specs(pkg=pkg_cls, paths=filtered_paths) + result.extend(detected) - operating_system, target = CompilerConfigFactory._extract_os_and_target(spec) - - compiler_entry = { - "compiler": { - "spec": str(compiler_spec), - "paths": paths, - "flags": extra_attributes.get("flags", {}), - "operating_system": str(operating_system), - "target": str(target.family), - "modules": getattr(spec, "external_modules", []), - "environment": extra_attributes.get("environment", {}), - "extra_rpaths": extra_attributes.get("extra_rpaths", []), - "implicit_rpaths": extra_attributes.get("implicit_rpaths", None), - } - } - return compiler_entry + for item in result: + if item.architecture: + item.architecture.complete_with_defaults() + item._finalize_concretization() + return result @staticmethod - def _extract_compiler_paths(spec: "spack.spec.Spec") -> Optional[Dict[str, str]]: - err_header = f"The external spec '{spec}' cannot be used as a compiler" - extra_attributes = spec.extra_attributes - # If I have 'extra_attributes' warn if 'compilers' is missing, - # or we don't have a C compiler - if _COMPILERS_KEY not in extra_attributes: - warnings.warn( - f"{err_header}: missing the '{_COMPILERS_KEY}' key under '{_EXTRA_ATTRIBUTES_KEY}'" - ) - return None - attribute_compilers = extra_attributes[_COMPILERS_KEY] + def from_compilers_yaml( + configuration: "spack.config.ConfigurationType", *, scope: Optional[str] = None + ) -> List["spack.spec.Spec"]: + """Returns the compiler specs defined in the "compilers" section of the configuration""" + result = [] + for item in configuration.get("compilers", scope=scope): + key = str(item) + if key not in CompilerFactory._COMPILERS_YAML_CACHE: + CompilerFactory._COMPILERS_YAML_CACHE[key] = CompilerFactory.from_legacy_yaml( + item["compiler"] + ) - if _C_KEY not in attribute_compilers: - warnings.warn( - f"{err_header}: missing the C compiler path under " - f"'{_EXTRA_ATTRIBUTES_KEY}:{_COMPILERS_KEY}'" - ) - return None - c_compiler = attribute_compilers[_C_KEY] - - # C++ and Fortran compilers are not mandatory, so let's just leave a debug trace - if _CXX_KEY not in attribute_compilers: - tty.debug(f"[{__file__}] The external spec {spec} does not have a C++ compiler") - - if _FORTRAN_KEY not in attribute_compilers: - tty.debug(f"[{__file__}] The external spec {spec} does not have a Fortran compiler") - - # compilers format has cc/fc/f77, externals format has "c/fortran" - return { - "cc": c_compiler, - "cxx": attribute_compilers.get(_CXX_KEY, None), - "fc": attribute_compilers.get(_FORTRAN_KEY, None), - "f77": attribute_compilers.get(_FORTRAN_KEY, None), - } - - @staticmethod - def _extract_os_and_target(spec: "spack.spec.Spec"): - if not spec.architecture: - host_platform = spack.platforms.host() - operating_system = host_platform.default_operating_system() - target = host_platform.default_target() - else: - target = spec.architecture.target - if not target: - target = spack.platforms.host().default_target() - - operating_system = spec.os - if not operating_system: - host_platform = spack.platforms.host() - operating_system = host_platform.default_operating_system() - return operating_system, target - - -class InvalidCompilerConfigurationError(spack.error.SpackError): - def __init__(self, compiler_spec): - super().__init__( - f'Invalid configuration for [compiler "{compiler_spec}"]: ', - f"Compiler configuration must contain entries for " - f"all compilers: {spack.compiler.PATH_INSTANCE_VARS}", - ) + result.extend(CompilerFactory._COMPILERS_YAML_CACHE[key]) + return result class UnknownCompilerError(spack.error.SpackError): def __init__(self, compiler_name): - super().__init__("Spack doesn't support the requested compiler: {0}".format(compiler_name)) - - -class NoCompilerForSpecError(spack.error.SpackError): - def __init__(self, compiler_spec, target): - super().__init__( - "No compilers for operating system %s satisfy spec %s" % (target, compiler_spec) - ) + super().__init__(f"Spack doesn't support the requested compiler: {compiler_name}") diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 79e7bd997c7..a580b810d70 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -718,6 +718,9 @@ def print_section(self, section: str, blame: bool = False, *, scope=None) -> Non raise spack.error.ConfigError(f"cannot read '{section}' configuration") from e +ConfigurationType = Union[Configuration, lang.Singleton] + + @contextlib.contextmanager def override( path_or_scope: Union[ConfigScope, str], value: Optional[Any] = None diff --git a/lib/spack/spack/cray_manifest.py b/lib/spack/spack/cray_manifest.py index 44872cacff6..1fc4a8124a1 100644 --- a/lib/spack/spack/cray_manifest.py +++ b/lib/spack/spack/cray_manifest.py @@ -222,13 +222,13 @@ def read(path, apply_updates): compilers = list() if "compilers" in json_data: compilers.extend(compiler_from_entry(x, path) for x in json_data["compilers"]) - tty.debug("{0}: {1} compilers read from manifest".format(path, str(len(compilers)))) + tty.debug(f"{path}: {str(len(compilers))} compilers read from manifest") # Filter out the compilers that already appear in the configuration compilers = spack.compilers.select_new_compilers(compilers) if apply_updates and compilers: for compiler in compilers: try: - spack.compilers.add_compilers_to_config([compiler]) + spack.compilers.add_compiler_to_config(compiler) except Exception: warnings.warn( f"Could not add compiler {str(compiler.spec)}: " diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py index 04f13d5d603..899452be5d0 100644 --- a/lib/spack/spack/package_base.py +++ b/lib/spack/spack/package_base.py @@ -1375,8 +1375,7 @@ def compiler(self): """Get the spack.compiler.Compiler object used to build this package""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") - - return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.architecture) + raise NotImplementedError("Wrapper to old API still to be implemented") def url_version(self, version): """