From 91b09cfeb6de3560c16e74cec020fff61e789c49 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 14 Jan 2025 14:29:34 +0100 Subject: [PATCH] Better handling of legacy compilers.yaml After this commit, entries in compilers.yaml will be "converted" to entries in packages.yaml, only if no other compiler is present, when Spack tries to initialize automatically the compiler configuration. --- lib/spack/spack/compilers/config.py | 67 ++++++++------- lib/spack/spack/test/compilers/conversion.py | 85 ++++++++++++++++++++ lib/spack/spack/test/conftest.py | 11 +++ 3 files changed, 136 insertions(+), 27 deletions(-) create mode 100644 lib/spack/spack/test/compilers/conversion.py diff --git a/lib/spack/spack/compilers/config.py b/lib/spack/spack/compilers/config.py index 3124573e41e..3951d82293e 100644 --- a/lib/spack/spack/compilers/config.py +++ b/lib/spack/spack/compilers/config.py @@ -18,6 +18,7 @@ import spack.config import spack.detection +import spack.detection.path import spack.error import spack.platforms import spack.repo @@ -124,12 +125,34 @@ def all_compilers( compilers = all_compilers_from(configuration=spack.config.CONFIG, scope=scope) if not compilers and init_config: - find_compilers(scope=scope) + _init_packages_yaml(spack.config.CONFIG, scope=scope) compilers = all_compilers_from(configuration=spack.config.CONFIG, scope=scope) return compilers +def _init_packages_yaml( + configuration: "spack.config.ConfigurationType", *, scope: Optional[str] +) -> None: + # Try importing from compilers.yaml + legacy_compilers = CompilerFactory.from_compilers_yaml(configuration, scope=scope) + if legacy_compilers: + by_name: Dict[str, List[spack.spec.Spec]] = {} + for legacy in legacy_compilers: + by_name.setdefault(legacy.name, []).append(legacy) + spack.detection.update_configuration(by_name, buildable=True, scope=scope) + warnings.warn("compilers have been automatically converted from existing 'compilers.yaml'") + return + + # Look for compilers in PATH + new_compilers = find_compilers(scope=scope) + if not new_compilers: + raise NoAvailableCompilerError( + "no compiler configured, and Spack cannot find working compilers in PATH" + ) + warnings.warn("compilers have been configured automatically from PATH inspection") + + def all_compilers_from( configuration: "spack.config.ConfigurationType", scope: Optional[str] = None ) -> List["spack.spec.Spec"]: @@ -141,19 +164,6 @@ def all_compilers_from( configuration is used. """ compilers = CompilerFactory.from_packages_yaml(configuration, scope=scope) - - if os.environ.get("SPACK_EXPERIMENTAL_DEPRECATE_COMPILERS_YAML") != "1": - legacy_compilers = CompilerFactory.from_compilers_yaml(configuration, scope=scope) - if legacy_compilers: - # FIXME (compiler as nodes): write how to update the file. Maybe an ad-hoc command - warnings.warn( - "Some compilers are still defined in 'compilers.yaml', which has been deprecated " - "in v0.23. Those configuration files will be ignored from Spack v0.25.\n" - ) - for legacy in legacy_compilers: - if not any(c.satisfies(f"{legacy.name}@{legacy.versions}") for c in compilers): - compilers.append(legacy) - return compilers @@ -299,7 +309,6 @@ class CompilerFactory: """Class aggregating all ways of constructing a list of compiler specs from config entries.""" _PACKAGES_YAML_CACHE: Dict[str, Optional["spack.spec.Spec"]] = {} - _COMPILERS_YAML_CACHE: Dict[str, List["spack.spec.Spec"]] = {} _GENERIC_TARGET = None @staticmethod @@ -370,22 +379,28 @@ def from_legacy_yaml(compiler_dict: Dict[str, Any]) -> List["spack.spec.Spec"]: """Returns a list of external specs, corresponding to a compiler entry from compilers.yaml. """ - from spack.detection.path import ExecutablesFinder - # FIXME (compiler as nodes): should we look at targets too? result = [] candidate_paths = [x for x in compiler_dict["paths"].values() if x is not None] - finder = ExecutablesFinder() + finder = spack.detection.path.ExecutablesFinder() for pkg_name in spack.repo.PATH.packages_with_tags("compiler"): pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name) pattern = re.compile(r"|".join(finder.search_patterns(pkg=pkg_cls))) filtered_paths = [x for x in candidate_paths if pattern.search(os.path.basename(x))] detected = finder.detect_specs(pkg=pkg_cls, paths=filtered_paths) + for s in detected: + for key in ("flags", "environment", "extra_rpaths"): + if key in compiler_dict: + s.extra_attributes[key] = compiler_dict[key] + + if "modules" in compiler_dict: + s.external_modules = list(compiler_dict["modules"]) + result.extend(detected) - for item in result: - CompilerFactory._finalize_external_concretization(item) + # for item in result: + # CompilerFactory._finalize_external_concretization(item) return result @@ -396,16 +411,14 @@ def from_compilers_yaml( """Returns the compiler specs defined in the "compilers" section of the configuration""" result: List["spack.spec.Spec"] = [] for item in configuration.get("compilers", scope=scope): - key = str(item) - if key not in CompilerFactory._COMPILERS_YAML_CACHE: - CompilerFactory._COMPILERS_YAML_CACHE[key] = CompilerFactory.from_legacy_yaml( - item["compiler"] - ) - - result.extend(CompilerFactory._COMPILERS_YAML_CACHE[key]) + result.extend(CompilerFactory.from_legacy_yaml(item["compiler"])) return result class UnknownCompilerError(spack.error.SpackError): def __init__(self, compiler_name): super().__init__(f"Spack doesn't support the requested compiler: {compiler_name}") + + +class NoAvailableCompilerError(spack.error.SpackError): + pass diff --git a/lib/spack/spack/test/compilers/conversion.py b/lib/spack/spack/test/compilers/conversion.py new file mode 100644 index 00000000000..84ce07f5ded --- /dev/null +++ b/lib/spack/spack/test/compilers/conversion.py @@ -0,0 +1,85 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Tests conversions from compilers.yaml""" +import pytest + +from spack.compilers.config import CompilerFactory + + +@pytest.fixture() +def mock_compiler(mock_executable): + gcc = mock_executable("gcc", "echo 13.2.0") + gxx = mock_executable("g++", "echo 13.2.0") + gfortran = mock_executable("gfortran", "echo 13.2.0") + return { + "spec": "gcc@13.2.0", + "paths": {"cc": str(gcc), "cxx": str(gxx), "f77": str(gfortran), "fc": str(gfortran)}, + } + + +# - compiler: +# spec: clang@=10.0.0 +# paths: +# cc: /usr/bin/clang +# cxx: /usr/bin/clang++ +# f77: null +# fc: null +# flags: {} +# operating_system: ubuntu20.04 +# target: x86_64 +# modules: [] +# environment: {} +# extra_rpaths: [] + + +def test_basic_compiler_conversion(mock_compiler, tmp_path): + """Tests the conversion of a compiler using a single toolchain, with default options.""" + compilers = CompilerFactory.from_legacy_yaml(mock_compiler) + compiler_spec = compilers[0] + assert compiler_spec.satisfies("gcc@13.2.0 languages=c,c++,fortran") + assert compiler_spec.external + assert compiler_spec.external_path == str(tmp_path) + + for language in ("c", "cxx", "fortran"): + assert language in compiler_spec.extra_attributes["compilers"] + + +def test_compiler_conversion_with_flags(mock_compiler): + """Tests that flags are converted appropriately for external compilers""" + mock_compiler["flags"] = {"cflags": "-O3", "cxxflags": "-O2 -g"} + compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0] + assert compiler_spec.external + assert "flags" in compiler_spec.extra_attributes + assert compiler_spec.extra_attributes["flags"]["cflags"] == "-O3" + assert compiler_spec.extra_attributes["flags"]["cxxflags"] == "-O2 -g" + + +def tests_compiler_conversion_with_environment(mock_compiler): + """Tests that custom environment modifications are converted appropriately + for external compilers + """ + mods = {"set": {"FOO": "foo", "BAR": "bar"}, "unset": ["BAZ"]} + mock_compiler["environment"] = mods + compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0] + assert compiler_spec.external + assert "environment" in compiler_spec.extra_attributes + assert compiler_spec.extra_attributes["environment"] == mods + + +def tests_compiler_conversion_extra_rpaths(mock_compiler): + """Tests that extra rpaths are converted appropriately for external compilers""" + mock_compiler["extra_rpaths"] = ["/foo/bar"] + compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0] + assert compiler_spec.external + assert "extra_rpaths" in compiler_spec.extra_attributes + assert compiler_spec.extra_attributes["extra_rpaths"] == ["/foo/bar"] + + +def tests_compiler_conversion_modules(mock_compiler): + """Tests that modules are converted appropriately for external compilers""" + modules = ["foo/4.1.2", "bar/5.1.4"] + mock_compiler["modules"] = modules + compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0] + assert compiler_spec.external + assert compiler_spec.external_modules == modules diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 4bc4da27cab..f2700f70017 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -35,6 +35,7 @@ import spack.binary_distribution import spack.bootstrap.core import spack.caches +import spack.compilers.config import spack.compilers.libraries import spack.concretize import spack.config @@ -2163,3 +2164,13 @@ def wrapper_dir(install_mockery): wrapper_pkg = wrapper.package PackageInstaller([wrapper_pkg], explicit=True).install() return wrapper_pkg.bin_dir() + + +def _noop(*args, **kwargs): + pass + + +@pytest.fixture(autouse=True) +def no_compilers_init(monkeypatch): + """Disables automatic compiler initialization""" + monkeypatch.setattr(spack.compilers.config, "_init_packages_yaml", _noop)