Better handling of legacy compilers.yaml

After this commit, entries in compilers.yaml will be "converted"
to entries in packages.yaml, only if no other compiler is present,
when Spack tries to initialize automatically the compiler
configuration.

lib/spack/spack/compilers/config.py

@@ -18,6 +18,7 @@
 
 import spack.config
 import spack.detection
+import spack.detection.path
 import spack.error
 import spack.platforms
 import spack.repo
@@ -124,12 +125,34 @@ def all_compilers(
     compilers = all_compilers_from(configuration=spack.config.CONFIG, scope=scope)
 
     if not compilers and init_config:
-        find_compilers(scope=scope)
+        _init_packages_yaml(spack.config.CONFIG, scope=scope)
         compilers = all_compilers_from(configuration=spack.config.CONFIG, scope=scope)
 
     return compilers
 
 
+def _init_packages_yaml(
+    configuration: "spack.config.ConfigurationType", *, scope: Optional[str]
+) -> None:
+    # Try importing from compilers.yaml
+    legacy_compilers = CompilerFactory.from_compilers_yaml(configuration, scope=scope)
+    if legacy_compilers:
+        by_name: Dict[str, List[spack.spec.Spec]] = {}
+        for legacy in legacy_compilers:
+            by_name.setdefault(legacy.name, []).append(legacy)
+        spack.detection.update_configuration(by_name, buildable=True, scope=scope)
+        warnings.warn("compilers have been automatically converted from existing 'compilers.yaml'")
+        return
+
+    # Look for compilers in PATH
+    new_compilers = find_compilers(scope=scope)
+    if not new_compilers:
+        raise NoAvailableCompilerError(
+            "no compiler configured, and Spack cannot find working compilers in PATH"
+        )
+    warnings.warn("compilers have been configured automatically from PATH inspection")
+
+
 def all_compilers_from(
     configuration: "spack.config.ConfigurationType", scope: Optional[str] = None
 ) -> List["spack.spec.Spec"]:
@@ -141,19 +164,6 @@ def all_compilers_from(
             configuration is used.
     """
     compilers = CompilerFactory.from_packages_yaml(configuration, scope=scope)
-
-    if os.environ.get("SPACK_EXPERIMENTAL_DEPRECATE_COMPILERS_YAML") != "1":
-        legacy_compilers = CompilerFactory.from_compilers_yaml(configuration, scope=scope)
-        if legacy_compilers:
-            # FIXME (compiler as nodes): write how to update the file. Maybe an ad-hoc command
-            warnings.warn(
-                "Some compilers are still defined in 'compilers.yaml', which has been deprecated "
-                "in v0.23. Those configuration files will be ignored from Spack v0.25.\n"
-            )
-            for legacy in legacy_compilers:
-                if not any(c.satisfies(f"{legacy.name}@{legacy.versions}") for c in compilers):
-                    compilers.append(legacy)
-
     return compilers
 
 
@@ -299,7 +309,6 @@ class CompilerFactory:
     """Class aggregating all ways of constructing a list of compiler specs from config entries."""
 
     _PACKAGES_YAML_CACHE: Dict[str, Optional["spack.spec.Spec"]] = {}
-    _COMPILERS_YAML_CACHE: Dict[str, List["spack.spec.Spec"]] = {}
     _GENERIC_TARGET = None
 
     @staticmethod
@@ -370,22 +379,28 @@ def from_legacy_yaml(compiler_dict: Dict[str, Any]) -> List["spack.spec.Spec"]:
         """Returns a list of external specs, corresponding to a compiler entry
         from compilers.yaml.
         """
-        from spack.detection.path import ExecutablesFinder
-
         # FIXME (compiler as nodes): should we look at targets too?
         result = []
         candidate_paths = [x for x in compiler_dict["paths"].values() if x is not None]
-        finder = ExecutablesFinder()
+        finder = spack.detection.path.ExecutablesFinder()
 
         for pkg_name in spack.repo.PATH.packages_with_tags("compiler"):
             pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
             pattern = re.compile(r"|".join(finder.search_patterns(pkg=pkg_cls)))
             filtered_paths = [x for x in candidate_paths if pattern.search(os.path.basename(x))]
             detected = finder.detect_specs(pkg=pkg_cls, paths=filtered_paths)
+            for s in detected:
+                for key in ("flags", "environment", "extra_rpaths"):
+                    if key in compiler_dict:
+                        s.extra_attributes[key] = compiler_dict[key]
+
+                if "modules" in compiler_dict:
+                    s.external_modules = list(compiler_dict["modules"])
+
             result.extend(detected)
 
-        for item in result:
-            CompilerFactory._finalize_external_concretization(item)
+        # for item in result:
+        #    CompilerFactory._finalize_external_concretization(item)
 
         return result
 
@@ -396,16 +411,14 @@ def from_compilers_yaml(
         """Returns the compiler specs defined in the "compilers" section of the configuration"""
         result: List["spack.spec.Spec"] = []
         for item in configuration.get("compilers", scope=scope):
-            key = str(item)
-            if key not in CompilerFactory._COMPILERS_YAML_CACHE:
-                CompilerFactory._COMPILERS_YAML_CACHE[key] = CompilerFactory.from_legacy_yaml(
-                    item["compiler"]
-                )
-
-            result.extend(CompilerFactory._COMPILERS_YAML_CACHE[key])
+            result.extend(CompilerFactory.from_legacy_yaml(item["compiler"]))
         return result
 
 
 class UnknownCompilerError(spack.error.SpackError):
     def __init__(self, compiler_name):
         super().__init__(f"Spack doesn't support the requested compiler: {compiler_name}")
+
+
+class NoAvailableCompilerError(spack.error.SpackError):
+    pass

lib/spack/spack/test/compilers/conversion.py

@@ -0,0 +1,85 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Tests conversions from compilers.yaml"""
+import pytest
+
+from spack.compilers.config import CompilerFactory
+
+
+@pytest.fixture()
+def mock_compiler(mock_executable):
+    gcc = mock_executable("gcc", "echo 13.2.0")
+    gxx = mock_executable("g++", "echo 13.2.0")
+    gfortran = mock_executable("gfortran", "echo 13.2.0")
+    return {
+        "spec": "gcc@13.2.0",
+        "paths": {"cc": str(gcc), "cxx": str(gxx), "f77": str(gfortran), "fc": str(gfortran)},
+    }
+
+
+# - compiler:
+#     spec: clang@=10.0.0
+#     paths:
+#       cc: /usr/bin/clang
+#       cxx: /usr/bin/clang++
+#       f77: null
+#       fc: null
+#     flags: {}
+#     operating_system: ubuntu20.04
+#     target: x86_64
+#     modules: []
+#     environment: {}
+#     extra_rpaths: []
+
+
+def test_basic_compiler_conversion(mock_compiler, tmp_path):
+    """Tests the conversion of a compiler using a single toolchain, with default options."""
+    compilers = CompilerFactory.from_legacy_yaml(mock_compiler)
+    compiler_spec = compilers[0]
+    assert compiler_spec.satisfies("gcc@13.2.0 languages=c,c++,fortran")
+    assert compiler_spec.external
+    assert compiler_spec.external_path == str(tmp_path)
+
+    for language in ("c", "cxx", "fortran"):
+        assert language in compiler_spec.extra_attributes["compilers"]
+
+
+def test_compiler_conversion_with_flags(mock_compiler):
+    """Tests that flags are converted appropriately for external compilers"""
+    mock_compiler["flags"] = {"cflags": "-O3", "cxxflags": "-O2 -g"}
+    compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0]
+    assert compiler_spec.external
+    assert "flags" in compiler_spec.extra_attributes
+    assert compiler_spec.extra_attributes["flags"]["cflags"] == "-O3"
+    assert compiler_spec.extra_attributes["flags"]["cxxflags"] == "-O2 -g"
+
+
+def tests_compiler_conversion_with_environment(mock_compiler):
+    """Tests that custom environment modifications are converted appropriately
+    for external compilers
+    """
+    mods = {"set": {"FOO": "foo", "BAR": "bar"}, "unset": ["BAZ"]}
+    mock_compiler["environment"] = mods
+    compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0]
+    assert compiler_spec.external
+    assert "environment" in compiler_spec.extra_attributes
+    assert compiler_spec.extra_attributes["environment"] == mods
+
+
+def tests_compiler_conversion_extra_rpaths(mock_compiler):
+    """Tests that extra rpaths are converted appropriately for external compilers"""
+    mock_compiler["extra_rpaths"] = ["/foo/bar"]
+    compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0]
+    assert compiler_spec.external
+    assert "extra_rpaths" in compiler_spec.extra_attributes
+    assert compiler_spec.extra_attributes["extra_rpaths"] == ["/foo/bar"]
+
+
+def tests_compiler_conversion_modules(mock_compiler):
+    """Tests that modules are converted appropriately for external compilers"""
+    modules = ["foo/4.1.2", "bar/5.1.4"]
+    mock_compiler["modules"] = modules
+    compiler_spec = CompilerFactory.from_legacy_yaml(mock_compiler)[0]
+    assert compiler_spec.external
+    assert compiler_spec.external_modules == modules

lib/spack/spack/test/conftest.py

@@ -35,6 +35,7 @@
 import spack.binary_distribution
 import spack.bootstrap.core
 import spack.caches
+import spack.compilers.config
 import spack.compilers.libraries
 import spack.concretize
 import spack.config
@@ -2163,3 +2164,13 @@ def wrapper_dir(install_mockery):
     wrapper_pkg = wrapper.package
     PackageInstaller([wrapper_pkg], explicit=True).install()
     return wrapper_pkg.bin_dir()
+
+
+def _noop(*args, **kwargs):
+    pass
+
+
+@pytest.fixture(autouse=True)
+def no_compilers_init(monkeypatch):
+    """Disables automatic compiler initialization"""
+    monkeypatch.setattr(spack.compilers.config, "_init_packages_yaml", _noop)