From 9ba0050e8b8986183eca0c615e2abc6762451035 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Mon, 1 Jul 2024 17:39:18 +0200 Subject: [PATCH] Recover mixed-toolchain options --- lib/spack/spack/cmd/compiler.py | 8 ++-- lib/spack/spack/compilers/__init__.py | 65 ++++++++++++++++++--------- lib/spack/spack/cray_manifest.py | 2 +- lib/spack/spack/installer.py | 4 +- lib/spack/spack/test/cmd/compiler.py | 38 ++++++++++++++++ lib/spack/spack/test/cray_manifest.py | 11 ++--- share/spack/spack-completion.fish | 8 ++-- 7 files changed, 98 insertions(+), 38 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index f6b076c5fe6..72cb1c71c35 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -35,13 +35,13 @@ def setup_parser(subparser): "--mixed-toolchain", action="store_true", default=sys.platform == "darwin", - help="(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)", + help="Allow mixed toolchains (for example: clang, clang++, gfortran)", ) mixed_toolchain_group.add_argument( "--no-mixed-toolchain", action="store_false", dest="mixed_toolchain", - help="(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)", + help="Do not allow mixed toolchains (for example: clang, clang++, gfortran)", ) find_parser.add_argument("add_paths", nargs=argparse.REMAINDER) find_parser.add_argument( @@ -81,7 +81,9 @@ def compiler_find(args): add them to Spack's configuration. """ paths = args.add_paths or None - new_compilers = spack.compilers.find_compilers(path_hints=paths, scope=args.scope) + new_compilers = spack.compilers.find_compilers( + path_hints=paths, scope=args.scope, mixed_toolchain=args.mixed_toolchain + ) if new_compilers: n = len(new_compilers) s = "s" if n > 1 else "" diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 5d88244dad4..8d66edec174 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -161,24 +161,19 @@ def compiler_config_files(): return config_files -def add_compilers_to_config(compilers, scope=None): - """Add compilers to the config for the specified architecture. +def add_compiler_to_config(compiler, scope=None): + """Add a Compiler object to the configuration, at the required scope.""" + if not compiler.cc: + tty.debug(f"{compiler.spec} does not have a C compiler") + if not compiler.cxx: + tty.debug(f"{compiler.spec} does not have a C++ compiler") + if not compiler.f77: + tty.debug(f"{compiler.spec} does not have a Fortran77 compiler") + if not compiler.fc: + tty.debug(f"{compiler.spec} does not have a Fortran compiler") - Arguments: - compilers: a list of Compiler objects. - scope: configuration scope to modify. - """ compiler_config = get_compiler_config(configuration=spack.config.CONFIG, scope=scope) - for compiler in compilers: - if not compiler.cc: - tty.debug(f"{compiler.spec} does not have a C compiler") - if not compiler.cxx: - tty.debug(f"{compiler.spec} does not have a C++ compiler") - if not compiler.f77: - tty.debug(f"{compiler.spec} does not have a Fortran77 compiler") - if not compiler.fc: - tty.debug(f"{compiler.spec} does not have a Fortran compiler") - compiler_config.append(_to_dict(compiler)) + compiler_config.append(_to_dict(compiler)) spack.config.set("compilers", compiler_config, scope=scope) @@ -303,6 +298,28 @@ def find_compilers( continue valid_compilers[name] = compilers + def _has_fortran_compilers(x): + if "compilers" not in x.spec.extra_attributes: + return False + + return "fortran" in x.spec.extra_attributes["compilers"] + + if mixed_toolchain: + gccs = [x for x in valid_compilers.get("gcc", []) if _has_fortran_compilers(x)] + if gccs: + best_gcc = sorted( + gccs, key=lambda x: spack.spec.parse_with_version_concrete(x.spec).version + )[-1] + gfortran = best_gcc.spec.extra_attributes["compilers"]["fortran"] + for name in ("llvm", "apple-clang"): + if name not in valid_compilers: + continue + candidates = valid_compilers[name] + for candidate in candidates: + if _has_fortran_compilers(candidate): + continue + candidate.spec.extra_attributes["compilers"]["fortran"] = gfortran + new_compilers = spack.detection.update_configuration( valid_compilers, buildable=True, scope=scope ) @@ -319,7 +336,9 @@ def select_new_compilers(compilers, scope=None): compilers_not_in_config = [] for c in compilers: arch_spec = spack.spec.ArchSpec((None, c.operating_system, c.target)) - same_specs = compilers_for_spec(c.spec, arch_spec, scope=scope, init_config=False) + same_specs = compilers_for_spec( + c.spec, arch_spec=arch_spec, scope=scope, init_config=False + ) if not same_specs: compilers_not_in_config.append(c) @@ -388,7 +407,12 @@ def find(compiler_spec, scope=None, init_config=True): def find_specs_by_arch(compiler_spec, arch_spec, scope=None, init_config=True): """Return specs of available compilers that match the supplied compiler spec. Return an empty list if nothing found.""" - return [c.spec for c in compilers_for_spec(compiler_spec, arch_spec, scope, True, init_config)] + return [ + c.spec + for c in compilers_for_spec( + compiler_spec, arch_spec=arch_spec, scope=scope, init_config=init_config + ) + ] def all_compilers(scope=None, init_config=True): @@ -410,14 +434,11 @@ def all_compilers_from(configuration, scope=None, init_config=True): @_auto_compiler_spec -def compilers_for_spec( - compiler_spec, arch_spec=None, scope=None, use_cache=True, init_config=True -): +def compilers_for_spec(compiler_spec, *, arch_spec=None, scope=None, init_config=True): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ config = all_compilers_config(spack.config.CONFIG, scope=scope, init_config=init_config) - matches = set(find(compiler_spec, scope, init_config)) compilers = [] for cspec in matches: diff --git a/lib/spack/spack/cray_manifest.py b/lib/spack/spack/cray_manifest.py index 22371f68f27..c688b6cea84 100644 --- a/lib/spack/spack/cray_manifest.py +++ b/lib/spack/spack/cray_manifest.py @@ -227,7 +227,7 @@ def read(path, apply_updates): if apply_updates and compilers: for compiler in compilers: try: - spack.compilers.add_compilers_to_config([compiler]) + spack.compilers.add_compiler_to_config(compiler) except Exception: warnings.warn( f"Could not add compiler {str(compiler.spec)}: " diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index dc94c95926a..9039a3738f5 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -1611,9 +1611,7 @@ def _add_tasks(self, request: BuildRequest, all_deps): def _add_compiler_package_to_config(self, pkg: "spack.package_base.PackageBase") -> None: compiler_search_prefix = getattr(pkg, "compiler_search_prefix", pkg.spec.prefix) - spack.compilers.add_compilers_to_config( - spack.compilers.find_compilers([compiler_search_prefix]) - ) + spack.compilers.find_compilers([compiler_search_prefix]) def _install_task(self, task: BuildTask, install_status: InstallStatus) -> None: """ diff --git a/lib/spack/spack/test/cmd/compiler.py b/lib/spack/spack/test/cmd/compiler.py index f881e0471e2..959791e379d 100644 --- a/lib/spack/spack/test/cmd/compiler.py +++ b/lib/spack/spack/test/cmd/compiler.py @@ -160,6 +160,44 @@ def test_compiler_add(mutable_config, mock_executable): assert new_compiler.version == spack.version.Version(expected_version) +@pytest.mark.not_on_windows("Cannot execute bash script on Windows") +@pytest.mark.regression("17590") +@pytest.mark.parametrize("mixed_toolchain", [True, False]) +def test_compiler_find_mixed_suffixes( + mixed_toolchain, no_compilers_yaml, working_env, compilers_dir +): + """Ensure that we'll mix compilers with different suffixes when necessary.""" + os.environ["PATH"] = str(compilers_dir) + output = compiler( + "find", "--scope=site", "--mixed-toolchain" if mixed_toolchain else "--no-mixed-toolchain" + ) + + assert "clang@=11.0.0" in output + assert "gcc@=8.4.0" in output + + config = spack.compilers.all_compilers_config( + no_compilers_yaml, scope="site", init_config=False + ) + clang = next(c["compiler"] for c in config if c["compiler"]["spec"] == "clang@=11.0.0") + gcc = next(c["compiler"] for c in config if c["compiler"]["spec"] == "gcc@=8.4.0") + + gfortran_path = str(compilers_dir / "gfortran-8") + + assert clang["paths"] == { + "cc": str(compilers_dir / "clang"), + "cxx": str(compilers_dir / "clang++"), + "f77": gfortran_path if mixed_toolchain else None, + "fc": gfortran_path if mixed_toolchain else None, + } + + assert gcc["paths"] == { + "cc": str(compilers_dir / "gcc-8"), + "cxx": str(compilers_dir / "g++-8"), + "f77": gfortran_path, + "fc": gfortran_path, + } + + @pytest.mark.not_on_windows("Cannot execute bash script on Windows") @pytest.mark.regression("17590") def test_compiler_find_prefer_no_suffix(no_compilers_yaml, working_env, compilers_dir): diff --git a/lib/spack/spack/test/cray_manifest.py b/lib/spack/spack/test/cray_manifest.py index 3a00180da8d..c336b6dc654 100644 --- a/lib/spack/spack/test/cray_manifest.py +++ b/lib/spack/spack/test/cray_manifest.py @@ -16,6 +16,7 @@ import spack import spack.cmd +import spack.cmd.external import spack.compilers import spack.config import spack.cray_manifest as cray_manifest @@ -365,20 +366,20 @@ def test_read_cray_manifest_add_compiler_failure( """Check that cray manifest can be read even if some compilers cannot be added. """ - orig_add_compilers_to_config = spack.compilers.add_compilers_to_config + orig_add_compiler_to_config = spack.compilers.add_compiler_to_config class fail_for_clang: def __init__(self): self.called_with_clang = False - def __call__(self, compilers, **kwargs): - if any(x.name == "clang" for x in compilers): + def __call__(self, compiler, **kwargs): + if compiler.name == "clang": self.called_with_clang = True raise Exception() - return orig_add_compilers_to_config(compilers, **kwargs) + return orig_add_compiler_to_config(compiler, **kwargs) checker = fail_for_clang() - monkeypatch.setattr(spack.compilers, "add_compilers_to_config", checker) + monkeypatch.setattr(spack.compilers, "add_compiler_to_config", checker) with tmpdir.as_cwd(): test_db_fname = "external-db.json" diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 464c001d0c2..6d891757c85 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -1064,9 +1064,9 @@ set -g __fish_spack_optspecs_spack_compiler_find h/help mixed-toolchain no-mixed complete -c spack -n '__fish_spack_using_command compiler find' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler find' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -f -a mixed_toolchain -complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' +complete -c spack -n '__fish_spack_using_command compiler find' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -f -a mixed_toolchain -complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' +complete -c spack -n '__fish_spack_using_command compiler find' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -f -a '_builtin defaults system site user command_line' complete -c spack -n '__fish_spack_using_command compiler find' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler find' -s j -l jobs -r -f -a jobs @@ -1078,9 +1078,9 @@ set -g __fish_spack_optspecs_spack_compiler_add h/help mixed-toolchain no-mixed- complete -c spack -n '__fish_spack_using_command compiler add' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command compiler add' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -f -a mixed_toolchain -complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d '(DEPRECATED) Allow mixed toolchains (for example: clang, clang++, gfortran)' +complete -c spack -n '__fish_spack_using_command compiler add' -l mixed-toolchain -d 'Allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -f -a mixed_toolchain -complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d '(DEPRECATED) Do not allow mixed toolchains (for example: clang, clang++, gfortran)' +complete -c spack -n '__fish_spack_using_command compiler add' -l no-mixed-toolchain -d 'Do not allow mixed toolchains (for example: clang, clang++, gfortran)' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -f -a '_builtin defaults system site user command_line' complete -c spack -n '__fish_spack_using_command compiler add' -l scope -r -d 'configuration scope to modify' complete -c spack -n '__fish_spack_using_command compiler add' -s j -l jobs -r -f -a jobs