autotools.py: order props / methods / callbacks

Should be sufficient to set CONFIG_SITE to /dev/null before running configure to 
prevent any config site file to be loaded, which currently causes non-determinism in builds.

Setting the variable CONFIG_SITE prevents configure files to check $prefix/share/config.site, 
$prefix/etc/config.site, $ac_default_prefix/share/config.site, $ac_default_prefix/etc/config.site 
so we don't have to patch a block of code.

.github/workflows/unit_tests.yaml

@@ -61,7 +61,7 @@ jobs:
       run: "brew install kcov"
     - name: Install Python packages
       run: |
-          pip install --upgrade pip setuptools pytest pytest-xdist pytest-cov clingo
+          pip install --upgrade pip setuptools pytest pytest-xdist pytest-cov
           pip install --upgrade flake8 "isort>=4.3.5" "mypy>=0.900" "click" "black"
     - name: Setup git configuration
       run: |
@@ -185,6 +185,7 @@ jobs:
         spack bootstrap disable github-actions-v0.5
         spack bootstrap disable github-actions-v0.6
         spack bootstrap status
+        spack solve zlib
         spack unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml lib/spack/spack/test/concretization/core.py
     - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b
       with:
@@ -222,7 +223,7 @@ jobs:
         . share/spack/setup-env.sh
         $(which spack) bootstrap disable spack-install
         $(which spack) solve zlib
-        common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python')
+        common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
         $(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
     - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b
       with:
@@ -253,7 +254,7 @@ jobs:
       env:
         COVERAGE_FILE: coverage/.coverage-windows
       run: |
-        spack unit-test --verbose --cov --cov-config=pyproject.toml
+        spack unit-test -x --verbose --cov --cov-config=pyproject.toml
         ./share/spack/qa/validate_last_exit.ps1
     - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b
       with:

etc/spack/defaults/config.yaml

@@ -19,7 +19,7 @@ config:
   install_tree:
     root: $spack/opt/spack
     projections:
-      all: "{architecture.platform}-{architecture.target}/{name}-{version}-{hash}"
+      all: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}"
     # install_tree can include an optional padded length (int or boolean)
     # default is False (do not pad)
     # if padded_length is True, Spack will pad as close to the system max path

etc/spack/defaults/darwin/packages.yaml

@@ -15,11 +15,12 @@
 # -------------------------------------------------------------------------
 packages:
   all:
+    compiler:
+    - apple-clang
+    - clang
+    - gcc
     providers:
-      c: [apple-clang, llvm, gcc]
-      cxx: [apple-clang, llvm, gcc]
       elf: [libelf]
-      fortran: [gcc]
       fuse: [macfuse]
       gl: [apple-gl]
       glu: [apple-glu]

etc/spack/defaults/packages.yaml

@@ -15,18 +15,19 @@
 # -------------------------------------------------------------------------
 packages:
   all:
+    compiler: [gcc, clang, oneapi, xl, nag, fj, aocc]
     providers:
       awk: [gawk]
       armci: [armcimpi]
       blas: [openblas, amdblis]
-      c: [gcc, llvm, intel-oneapi-compilers, xl, aocc]
-      cxx: [gcc, llvm, intel-oneapi-compilers, xl, aocc]
+      c: [gcc]
+      cxx: [gcc]
       D: [ldc]
       daal: [intel-oneapi-daal]
       elf: [elfutils]
       fftw-api: [fftw, amdfftw]
       flame: [libflame, amdlibflame]
-      fortran: [gcc, llvm]
+      fortran: [gcc]
       fortran-rt: [gcc-runtime, intel-oneapi-runtime]
       fuse: [libfuse]
       gl: [glx, osmesa]

etc/spack/defaults/windows/packages.yaml

@@ -15,8 +15,8 @@
 # -------------------------------------------------------------------------
 packages:
   all:
+    compiler:
+    - msvc
     providers:
-      c : [msvc]
-      cxx: [msvc]
       mpi: [msmpi]
       gl: [wgl]

lib/spack/docs/build_systems/custompackage.rst

@@ -56,13 +56,13 @@ If you look at the ``perl`` package, you'll see:
 
 .. code-block:: python
 
-   phases = ["configure", "build", "install"]
+   phases = ("configure", "build", "install")
 
 Similarly, ``cmake`` defines:
 
 .. code-block:: python
 
-   phases = ["bootstrap", "build", "install"]
+   phases = ("bootstrap", "build", "install")
 
 If we look at the ``cmake`` example, this tells Spack's ``PackageBase``
 class to run the ``bootstrap``, ``build``, and ``install`` functions

lib/spack/docs/developer_guide.rst

@@ -543,10 +543,10 @@ With either interpreter you can run a single command:
 
 .. code-block:: console
 
-   $ spack python -c 'from spack.spec import Spec; Spec("python").concretized()'
+   $ spack python -c 'from spack.concretize import concretize_one; concretize_one("python")'
    ...
 
-   $ spack python -i ipython -c 'from spack.spec import Spec; Spec("python").concretized()'
+   $ spack python -i ipython -c 'from spack.concretize import concretize_one; concretize_one("python")'
    Out[1]: ...
 
 or a file:

lib/spack/docs/module_file_support.rst

@@ -456,14 +456,13 @@ For instance, the following config options,
        tcl:
          all:
            suffixes:
-             ^python@3: 'python{^python.version}'
+             ^python@3: 'python{^python.version.up_to_2}'
              ^openblas: 'openblas'
 
-will add a ``python-3.12.1`` version string to any packages compiled with
-Python matching the spec, ``python@3``. This is useful to know which
-version of Python a set of Python extensions is associated with. Likewise, the
-``openblas`` string is attached to any program that has openblas in the spec,
-most likely via the ``+blas`` variant specification.
+will add a ``python3.12`` to module names of packages compiled with Python 3.12, and similarly for
+all specs depending on ``python@3``. This is useful to know which version of Python a set of Python
+extensions is associated with. Likewise, the ``openblas`` string is attached to any program that
+has openblas in the spec, most likely via the ``+blas`` variant specification.
 
 The most heavyweight solution to module naming is to change the entire
 naming convention for module files. This uses the projections format

lib/spack/docs/pipelines.rst

@@ -330,7 +330,7 @@ that ``--tests`` is passed to ``spack ci rebuild`` as part of the
           - spack --version
           - cd ${SPACK_CONCRETE_ENV_DIR}
           - spack env activate --without-view .
-          - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture.platform}-{architecture.target}/{name}-{version}-{hash}'"
+          - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
            - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
            - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi
            - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi

lib/spack/env/aocc/clang

@@ -0,0 +1 @@
+../cc

lib/spack/env/aocc/clang++

@@ -0,0 +1 @@
+../cpp

lib/spack/env/aocc/flang

@@ -0,0 +1 @@
+../fc

lib/spack/env/arm/armclang

@@ -0,0 +1 @@
+../cc

lib/spack/env/arm/armclang++

@@ -0,0 +1 @@
+../cc

lib/spack/env/arm/armflang

@@ -0,0 +1 @@
+../cc

lib/spack/env/c++

@@ -0,0 +1 @@
+cc

lib/spack/env/c89

@@ -0,0 +1 @@
+cc

lib/spack/env/c99

@@ -0,0 +1 @@
+cc

lib/spack/env/case-insensitive/CC

@@ -0,0 +1 @@
+../cc

lib/spack/env/cc

@@ -39,6 +39,12 @@ readonly params="\
 SPACK_ENV_PATH
 SPACK_DEBUG_LOG_DIR
 SPACK_DEBUG_LOG_ID
+SPACK_COMPILER_SPEC
+SPACK_CC_RPATH_ARG
+SPACK_CXX_RPATH_ARG
+SPACK_F77_RPATH_ARG
+SPACK_FC_RPATH_ARG
+SPACK_LINKER_ARG
 SPACK_SHORT_SPEC
 SPACK_SYSTEM_DIRS
 SPACK_MANAGED_DIRS"
@@ -339,9 +345,6 @@ case "$command" in
         ;;
     ld|ld.gold|ld.lld)
         mode=ld
-        if [ -z "$SPACK_CC_RPATH_ARG" ]; then
-            comp="CXX"
-        fi
         ;;
     *)
         die "Unknown compiler: $command"
@@ -396,12 +399,10 @@ fi
 #
 dtags_to_add="${SPACK_DTAGS_TO_ADD}"
 dtags_to_strip="${SPACK_DTAGS_TO_STRIP}"
-
-linker_arg="ERROR: LINKER ARG WAS NOT SET, MAYBE THE PACKAGE DOES NOT DEPEND ON ${comp}?"
-eval "linker_arg=\${SPACK_${comp}_LINKER_ARG:?${linker_arg}}"
+linker_arg="${SPACK_LINKER_ARG}"
 
 # Set up rpath variable according to language.
-rpath="ERROR: RPATH ARG WAS NOT SET, MAYBE THE PACKAGE DOES NOT DEPEND ON ${comp}?"
+rpath="ERROR: RPATH ARG WAS NOT SET"
 eval "rpath=\${SPACK_${comp}_RPATH_ARG:?${rpath}}"
 
 # Dump the mode and exit if the command is dump-mode.
@@ -410,6 +411,13 @@ if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then
     exit
 fi
 
+# If, say, SPACK_CC is set but SPACK_FC is not, we want to know. Compilers do not
+# *have* to set up Fortran executables, so we need to tell the user when a build is
+# about to attempt to use them unsuccessfully.
+if [ -z "$command" ]; then
+    die "Compiler '$SPACK_COMPILER_SPEC' does not have a $language compiler configured."
+fi
+
 #
 # Filter '.' and Spack environment directories out of PATH so that
 # this script doesn't just call itself
@@ -779,17 +787,15 @@ case "$mode" in
             C)
                 extend spack_flags_list SPACK_ALWAYS_CFLAGS
                 extend spack_flags_list SPACK_CFLAGS
-                preextend flags_list SPACK_TARGET_ARGS_CC
                 ;;
             CXX)
                 extend spack_flags_list SPACK_ALWAYS_CXXFLAGS
                 extend spack_flags_list SPACK_CXXFLAGS
-                preextend flags_list SPACK_TARGET_ARGS_CXX
-                ;;
-            F)
-                preextend flags_list SPACK_TARGET_ARGS_FORTRAN
                 ;;
         esac
+
+        # prepend target args
+        preextend flags_list SPACK_TARGET_ARGS
         ;;
 esac
 

lib/spack/env/cce/case-insensitive/CC

@@ -0,0 +1 @@
+../../cc

lib/spack/env/cce/case-insensitive/crayCC

@@ -0,0 +1 @@
+../../cc

lib/spack/env/cce/cc

@@ -0,0 +1 @@
+../cc

lib/spack/env/cce/craycc

@@ -0,0 +1 @@
+../cc

lib/spack/env/cce/crayftn

@@ -0,0 +1 @@
+../cc

lib/spack/env/cce/ftn

@@ -0,0 +1 @@
+../cc

lib/spack/env/clang/clang

@@ -0,0 +1 @@
+../cc

lib/spack/env/clang/clang++

@@ -0,0 +1 @@
+../cc

lib/spack/env/clang/flang

@@ -0,0 +1 @@
+../cc

lib/spack/env/clang/gfortran

@@ -0,0 +1 @@
+../cc

lib/spack/env/cpp

@@ -0,0 +1 @@
+cc

lib/spack/env/f77

@@ -0,0 +1 @@
+cc

lib/spack/env/f90

@@ -0,0 +1 @@
+cc

lib/spack/env/f95

@@ -0,0 +1 @@
+cc

lib/spack/env/fc

@@ -0,0 +1 @@
+cc

lib/spack/env/fj/case-insensitive/FCC

@@ -0,0 +1 @@
+../../cc

lib/spack/env/fj/fcc

@@ -0,0 +1 @@
+../cc

lib/spack/env/fj/frt

@@ -0,0 +1 @@
+../cc

lib/spack/env/ftn

@@ -0,0 +1 @@
+cc

lib/spack/env/gcc/g++

@@ -0,0 +1 @@
+../cc

lib/spack/env/gcc/gcc

@@ -0,0 +1 @@
+../cc

lib/spack/env/gcc/gfortran

@@ -0,0 +1 @@
+../cc

lib/spack/env/intel/icc

@@ -0,0 +1 @@
+../cc

lib/spack/env/intel/icpc

@@ -0,0 +1 @@
+../cc

lib/spack/env/intel/ifort

@@ -0,0 +1 @@
+../cc

lib/spack/env/ld

@@ -0,0 +1 @@
+cc

lib/spack/env/ld.gold

@@ -0,0 +1 @@
+cc

lib/spack/env/ld.lld

@@ -0,0 +1 @@
+cc

lib/spack/env/nag/nagfor

@@ -0,0 +1 @@
+../cc

lib/spack/env/nvhpc/nvc

@@ -0,0 +1 @@
+../cc

lib/spack/env/nvhpc/nvc++

@@ -0,0 +1 @@
+../cc

lib/spack/env/nvhpc/nvfortran

@@ -0,0 +1 @@
+../cc

lib/spack/env/oneapi/dpcpp

@@ -0,0 +1 @@
+../cc

lib/spack/env/oneapi/icpx

@@ -0,0 +1 @@
+../cc

lib/spack/env/oneapi/icx

@@ -0,0 +1 @@
+../cc

lib/spack/env/oneapi/ifx

@@ -0,0 +1 @@
+../cc

lib/spack/env/pgi/pgc++

@@ -0,0 +1 @@
+../cc

lib/spack/env/pgi/pgcc

@@ -0,0 +1 @@
+../cc

lib/spack/env/pgi/pgfortran

@@ -0,0 +1 @@
+../cc

lib/spack/env/rocmcc/amdclang

@@ -0,0 +1 @@
+../cc

lib/spack/env/rocmcc/amdclang++

@@ -0,0 +1 @@
+../cpp

lib/spack/env/rocmcc/amdflang

@@ -0,0 +1 @@
+../fc

lib/spack/env/xl/xlc

@@ -0,0 +1 @@
+../cc

lib/spack/env/xl/xlc++

@@ -0,0 +1 @@
+../cc

lib/spack/env/xl/xlf

@@ -0,0 +1 @@
+../cc

lib/spack/env/xl/xlf90

@@ -0,0 +1 @@
+../cc

lib/spack/env/xl_r/xlc++_r

@@ -0,0 +1 @@
+../cc

lib/spack/env/xl_r/xlc_r

@@ -0,0 +1 @@
+../cc

lib/spack/env/xl_r/xlf90_r

@@ -0,0 +1 @@
+../cc

lib/spack/env/xl_r/xlf_r

@@ -0,0 +1 @@
+../cc

lib/spack/llnl/util/filesystem.py

@@ -75,7 +75,6 @@
     "install_tree",
     "is_exe",
     "join_path",
-    "last_modification_time_recursive",
     "library_extensions",
     "mkdirp",
     "partition_path",
@@ -1470,15 +1469,36 @@ def set_executable(path):
 
 
 @system_path_filter
-def last_modification_time_recursive(path):
-    path = os.path.abspath(path)
-    times = [os.stat(path).st_mtime]
-    times.extend(
-        os.lstat(os.path.join(root, name)).st_mtime
-        for root, dirs, files in os.walk(path)
-        for name in dirs + files
-    )
-    return max(times)
+def recursive_mtime_greater_than(path: str, time: float) -> bool:
+    """Returns true if any file or dir recursively under `path` has mtime greater than `time`."""
+    # use bfs order to increase likelihood of early return
+    queue: Deque[str] = collections.deque()
+
+    if os.stat(path).st_mtime > time:
+        return True
+
+    while queue:
+        current = queue.popleft()
+
+        try:
+            entries = os.scandir(current)
+        except OSError:
+            continue
+
+        with entries:
+            for entry in entries:
+                try:
+                    st = entry.stat(follow_symlinks=False)
+                except OSError:
+                    continue
+
+                if st.st_mtime > time:
+                    return True
+
+                if entry.is_dir(follow_symlinks=False):
+                    queue.append(entry.path)
+
+    return False
 
 
 @system_path_filter
@@ -1740,8 +1760,7 @@ def find(
 
 
 def _log_file_access_issue(e: OSError, path: str) -> None:
-    errno_name = errno.errorcode.get(e.errno, "UNKNOWN")
-    tty.debug(f"find must skip {path}: {errno_name} {e}")
+    tty.debug(f"find must skip {path}: {e}")
 
 
 def _file_id(s: os.stat_result) -> Tuple[int, int]:

lib/spack/llnl/util/lang.py

@@ -72,7 +72,7 @@ def index_by(objects, *funcs):
     if isinstance(f, str):
         f = lambda x: getattr(x, funcs[0])
     elif isinstance(f, tuple):
-        f = lambda x: tuple(getattr(x, p, None) for p in funcs[0])
+        f = lambda x: tuple(getattr(x, p) for p in funcs[0])
 
     result = {}
     for o in objects:
@@ -996,8 +996,11 @@ def _receive_forwarded(self, context: str, exc: Exception, tb: List[str]):
     def grouped_message(self, with_tracebacks: bool = True) -> str:
         """Print out an error message coalescing all the forwarded errors."""
         each_exception_message = [
-            "\n\t{0} raised {1}: {2}\n{3}".format(
-                context, exc.__class__.__name__, exc, f"\n{''.join(tb)}" if with_tracebacks else ""
+            "{0} raised {1}: {2}{3}".format(
+                context,
+                exc.__class__.__name__,
+                exc,
+                "\n{0}".format("".join(tb)) if with_tracebacks else "",
             )
             for context, exc, tb in self.exceptions
         ]

lib/spack/spack/__init__.py

@@ -10,7 +10,7 @@
 import spack.util.git
 
 #: PEP440 canonical <major>.<minor>.<micro>.<devN> string
-__version__ = "1.0.0-alpha.3"
+__version__ = "0.24.0.dev0"
 spack_version = __version__
 
 

lib/spack/spack/audit.py

@@ -1356,14 +1356,8 @@ def _test_detection_by_executable(pkgs, debug_log, error_cls):
 
             def _compare_extra_attribute(_expected, _detected, *, _spec):
                 result = []
-                # Check items are of the same type
-                if not isinstance(_detected, type(_expected)):
-                    _summary = f'{pkg_name}: error when trying to detect "{_expected}"'
-                    _details = [f"{_detected} was detected instead"]
-                    return [error_cls(summary=_summary, details=_details)]
-
                 # If they are string expected is a regex
-                if isinstance(_expected, str):
+                if isinstance(_expected, str) and isinstance(_detected, str):
                     try:
                         _regex = re.compile(_expected)
                     except re.error:
@@ -1379,7 +1373,7 @@ def _compare_extra_attribute(_expected, _detected, *, _spec):
                         _details = [f"{_detected} does not match the regex"]
                         return [error_cls(summary=_summary, details=_details)]
 
-                if isinstance(_expected, dict):
+                elif isinstance(_expected, dict) and isinstance(_detected, dict):
                     _not_detected = set(_expected.keys()) - set(_detected.keys())
                     if _not_detected:
                         _summary = f"{pkg_name}: cannot detect some attributes for spec {_spec}"
@@ -1394,6 +1388,10 @@ def _compare_extra_attribute(_expected, _detected, *, _spec):
                         result.extend(
                             _compare_extra_attribute(_expected[_key], _detected[_key], _spec=_spec)
                         )
+                else:
+                    _summary = f'{pkg_name}: error when trying to detect "{_expected}"'
+                    _details = [f"{_detected} was detected instead"]
+                    return [error_cls(summary=_summary, details=_details)]
 
                 return result
 

lib/spack/spack/binary_distribution.py

@@ -23,7 +23,7 @@
 import urllib.request
 import warnings
 from contextlib import closing
-from typing import IO, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple, Union
+from typing import IO, Callable, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple, Union
 
 import llnl.util.filesystem as fsys
 import llnl.util.lang
@@ -91,6 +91,9 @@
 CURRENT_BUILD_CACHE_LAYOUT_VERSION = 2
 
 
+INDEX_HASH_FILE = "index.json.hash"
+
+
 class BuildCacheDatabase(spack_db.Database):
     """A database for binary buildcaches.
 
@@ -502,7 +505,7 @@ def _fetch_and_cache_index(self, mirror_url, cache_entry={}):
         scheme = urllib.parse.urlparse(mirror_url).scheme
 
         if scheme != "oci" and not web_util.url_exists(
-            url_util.join(mirror_url, BUILD_CACHE_RELATIVE_PATH, "index.json")
+            url_util.join(mirror_url, BUILD_CACHE_RELATIVE_PATH, spack_db.INDEX_JSON_FILE)
         ):
             return False
 
@@ -625,14 +628,7 @@ def tarball_directory_name(spec):
     Return name of the tarball directory according to the convention
     <os>-<architecture>/<compiler>/<package>-<version>/
     """
-    if spec.original_spec_format() < 5:
-        compiler = spec.annotations.compiler_node_attribute
-        assert compiler is not None, "a compiler spec is expected"
-        return spec.format_path(
-            f"{spec.architecture}/{compiler.name}-{compiler.version}/{spec.name}-{spec.version}"
-        )
-
-    return spec.format_path(f"{spec.architecture.platform}/{spec.name}-{spec.version}")
+    return spec.format_path("{architecture}/{compiler.name}-{compiler.version}/{name}-{version}")
 
 
 def tarball_name(spec, ext):
@@ -640,17 +636,9 @@ def tarball_name(spec, ext):
     Return the name of the tarfile according to the convention
     <os>-<architecture>-<package>-<dag_hash><ext>
     """
-    if spec.original_spec_format() < 5:
-        compiler = spec.annotations.compiler_node_attribute
-        assert compiler is not None, "a compiler spec is expected"
-        spec_formatted = (
-            f"{spec.architecture}-{compiler.name}-{compiler.version}-{spec.name}"
-            f"-{spec.version}-{spec.dag_hash()}"
-        )
-    else:
-        spec_formatted = (
-            f"{spec.architecture.platform}-{spec.name}-{spec.version}-{spec.dag_hash()}"
-        )
+    spec_formatted = spec.format_path(
+        "{architecture}-{compiler.name}-{compiler.version}-{name}-{version}-{hash}"
+    )
     return f"{spec_formatted}{ext}"
 
 
@@ -684,19 +672,24 @@ def sign_specfile(key: str, specfile_path: str) -> str:
 
 
 def _read_specs_and_push_index(
-    file_list, read_method, cache_prefix, db: BuildCacheDatabase, temp_dir, concurrency
+    file_list: List[str],
+    read_method: Callable,
+    cache_prefix: str,
+    db: BuildCacheDatabase,
+    temp_dir: str,
+    concurrency: int,
 ):
     """Read all the specs listed in the provided list, using thread given thread parallelism,
         generate the index, and push it to the mirror.
 
     Args:
-        file_list (list(str)): List of urls or file paths pointing at spec files to read
+        file_list: List of urls or file paths pointing at spec files to read
         read_method: A function taking a single argument, either a url or a file path,
             and which reads the spec file at that location, and returns the spec.
-        cache_prefix (str): prefix of the build cache on s3 where index should be pushed.
+        cache_prefix: prefix of the build cache on s3 where index should be pushed.
         db: A spack database used for adding specs and then writing the index.
-        temp_dir (str): Location to write index.json and hash for pushing
-        concurrency (int): Number of parallel processes to use when fetching
+        temp_dir: Location to write index.json and hash for pushing
+        concurrency: Number of parallel processes to use when fetching
     """
     for file in file_list:
         contents = read_method(file)
@@ -714,7 +707,7 @@ def _read_specs_and_push_index(
 
     # Now generate the index, compute its hash, and push the two files to
     # the mirror.
-    index_json_path = os.path.join(temp_dir, "index.json")
+    index_json_path = os.path.join(temp_dir, spack_db.INDEX_JSON_FILE)
     with open(index_json_path, "w", encoding="utf-8") as f:
         db._write_to_file(f)
 
@@ -724,14 +717,14 @@ def _read_specs_and_push_index(
         index_hash = compute_hash(index_string)
 
     # Write the hash out to a local file
-    index_hash_path = os.path.join(temp_dir, "index.json.hash")
+    index_hash_path = os.path.join(temp_dir, INDEX_HASH_FILE)
     with open(index_hash_path, "w", encoding="utf-8") as f:
         f.write(index_hash)
 
     # Push the index itself
     web_util.push_to_url(
         index_json_path,
-        url_util.join(cache_prefix, "index.json"),
+        url_util.join(cache_prefix, spack_db.INDEX_JSON_FILE),
         keep_original=False,
         extra_args={"ContentType": "application/json", "CacheControl": "no-cache"},
     )
@@ -739,7 +732,7 @@ def _read_specs_and_push_index(
     # Push the hash
     web_util.push_to_url(
         index_hash_path,
-        url_util.join(cache_prefix, "index.json.hash"),
+        url_util.join(cache_prefix, INDEX_HASH_FILE),
         keep_original=False,
         extra_args={"ContentType": "text/plain", "CacheControl": "no-cache"},
     )
@@ -876,9 +869,12 @@ def _url_generate_package_index(url: str, tmpdir: str, concurrency: int = 32):
     tty.debug(f"Retrieving spec descriptor files from {url} to build index")
 
     db = BuildCacheDatabase(tmpdir)
+    db._write()
 
     try:
-        _read_specs_and_push_index(file_list, read_fn, url, db, db.database_directory, concurrency)
+        _read_specs_and_push_index(
+            file_list, read_fn, url, db, str(db.database_directory), concurrency
+        )
     except Exception as e:
         raise GenerateIndexError(f"Encountered problem pushing package index to {url}: {e}") from e
 
@@ -1792,7 +1788,7 @@ def _oci_update_index(
         db.mark(spec, "in_buildcache", True)
 
     # Create the index.json file
-    index_json_path = os.path.join(tmpdir, "index.json")
+    index_json_path = os.path.join(tmpdir, spack_db.INDEX_JSON_FILE)
     with open(index_json_path, "w", encoding="utf-8") as f:
         db._write_to_file(f)
 
@@ -2140,10 +2136,9 @@ def fetch_url_to_mirror(url):
 
 
 def dedupe_hardlinks_if_necessary(root, buildinfo):
-    """Updates a buildinfo dict for old archives that did
-    not dedupe hardlinks. De-duping hardlinks is necessary
-    when relocating files in parallel and in-place. This
-    means we must preserve inodes when relocating."""
+    """Updates a buildinfo dict for old archives that did not dedupe hardlinks. De-duping hardlinks
+    is necessary when relocating files in parallel and in-place. This means we must preserve inodes
+    when relocating."""
 
     # New archives don't need this.
     if buildinfo.get("hardlinks_deduped", False):
@@ -2172,69 +2167,47 @@ def dedupe_hardlinks_if_necessary(root, buildinfo):
         buildinfo[key] = new_list
 
 
-def relocate_package(spec):
-    """
-    Relocate the given package
-    """
-    workdir = str(spec.prefix)
-    buildinfo = read_buildinfo_file(workdir)
-    new_layout_root = str(spack.store.STORE.layout.root)
-    new_prefix = str(spec.prefix)
-    new_rel_prefix = str(os.path.relpath(new_prefix, new_layout_root))
-    new_spack_prefix = str(spack.paths.prefix)
-
-    old_sbang_install_path = None
-    if "sbang_install_path" in buildinfo:
-        old_sbang_install_path = str(buildinfo["sbang_install_path"])
+def relocate_package(spec: spack.spec.Spec) -> None:
+    """Relocate binaries and text files in the given spec prefix, based on its buildinfo file."""
+    spec_prefix = str(spec.prefix)
+    buildinfo = read_buildinfo_file(spec_prefix)
     old_layout_root = str(buildinfo["buildpath"])
-    old_spack_prefix = str(buildinfo.get("spackprefix"))
-    old_rel_prefix = buildinfo.get("relative_prefix")
-    old_prefix = os.path.join(old_layout_root, old_rel_prefix)
 
-    # Warn about old style tarballs created with the now removed --rel flag.
+    # Warn about old style tarballs created with the --rel flag (removed in Spack v0.20)
     if buildinfo.get("relative_rpaths", False):
         tty.warn(
-            f"Tarball for {spec} uses relative rpaths, " "which can cause library loading issues."
+            f"Tarball for {spec} uses relative rpaths, which can cause library loading issues."
         )
 
-    # In the past prefix_to_hash was the default and externals were not dropped, so prefixes
-    # were not unique.
+    # In Spack 0.19 and older prefix_to_hash was the default and externals were not dropped, so
+    # prefixes were not unique.
     if "hash_to_prefix" in buildinfo:
         hash_to_old_prefix = buildinfo["hash_to_prefix"]
     elif "prefix_to_hash" in buildinfo:
-        hash_to_old_prefix = dict((v, k) for (k, v) in buildinfo["prefix_to_hash"].items())
+        hash_to_old_prefix = {v: k for (k, v) in buildinfo["prefix_to_hash"].items()}
     else:
-        hash_to_old_prefix = dict()
+        raise NewLayoutException(
+            "Package tarball was created from an install prefix with a different directory layout "
+            "and an older buildcache create implementation. It cannot be relocated."
+        )
 
-    if old_rel_prefix != new_rel_prefix and not hash_to_old_prefix:
-        msg = "Package tarball was created from an install "
-        msg += "prefix with a different directory layout and an older "
-        msg += "buildcache create implementation. It cannot be relocated."
-        raise NewLayoutException(msg)
+    prefix_to_prefix: Dict[str, str] = {}
 
-    # Spurious replacements (e.g. sbang) will cause issues with binaries
-    # For example, the new sbang can be longer than the old one.
-    # Hence 2 dictionaries are maintained here.
-    prefix_to_prefix_text = collections.OrderedDict()
-    prefix_to_prefix_bin = collections.OrderedDict()
+    if "sbang_install_path" in buildinfo:
+        old_sbang_install_path = str(buildinfo["sbang_install_path"])
+        prefix_to_prefix[old_sbang_install_path] = spack.hooks.sbang.sbang_install_path()
 
-    if old_sbang_install_path:
-        install_path = spack.hooks.sbang.sbang_install_path()
-        prefix_to_prefix_text[old_sbang_install_path] = install_path
+    # First match specific prefix paths. Possibly the *local* install prefix of some dependency is
+    # in an upstream, so we cannot assume the original spack store root can be mapped uniformly to
+    # the new spack store root.
 
-    # First match specific prefix paths. Possibly the *local* install prefix
-    # of some dependency is in an upstream, so we cannot assume the original
-    # spack store root can be mapped uniformly to the new spack store root.
-    #
-    # If the spec is spliced, we need to handle the simultaneous mapping
-    # from the old install_tree to the new install_tree and from the build_spec
-    # to the spliced spec.
-    # Because foo.build_spec is foo for any non-spliced spec, we can simplify
-    # by checking for spliced-in nodes by checking for nodes not in the build_spec
-    # without any explicit check for whether the spec is spliced.
-    # An analog in this algorithm is any spec that shares a name or provides the same virtuals
-    # in the context of the relevant root spec. This ensures that the analog for a spec s
-    # is the spec that s replaced when we spliced.
+    # If the spec is spliced, we need to handle the simultaneous mapping from the old install_tree
+    # to the new install_tree and from the build_spec to the spliced spec. Because foo.build_spec
+    # is foo for any non-spliced spec, we can simplify by checking for spliced-in nodes by checking
+    # for nodes not in the build_spec without any explicit check for whether the spec is spliced.
+    # An analog in this algorithm is any spec that shares a name or provides the same virtuals in
+    # the context of the relevant root spec. This ensures that the analog for a spec s is the spec
+    # that s replaced when we spliced.
     relocation_specs = specs_to_relocate(spec)
     build_spec_ids = set(id(s) for s in spec.build_spec.traverse(deptype=dt.ALL & ~dt.BUILD))
     for s in relocation_specs:
@@ -2254,72 +2227,48 @@ def relocate_package(spec):
         lookup_dag_hash = analog.dag_hash()
         if lookup_dag_hash in hash_to_old_prefix:
             old_dep_prefix = hash_to_old_prefix[lookup_dag_hash]
-            prefix_to_prefix_bin[old_dep_prefix] = str(s.prefix)
-            prefix_to_prefix_text[old_dep_prefix] = str(s.prefix)
+            prefix_to_prefix[old_dep_prefix] = str(s.prefix)
 
     # Only then add the generic fallback of install prefix -> install prefix.
-    prefix_to_prefix_text[old_prefix] = new_prefix
-    prefix_to_prefix_bin[old_prefix] = new_prefix
-    prefix_to_prefix_text[old_layout_root] = new_layout_root
-    prefix_to_prefix_bin[old_layout_root] = new_layout_root
+    prefix_to_prefix[old_layout_root] = str(spack.store.STORE.layout.root)
 
-    # This is vestigial code for the *old* location of sbang. Previously,
-    # sbang was a bash script, and it lived in the spack prefix. It is
-    # now a POSIX script that lives in the install prefix. Old packages
-    # will have the old sbang location in their shebangs.
-    orig_sbang = "#!/bin/bash {0}/bin/sbang".format(old_spack_prefix)
-    new_sbang = spack.hooks.sbang.sbang_shebang_line()
-    prefix_to_prefix_text[orig_sbang] = new_sbang
+    # Delete identity mappings from prefix_to_prefix
+    prefix_to_prefix = {k: v for k, v in prefix_to_prefix.items() if k != v}
 
-    tty.debug("Relocating package from", "%s to %s." % (old_layout_root, new_layout_root))
+    # If there's nothing to relocate, we're done.
+    if not prefix_to_prefix:
+        return
+
+    for old, new in prefix_to_prefix.items():
+        tty.debug(f"Relocating: {old} => {new}.")
 
     # Old archives may have hardlinks repeated.
-    dedupe_hardlinks_if_necessary(workdir, buildinfo)
+    dedupe_hardlinks_if_necessary(spec_prefix, buildinfo)
 
     # Text files containing the prefix text
-    text_names = [os.path.join(workdir, f) for f in buildinfo["relocate_textfiles"]]
+    textfiles = [os.path.join(spec_prefix, f) for f in buildinfo["relocate_textfiles"]]
+    binaries = [os.path.join(spec_prefix, f) for f in buildinfo.get("relocate_binaries")]
+    links = [os.path.join(spec_prefix, f) for f in buildinfo.get("relocate_links", [])]
 
-    # If we are not installing back to the same install tree do the relocation
-    if old_prefix != new_prefix:
-        files_to_relocate = [
-            os.path.join(workdir, filename) for filename in buildinfo.get("relocate_binaries")
-        ]
-        # If the buildcache was not created with relativized rpaths
-        # do the relocation of path in binaries
-        platform = spack.platforms.by_name(spec.platform)
-        if "macho" in platform.binary_formats:
-            relocate.relocate_macho_binaries(files_to_relocate, prefix_to_prefix_bin)
-        elif "elf" in platform.binary_formats:
-            # The new ELF dynamic section relocation logic only handles absolute to
-            # absolute relocation.
-            relocate.relocate_elf_binaries(files_to_relocate, prefix_to_prefix_bin)
+    platform = spack.platforms.by_name(spec.platform)
+    if "macho" in platform.binary_formats:
+        relocate.relocate_macho_binaries(binaries, prefix_to_prefix)
+    elif "elf" in platform.binary_formats:
+        relocate.relocate_elf_binaries(binaries, prefix_to_prefix)
 
-        # Relocate links to the new install prefix
-        links = [os.path.join(workdir, f) for f in buildinfo.get("relocate_links", [])]
-        relocate.relocate_links(links, prefix_to_prefix_bin)
+    relocate.relocate_links(links, prefix_to_prefix)
+    relocate.relocate_text(textfiles, prefix_to_prefix)
+    changed_files = relocate.relocate_text_bin(binaries, prefix_to_prefix)
 
-        # For all buildcaches
-        # relocate the install prefixes in text files including dependencies
-        relocate.relocate_text(text_names, prefix_to_prefix_text)
-
-        # relocate the install prefixes in binary files including dependencies
-        changed_files = relocate.relocate_text_bin(files_to_relocate, prefix_to_prefix_bin)
-
-        # Add ad-hoc signatures to patched macho files when on macOS.
-        if "macho" in platform.binary_formats and sys.platform == "darwin":
-            codesign = which("codesign")
-            if not codesign:
-                return
-            for binary in changed_files:
-                # preserve the original inode by running codesign on a copy
-                with fsys.edit_in_place_through_temporary_file(binary) as tmp_binary:
-                    codesign("-fs-", tmp_binary)
-
-    # If we are installing back to the same location
-    # relocate the sbang location if the spack directory changed
-    else:
-        if old_spack_prefix != new_spack_prefix:
-            relocate.relocate_text(text_names, prefix_to_prefix_text)
+    # Add ad-hoc signatures to patched macho files when on macOS.
+    if "macho" in platform.binary_formats and sys.platform == "darwin":
+        codesign = which("codesign")
+        if not codesign:
+            return
+        for binary in changed_files:
+            # preserve the original inode by running codesign on a copy
+            with fsys.edit_in_place_through_temporary_file(binary) as tmp_binary:
+                codesign("-fs-", tmp_binary)
 
 
 def _extract_inner_tarball(spec, filename, extract_to, signature_required: bool, remote_checksum):
@@ -2997,7 +2946,7 @@ def __init__(self, url, local_hash, urlopen=web_util.urlopen):
 
     def get_remote_hash(self):
         # Failure to fetch index.json.hash is not fatal
-        url_index_hash = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json.hash")
+        url_index_hash = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, INDEX_HASH_FILE)
         try:
             response = self.urlopen(urllib.request.Request(url_index_hash, headers=self.headers))
         except (TimeoutError, urllib.error.URLError):
@@ -3018,7 +2967,7 @@ def conditional_fetch(self) -> FetchIndexResult:
             return FetchIndexResult(etag=None, hash=None, data=None, fresh=True)
 
         # Otherwise, download index.json
-        url_index = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json")
+        url_index = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, spack_db.INDEX_JSON_FILE)
 
         try:
             response = self.urlopen(urllib.request.Request(url_index, headers=self.headers))
@@ -3062,7 +3011,7 @@ def __init__(self, url, etag, urlopen=web_util.urlopen):
 
     def conditional_fetch(self) -> FetchIndexResult:
         # Just do a conditional fetch immediately
-        url = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json")
+        url = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, spack_db.INDEX_JSON_FILE)
         headers = {"User-Agent": web_util.SPACK_USER_AGENT, "If-None-Match": f'"{self.etag}"'}
 
         try:

lib/spack/spack/bootstrap/_common.py

@@ -10,7 +10,9 @@
 import sys
 import sysconfig
 import warnings
-from typing import Dict, Optional, Sequence, Union
+from typing import Optional, Sequence, Union
+
+from typing_extensions import TypedDict
 
 import archspec.cpu
 
@@ -18,13 +20,17 @@
 from llnl.util import tty
 
 import spack.platforms
+import spack.spec
 import spack.store
 import spack.util.environment
 import spack.util.executable
 
 from .config import spec_for_current_python
 
-QueryInfo = Dict[str, "spack.spec.Spec"]
+
+class QueryInfo(TypedDict, total=False):
+    spec: spack.spec.Spec
+    command: spack.util.executable.Executable
 
 
 def _python_import(module: str) -> bool:
@@ -211,7 +217,9 @@ def _executables_in_store(
             ):
                 spack.util.environment.path_put_first("PATH", [bin_dir])
                 if query_info is not None:
-                    query_info["command"] = spack.util.executable.which(*executables, path=bin_dir)
+                    query_info["command"] = spack.util.executable.which(
+                        *executables, path=bin_dir, required=True
+                    )
                     query_info["spec"] = concrete_spec
                 return True
     return False
@@ -226,13 +234,12 @@ def _root_spec(spec_str: str) -> str:
     # Add a compiler and platform requirement to the root spec.
     platform = str(spack.platforms.host())
 
-    # FIXME (compiler as nodes): recover the compiler for source bootstrapping
-    # if platform == "darwin":
-    #    spec_str += " %apple-clang"
-    if platform == "windows":
+    if platform == "darwin":
+        spec_str += " %apple-clang"
+    elif platform == "windows":
         spec_str += " %msvc"
-    # elif platform == "linux":
-    #    spec_str += " %gcc"
+    elif platform == "linux":
+        spec_str += " %gcc"
     elif platform == "freebsd":
         spec_str += " %clang"
     spec_str += f" platform={platform}"

lib/spack/spack/bootstrap/clingo.py

@@ -15,13 +15,11 @@
 
 import archspec.cpu
 
-import spack.compilers.config
-import spack.compilers.libraries
-import spack.config
+import spack.compiler
+import spack.compilers
 import spack.platforms
 import spack.spec
 import spack.traverse
-import spack.version
 
 from .config import spec_for_current_python
 
@@ -40,7 +38,7 @@ def __init__(self, configuration):
 
         self.external_cmake, self.external_bison = self._externals_from_yaml(configuration)
 
-    def _valid_compiler_or_raise(self):
+    def _valid_compiler_or_raise(self) -> "spack.compiler.Compiler":
         if str(self.host_platform) == "linux":
             compiler_name = "gcc"
         elif str(self.host_platform) == "darwin":
@@ -48,30 +46,17 @@ def _valid_compiler_or_raise(self):
         elif str(self.host_platform) == "windows":
             compiler_name = "msvc"
         elif str(self.host_platform) == "freebsd":
-            compiler_name = "llvm"
+            compiler_name = "clang"
         else:
             raise RuntimeError(f"Cannot bootstrap clingo from sources on {self.host_platform}")
-
-        candidates = [
-            x
-            for x in spack.compilers.config.CompilerFactory.from_packages_yaml(spack.config.CONFIG)
-            if x.name == compiler_name
-        ]
+        candidates = spack.compilers.compilers_for_spec(
+            compiler_name, arch_spec=self.host_architecture
+        )
         if not candidates:
             raise RuntimeError(
                 f"Cannot find any version of {compiler_name} to bootstrap clingo from sources"
             )
-        candidates.sort(key=lambda x: x.version, reverse=True)
-        best = candidates[0]
-        # Get compilers for bootstrapping from the 'builtin' repository
-        best.namespace = "builtin"
-        # If the compiler does not support C++ 14, fail with a legible error message
-        try:
-            _ = best.package.standard_flag(language="cxx", standard="14")
-        except RuntimeError as e:
-            raise RuntimeError(
-                "cannot find a compiler supporting C++ 14 [needed to bootstrap clingo]"
-            ) from e
+        candidates.sort(key=lambda x: x.spec.version, reverse=True)
         return candidates[0]
 
     def _externals_from_yaml(
@@ -90,6 +75,9 @@ def _externals_from_yaml(
                 if not s.satisfies(requirements[pkg_name]):
                     continue
 
+                if not s.intersects(f"%{self.host_compiler.spec}"):
+                    continue
+
                 if not s.intersects(f"arch={self.host_architecture}"):
                     continue
 
@@ -122,14 +110,11 @@ def concretize(self) -> "spack.spec.Spec":
         # Tweak it to conform to the host architecture
         for node in s.traverse():
             node.architecture.os = str(self.host_os)
+            node.compiler = self.host_compiler.spec
             node.architecture = self.host_architecture
 
             if node.name == "gcc-runtime":
-                node.versions = self.host_compiler.versions
-
-        # Can't use re2c@3.1 with Python 3.6
-        if self.host_python.satisfies("@3.6"):
-            s["re2c"].versions.versions = [spack.version.from_string("=2.2")]
+                node.versions = self.host_compiler.spec.versions
 
         for edge in spack.traverse.traverse_edges([s], cover="edges"):
             if edge.spec.name == "python":
@@ -141,9 +126,6 @@ def concretize(self) -> "spack.spec.Spec":
             if edge.spec.name == "cmake" and self.external_cmake:
                 edge.spec = self.external_cmake
 
-            if edge.spec.name == self.host_compiler.name:
-                edge.spec = self.host_compiler
-
             if "libc" in edge.virtuals:
                 edge.spec = self.host_libc
 
@@ -159,12 +141,12 @@ def python_external_spec(self) -> "spack.spec.Spec":
         return self._external_spec(result)
 
     def libc_external_spec(self) -> "spack.spec.Spec":
-        detector = spack.compilers.libraries.CompilerPropertyDetector(self.host_compiler)
-        result = detector.default_libc()
+        result = self.host_compiler.default_libc
         return self._external_spec(result)
 
     def _external_spec(self, initial_spec) -> "spack.spec.Spec":
         initial_spec.namespace = "builtin"
+        initial_spec.compiler = self.host_compiler.spec
         initial_spec.architecture = self.host_architecture
         for flag_type in spack.spec.FlagMap.valid_compiler_flags():
             initial_spec.compiler_flags[flag_type] = []

lib/spack/spack/bootstrap/config.py

@@ -10,7 +10,7 @@
 
 from llnl.util import tty
 
-import spack.compilers.config
+import spack.compilers
 import spack.config
 import spack.environment
 import spack.modules
@@ -142,8 +142,8 @@ def _bootstrap_config_scopes() -> Sequence["spack.config.ConfigScope"]:
 
 def _add_compilers_if_missing() -> None:
     arch = spack.spec.ArchSpec.frontend_arch()
-    if not spack.compilers.config.compilers_for_arch(arch):
-        spack.compilers.config.find_compilers()
+    if not spack.compilers.compilers_for_arch(arch):
+        spack.compilers.find_compilers()
 
 
 @contextlib.contextmanager

lib/spack/spack/bootstrap/core.py

@@ -34,8 +34,10 @@
 from llnl.util.lang import GroupedExceptionHandler
 
 import spack.binary_distribution
+import spack.concretize
 import spack.config
 import spack.detection
+import spack.error
 import spack.mirrors.mirror
 import spack.platforms
 import spack.spec
@@ -47,7 +49,13 @@
 import spack.version
 from spack.installer import PackageInstaller
 
-from ._common import _executables_in_store, _python_import, _root_spec, _try_import_from_store
+from ._common import (
+    QueryInfo,
+    _executables_in_store,
+    _python_import,
+    _root_spec,
+    _try_import_from_store,
+)
 from .clingo import ClingoBootstrapConcretizer
 from .config import spack_python_interpreter, spec_for_current_python
 
@@ -134,7 +142,7 @@ class BuildcacheBootstrapper(Bootstrapper):
 
     def __init__(self, conf) -> None:
         super().__init__(conf)
-        self.last_search: Optional[ConfigDictionary] = None
+        self.last_search: Optional[QueryInfo] = None
         self.config_scope_name = f"bootstrap_buildcache-{uuid.uuid4()}"
 
     @staticmethod
@@ -211,14 +219,14 @@ def _install_and_test(
                 for _, pkg_hash, pkg_sha256 in item["binaries"]:
                     self._install_by_hash(pkg_hash, pkg_sha256, bincache_platform)
 
-                info: ConfigDictionary = {}
+                info: QueryInfo = {}
                 if test_fn(query_spec=abstract_spec, query_info=info):
                     self.last_search = info
                     return True
         return False
 
     def try_import(self, module: str, abstract_spec_str: str) -> bool:
-        info: ConfigDictionary
+        info: QueryInfo
         test_fn, info = functools.partial(_try_import_from_store, module), {}
         if test_fn(query_spec=abstract_spec_str, query_info=info):
             return True
@@ -231,7 +239,7 @@ def try_import(self, module: str, abstract_spec_str: str) -> bool:
         return self._install_and_test(abstract_spec, bincache_platform, data, test_fn)
 
     def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool:
-        info: ConfigDictionary
+        info: QueryInfo
         test_fn, info = functools.partial(_executables_in_store, executables), {}
         if test_fn(query_spec=abstract_spec_str, query_info=info):
             self.last_search = info
@@ -249,11 +257,11 @@ class SourceBootstrapper(Bootstrapper):
 
     def __init__(self, conf) -> None:
         super().__init__(conf)
-        self.last_search: Optional[ConfigDictionary] = None
+        self.last_search: Optional[QueryInfo] = None
         self.config_scope_name = f"bootstrap_source-{uuid.uuid4()}"
 
     def try_import(self, module: str, abstract_spec_str: str) -> bool:
-        info: ConfigDictionary = {}
+        info: QueryInfo = {}
         if _try_import_from_store(module, abstract_spec_str, query_info=info):
             self.last_search = info
             return True
@@ -270,22 +278,17 @@ def try_import(self, module: str, abstract_spec_str: str) -> bool:
                 bootstrapper = ClingoBootstrapConcretizer(configuration=spack.config.CONFIG)
                 concrete_spec = bootstrapper.concretize()
             else:
-                concrete_spec = spack.spec.Spec(
+                abstract_spec = spack.spec.Spec(
                     abstract_spec_str + " ^" + spec_for_current_python()
                 )
-                concrete_spec.concretize()
+                concrete_spec = spack.concretize.concretize_one(abstract_spec)
 
         msg = "[BOOTSTRAP MODULE {0}] Try installing '{1}' from sources"
         tty.debug(msg.format(module, abstract_spec_str))
 
         # Install the spec that should make the module importable
         with spack.config.override(self.mirror_scope):
-            PackageInstaller(
-                [concrete_spec.package],
-                fail_fast=True,
-                package_use_cache=False,
-                dependencies_use_cache=False,
-            ).install()
+            PackageInstaller([concrete_spec.package], fail_fast=True).install()
 
         if _try_import_from_store(module, query_spec=concrete_spec, query_info=info):
             self.last_search = info
@@ -293,7 +296,7 @@ def try_import(self, module: str, abstract_spec_str: str) -> bool:
         return False
 
     def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool:
-        info: ConfigDictionary = {}
+        info: QueryInfo = {}
         if _executables_in_store(executables, abstract_spec_str, query_info=info):
             self.last_search = info
             return True
@@ -304,7 +307,7 @@ def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bo
         # might reduce compilation time by a fair amount
         _add_externals_if_missing()
 
-        concrete_spec = spack.spec.Spec(abstract_spec_str).concretized()
+        concrete_spec = spack.concretize.concretize_one(abstract_spec_str)
         msg = "[BOOTSTRAP] Try installing '{0}' from sources"
         tty.debug(msg.format(abstract_spec_str))
         with spack.config.override(self.mirror_scope):
@@ -321,10 +324,9 @@ def create_bootstrapper(conf: ConfigDictionary):
     return _bootstrap_methods[btype](conf)
 
 
-def source_is_enabled(conf: ConfigDictionary):
-    """Raise ValueError if the source is not enabled for bootstrapping"""
-    trusted, name = spack.config.get("bootstrap:trusted"), conf["name"]
-    return trusted.get(name, False)
+def source_is_enabled(conf: ConfigDictionary) -> bool:
+    """Returns true if the source is not enabled for bootstrapping"""
+    return spack.config.get("bootstrap:trusted").get(conf["name"], False)
 
 
 def ensure_module_importable_or_raise(module: str, abstract_spec: Optional[str] = None):
@@ -356,20 +358,21 @@ def ensure_module_importable_or_raise(module: str, abstract_spec: Optional[str]
     for current_config in bootstrapping_sources():
         if not source_is_enabled(current_config):
             continue
-
         with exception_handler.forward(current_config["name"], Exception):
-            current_bootstrapper = create_bootstrapper(current_config)
-            if current_bootstrapper.try_import(module, abstract_spec):
+            if create_bootstrapper(current_config).try_import(module, abstract_spec):
                 return
 
-    assert exception_handler, (
-        f"expected at least one exception to have been raised at this point: "
-        f"while bootstrapping {module}"
-    )
     msg = f'cannot bootstrap the "{module}" Python module '
     if abstract_spec:
         msg += f'from spec "{abstract_spec}" '
-    msg += exception_handler.grouped_message(with_tracebacks=tty.is_debug())
+
+    if not exception_handler:
+        msg += ": no bootstrapping sources are enabled"
+    elif spack.error.debug or spack.error.SHOW_BACKTRACE:
+        msg += exception_handler.grouped_message(with_tracebacks=True)
+    else:
+        msg += exception_handler.grouped_message(with_tracebacks=False)
+        msg += "\nRun `spack --backtrace ...` for more detailed errors"
     raise ImportError(msg)
 
 
@@ -417,6 +420,7 @@ def ensure_executables_in_path_or_raise(
                     current_bootstrapper.last_search["spec"],
                     current_bootstrapper.last_search["command"],
                 )
+                assert cmd is not None, "expected an Executable"
                 cmd.add_default_envmod(
                     spack.user_environment.environment_modifications_for_specs(
                         concrete_spec, set_package_py_globals=False
@@ -424,18 +428,17 @@ def ensure_executables_in_path_or_raise(
                 )
                 return cmd
 
-    assert exception_handler, (
-        f"expected at least one exception to have been raised at this point: "
-        f"while bootstrapping {executables_str}"
-    )
     msg = f"cannot bootstrap any of the {executables_str} executables "
     if abstract_spec:
         msg += f'from spec "{abstract_spec}" '
-    if tty.is_debug():
+
+    if not exception_handler:
+        msg += ": no bootstrapping sources are enabled"
+    elif spack.error.debug or spack.error.SHOW_BACKTRACE:
         msg += exception_handler.grouped_message(with_tracebacks=True)
     else:
         msg += exception_handler.grouped_message(with_tracebacks=False)
-        msg += "\nRun `spack --debug ...` for more detailed errors"
+        msg += "\nRun `spack --backtrace ...` for more detailed errors"
     raise RuntimeError(msg)
 
 

lib/spack/spack/bootstrap/status.py

@@ -63,7 +63,6 @@ def _missing(name: str, purpose: str, system_only: bool = True) -> str:
 
 def _core_requirements() -> List[RequiredResponseType]:
     _core_system_exes = {
-        "make": _missing("make", "required to build software from sources"),
         "patch": _missing("patch", "required to patch source code before building"),
         "tar": _missing("tar", "required to manage code archives"),
         "gzip": _missing("gzip", "required to compress/decompress code archives"),

lib/spack/spack/build_environment.py

@@ -36,6 +36,7 @@
 import multiprocessing
 import os
 import re
+import stat
 import sys
 import traceback
 import types
@@ -43,7 +44,19 @@
 from enum import Flag, auto
 from itertools import chain
 from multiprocessing.connection import Connection
-from typing import Callable, Dict, List, Optional, Set, Tuple
+from typing import (
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Sequence,
+    Set,
+    TextIO,
+    Tuple,
+    Type,
+    Union,
+    overload,
+)
 
 import archspec.cpu
 
@@ -58,7 +71,7 @@
 import spack.build_systems.meson
 import spack.build_systems.python
 import spack.builder
-import spack.compilers.libraries
+import spack.compilers
 import spack.config
 import spack.deptypes as dt
 import spack.error
@@ -72,6 +85,7 @@
 import spack.store
 import spack.subprocess_context
 import spack.util.executable
+import spack.util.libc
 from spack import traverse
 from spack.context import Context
 from spack.error import InstallError, NoHeadersError, NoLibrariesError
@@ -79,7 +93,6 @@
 from spack.util.environment import (
     SYSTEM_DIR_CASE_ENTRY,
     EnvironmentModifications,
-    PrependPath,
     env_flag,
     filter_system_paths,
     get_path,
@@ -145,48 +158,128 @@ def get_effective_jobs(jobs, parallel=True, supports_jobserver=False):
 
 
 class MakeExecutable(Executable):
-    """Special callable executable object for make so the user can specify
-    parallelism options on a per-invocation basis.  Specifying
-    'parallel' to the call will override whatever the package's
-    global setting is, so you can either default to true or false and
-    override particular calls. Specifying 'jobs_env' to a particular
-    call will name an environment variable which will be set to the
-    parallelism level (without affecting the normal invocation with
-    -j).
+    """Special callable executable object for make so the user can specify parallelism options
+    on a per-invocation basis.
     """
 
-    def __init__(self, name, jobs, **kwargs):
-        supports_jobserver = kwargs.pop("supports_jobserver", True)
-        super().__init__(name, **kwargs)
+    def __init__(self, name: str, *, jobs: int, supports_jobserver: bool = True) -> None:
+        super().__init__(name)
         self.supports_jobserver = supports_jobserver
         self.jobs = jobs
 
-    def __call__(self, *args, **kwargs):
-        """parallel, and jobs_env from kwargs are swallowed and used here;
-        remaining arguments are passed through to the superclass.
-        """
-        parallel = kwargs.pop("parallel", True)
-        jobs_env = kwargs.pop("jobs_env", None)
-        jobs_env_supports_jobserver = kwargs.pop("jobs_env_supports_jobserver", False)
+    @overload
+    def __call__(
+        self,
+        *args: str,
+        parallel: bool = ...,
+        jobs_env: Optional[str] = ...,
+        jobs_env_supports_jobserver: bool = ...,
+        fail_on_error: bool = ...,
+        ignore_errors: Union[int, Sequence[int]] = ...,
+        ignore_quotes: Optional[bool] = ...,
+        timeout: Optional[int] = ...,
+        env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,
+        extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,
+        input: Optional[TextIO] = ...,
+        output: Union[Optional[TextIO], str] = ...,
+        error: Union[Optional[TextIO], str] = ...,
+        _dump_env: Optional[Dict[str, str]] = ...,
+    ) -> None: ...
 
+    @overload
+    def __call__(
+        self,
+        *args: str,
+        parallel: bool = ...,
+        jobs_env: Optional[str] = ...,
+        jobs_env_supports_jobserver: bool = ...,
+        fail_on_error: bool = ...,
+        ignore_errors: Union[int, Sequence[int]] = ...,
+        ignore_quotes: Optional[bool] = ...,
+        timeout: Optional[int] = ...,
+        env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,
+        extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,
+        input: Optional[TextIO] = ...,
+        output: Union[Type[str], Callable] = ...,
+        error: Union[Optional[TextIO], str, Type[str], Callable] = ...,
+        _dump_env: Optional[Dict[str, str]] = ...,
+    ) -> str: ...
+
+    @overload
+    def __call__(
+        self,
+        *args: str,
+        parallel: bool = ...,
+        jobs_env: Optional[str] = ...,
+        jobs_env_supports_jobserver: bool = ...,
+        fail_on_error: bool = ...,
+        ignore_errors: Union[int, Sequence[int]] = ...,
+        ignore_quotes: Optional[bool] = ...,
+        timeout: Optional[int] = ...,
+        env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,
+        extra_env: Optional[Union[Dict[str, str], EnvironmentModifications]] = ...,
+        input: Optional[TextIO] = ...,
+        output: Union[Optional[TextIO], str, Type[str], Callable] = ...,
+        error: Union[Type[str], Callable] = ...,
+        _dump_env: Optional[Dict[str, str]] = ...,
+    ) -> str: ...
+
+    def __call__(
+        self,
+        *args: str,
+        parallel: bool = True,
+        jobs_env: Optional[str] = None,
+        jobs_env_supports_jobserver: bool = False,
+        **kwargs,
+    ) -> Optional[str]:
+        """Runs this "make" executable in a subprocess.
+
+        Args:
+            parallel: if False, parallelism is disabled
+            jobs_env: environment variable that will be set to the current level of parallelism
+            jobs_env_supports_jobserver: whether the jobs env supports a job server
+
+        For all the other **kwargs, refer to the base class.
+        """
         jobs = get_effective_jobs(
             self.jobs, parallel=parallel, supports_jobserver=self.supports_jobserver
         )
         if jobs is not None:
-            args = ("-j{0}".format(jobs),) + args
+            args = (f"-j{jobs}",) + args
 
         if jobs_env:
-            # Caller wants us to set an environment variable to
-            # control the parallelism.
+            # Caller wants us to set an environment variable to control the parallelism
             jobs_env_jobs = get_effective_jobs(
                 self.jobs, parallel=parallel, supports_jobserver=jobs_env_supports_jobserver
             )
             if jobs_env_jobs is not None:
-                kwargs["extra_env"] = {jobs_env: str(jobs_env_jobs)}
+                extra_env = kwargs.setdefault("extra_env", {})
+                extra_env.update({jobs_env: str(jobs_env_jobs)})
 
         return super().__call__(*args, **kwargs)
 
 
+class UndeclaredDependencyError(spack.error.SpackError):
+    """Raised if a dependency is invoking an executable through a module global, without
+    declaring a dependency on it.
+    """
+
+
+class DeprecatedExecutable:
+    def __init__(self, pkg: str, exe: str, exe_pkg: str) -> None:
+        self.pkg = pkg
+        self.exe = exe
+        self.exe_pkg = exe_pkg
+
+    def __call__(self, *args, **kwargs):
+        raise UndeclaredDependencyError(
+            f"{self.pkg} is using {self.exe} without declaring a dependency on {self.exe_pkg}"
+        )
+
+    def add_default_env(self, key: str, value: str):
+        self.__call__()
+
+
 def clean_environment():
     # Stuff in here sanitizes the build environment to eliminate
     # anything the user has set that may interfere. We apply it immediately
@@ -208,11 +301,13 @@ def clean_environment():
     env.unset("CPLUS_INCLUDE_PATH")
     env.unset("OBJC_INCLUDE_PATH")
 
+    # prevent configure scripts from sourcing variables from config site file (AC_SITE_LOAD).
+    env.set("CONFIG_SITE", os.devnull)
     env.unset("CMAKE_PREFIX_PATH")
+
     env.unset("PYTHONPATH")
     env.unset("R_HOME")
     env.unset("R_ENVIRON")
-
     env.unset("LUA_PATH")
     env.unset("LUA_CPATH")
 
@@ -295,10 +390,62 @@ def _add_werror_handling(keep_werror, env):
     env.set("SPACK_COMPILER_FLAGS_REPLACE", " ".join(["|".join(item) for item in replace_flags]))
 
 
-def set_wrapper_environment_variables_for_flags(pkg, env):
+def set_compiler_environment_variables(pkg, env):
     assert pkg.spec.concrete
+    compiler = pkg.compiler
     spec = pkg.spec
 
+    # Make sure the executables for this compiler exist
+    compiler.verify_executables()
+
+    # Set compiler variables used by CMake and autotools
+    assert all(key in compiler.link_paths for key in ("cc", "cxx", "f77", "fc"))
+
+    # Populate an object with the list of environment modifications
+    # and return it
+    # TODO : add additional kwargs for better diagnostics, like requestor,
+    # ttyout, ttyerr, etc.
+    link_dir = spack.paths.build_env_path
+
+    # Set SPACK compiler variables so that our wrapper knows what to
+    # call.  If there is no compiler configured then use a default
+    # wrapper which will emit an error if it is used.
+    if compiler.cc:
+        env.set("SPACK_CC", compiler.cc)
+        env.set("CC", os.path.join(link_dir, compiler.link_paths["cc"]))
+    else:
+        env.set("CC", os.path.join(link_dir, "cc"))
+    if compiler.cxx:
+        env.set("SPACK_CXX", compiler.cxx)
+        env.set("CXX", os.path.join(link_dir, compiler.link_paths["cxx"]))
+    else:
+        env.set("CC", os.path.join(link_dir, "c++"))
+    if compiler.f77:
+        env.set("SPACK_F77", compiler.f77)
+        env.set("F77", os.path.join(link_dir, compiler.link_paths["f77"]))
+    else:
+        env.set("F77", os.path.join(link_dir, "f77"))
+    if compiler.fc:
+        env.set("SPACK_FC", compiler.fc)
+        env.set("FC", os.path.join(link_dir, compiler.link_paths["fc"]))
+    else:
+        env.set("FC", os.path.join(link_dir, "fc"))
+
+    # Set SPACK compiler rpath flags so that our wrapper knows what to use
+    env.set("SPACK_CC_RPATH_ARG", compiler.cc_rpath_arg)
+    env.set("SPACK_CXX_RPATH_ARG", compiler.cxx_rpath_arg)
+    env.set("SPACK_F77_RPATH_ARG", compiler.f77_rpath_arg)
+    env.set("SPACK_FC_RPATH_ARG", compiler.fc_rpath_arg)
+    env.set("SPACK_LINKER_ARG", compiler.linker_arg)
+
+    # Check whether we want to force RPATH or RUNPATH
+    if spack.config.get("config:shared_linking:type") == "rpath":
+        env.set("SPACK_DTAGS_TO_STRIP", compiler.enable_new_dtags)
+        env.set("SPACK_DTAGS_TO_ADD", compiler.disable_new_dtags)
+    else:
+        env.set("SPACK_DTAGS_TO_STRIP", compiler.disable_new_dtags)
+        env.set("SPACK_DTAGS_TO_ADD", compiler.enable_new_dtags)
+
     if pkg.keep_werror is not None:
         keep_werror = pkg.keep_werror
     else:
@@ -306,6 +453,10 @@ def set_wrapper_environment_variables_for_flags(pkg, env):
 
     _add_werror_handling(keep_werror, env)
 
+    # Set the target parameters that the compiler will add
+    isa_arg = optimization_flags(compiler, spec.target)
+    env.set("SPACK_TARGET_ARGS", isa_arg)
+
     # Trap spack-tracked compiler flags as appropriate.
     # env_flags are easy to accidentally override.
     inject_flags = {}
@@ -338,23 +489,75 @@ def set_wrapper_environment_variables_for_flags(pkg, env):
             # implicit variables
             env.set(flag.upper(), " ".join(f for f in env_flags[flag]))
     pkg.flags_to_build_system_args(build_system_flags)
+
+    env.set("SPACK_COMPILER_SPEC", str(spec.compiler))
+
     env.set("SPACK_SYSTEM_DIRS", SYSTEM_DIR_CASE_ENTRY)
+
+    compiler.setup_custom_environment(pkg, env)
+
     return env
 
 
 def optimization_flags(compiler, target):
+    if spack.compilers.is_mixed_toolchain(compiler):
+        msg = (
+            "microarchitecture specific optimizations are not "
+            "supported yet on mixed compiler toolchains [check"
+            f" {compiler.name}@{compiler.version} for further details]"
+        )
+        tty.debug(msg)
+        return ""
+
     # Try to check if the current compiler comes with a version number or
     # has an unexpected suffix. If so, treat it as a compiler with a
     # custom spec.
-    version_number, _ = archspec.cpu.version_components(compiler.version.dotted_numeric_string)
+    compiler_version = compiler.version
+    version_number, suffix = archspec.cpu.version_components(compiler.version)
+    if not version_number or suffix:
+        try:
+            compiler_version = compiler.real_version
+        except spack.util.executable.ProcessError as e:
+            # log this and just return compiler.version instead
+            tty.debug(str(e))
+
     try:
-        result = target.optimization_flags(compiler.name, version_number)
+        result = target.optimization_flags(compiler.name, compiler_version.dotted_numeric_string)
     except (ValueError, archspec.cpu.UnsupportedMicroarchitecture):
         result = ""
 
     return result
 
 
+class FilterDefaultDynamicLinkerSearchPaths:
+    """Remove rpaths to directories that are default search paths of the dynamic linker."""
+
+    def __init__(self, dynamic_linker: Optional[str]) -> None:
+        # Identify directories by (inode, device) tuple, which handles symlinks too.
+        self.default_path_identifiers: Set[Tuple[int, int]] = set()
+        if not dynamic_linker:
+            return
+        for path in spack.util.libc.default_search_paths_from_dynamic_linker(dynamic_linker):
+            try:
+                s = os.stat(path)
+                if stat.S_ISDIR(s.st_mode):
+                    self.default_path_identifiers.add((s.st_ino, s.st_dev))
+            except OSError:
+                continue
+
+    def is_dynamic_loader_default_path(self, p: str) -> bool:
+        try:
+            s = os.stat(p)
+            return (s.st_ino, s.st_dev) in self.default_path_identifiers
+        except OSError:
+            return False
+
+    def __call__(self, dirs: List[str]) -> List[str]:
+        if not self.default_path_identifiers:
+            return dirs
+        return [p for p in dirs if not self.is_dynamic_loader_default_path(p)]
+
+
 def set_wrapper_variables(pkg, env):
     """Set environment variables used by the Spack compiler wrapper (which have the prefix
     `SPACK_`) and also add the compiler wrappers to PATH.
@@ -363,8 +566,39 @@ def set_wrapper_variables(pkg, env):
     this function computes these options in a manner that is intended to match the DAG traversal
     order in `SetupContext`. TODO: this is not the case yet, we're using post order, SetupContext
     is using topo order."""
-    # Set compiler flags injected from the spec
-    set_wrapper_environment_variables_for_flags(pkg, env)
+    # Set environment variables if specified for
+    # the given compiler
+    compiler = pkg.compiler
+    env.extend(spack.schema.environment.parse(compiler.environment))
+
+    if compiler.extra_rpaths:
+        extra_rpaths = ":".join(compiler.extra_rpaths)
+        env.set("SPACK_COMPILER_EXTRA_RPATHS", extra_rpaths)
+
+    # Add spack build environment path with compiler wrappers first in
+    # the path. We add the compiler wrapper path, which includes default
+    # wrappers (cc, c++, f77, f90), AND a subdirectory containing
+    # compiler-specific symlinks.  The latter ensures that builds that
+    # are sensitive to the *name* of the compiler see the right name when
+    # we're building with the wrappers.
+    #
+    # Conflicts on case-insensitive systems (like "CC" and "cc") are
+    # handled by putting one in the <build_env_path>/case-insensitive
+    # directory.  Add that to the path too.
+    env_paths = []
+    compiler_specific = os.path.join(
+        spack.paths.build_env_path, os.path.dirname(pkg.compiler.link_paths["cc"])
+    )
+    for item in [spack.paths.build_env_path, compiler_specific]:
+        env_paths.append(item)
+        ci = os.path.join(item, "case-insensitive")
+        if os.path.isdir(ci):
+            env_paths.append(ci)
+
+    tty.debug("Adding compiler bin/ paths: " + " ".join(env_paths))
+    for item in env_paths:
+        env.prepend_path("PATH", item)
+    env.set_path(SPACK_ENV_PATH, env_paths)
 
     # Working directory for the spack command itself, for debug logs.
     if spack.config.get("config:debug"):
@@ -430,15 +664,22 @@ def set_wrapper_variables(pkg, env):
         lib_path = os.path.join(pkg.prefix, libdir)
         rpath_dirs.insert(0, lib_path)
 
+    filter_default_dynamic_linker_search_paths = FilterDefaultDynamicLinkerSearchPaths(
+        pkg.compiler.default_dynamic_linker
+    )
+
     # TODO: filter_system_paths is again wrong (and probably unnecessary due to the is_system_path
     # branch above). link_dirs should be filtered with entries from _parse_link_paths.
     link_dirs = list(dedupe(filter_system_paths(link_dirs)))
     include_dirs = list(dedupe(filter_system_paths(include_dirs)))
     rpath_dirs = list(dedupe(filter_system_paths(rpath_dirs)))
+    rpath_dirs = filter_default_dynamic_linker_search_paths(rpath_dirs)
 
-    default_dynamic_linker_filter = spack.compilers.libraries.dynamic_linker_filter_for(pkg.spec)
-    if default_dynamic_linker_filter:
-        rpath_dirs = default_dynamic_linker_filter(rpath_dirs)
+    # TODO: implicit_rpaths is prefiltered by is_system_path, that should be removed in favor of
+    # just this filter.
+    implicit_rpaths = filter_default_dynamic_linker_search_paths(pkg.compiler.implicit_rpaths())
+    if implicit_rpaths:
+        env.set("SPACK_COMPILER_IMPLICIT_RPATHS", ":".join(implicit_rpaths))
 
     # Spack managed directories include the stage, store and upstream stores. We extend this with
     # their real paths to make it more robust (e.g. /tmp vs /private/tmp on macOS).
@@ -474,10 +715,9 @@ def set_package_py_globals(pkg, context: Context = Context.BUILD):
         module.std_meson_args = spack.build_systems.meson.MesonBuilder.std_args(pkg)
         module.std_pip_args = spack.build_systems.python.PythonPipBuilder.std_args(pkg)
 
-    # TODO: make these build deps that can be installed if not found.
-    module.make = MakeExecutable("make", jobs)
-    module.gmake = MakeExecutable("gmake", jobs)
-    module.ninja = MakeExecutable("ninja", jobs, supports_jobserver=False)
+    module.make = DeprecatedExecutable(pkg.name, "make", "gmake")
+    module.gmake = DeprecatedExecutable(pkg.name, "gmake", "gmake")
+    module.ninja = DeprecatedExecutable(pkg.name, "ninja", "ninja")
     # TODO: johnwparent: add package or builder support to define these build tools
     # for now there is no entrypoint for builders to define these on their
     # own
@@ -491,6 +731,26 @@ def set_package_py_globals(pkg, context: Context = Context.BUILD):
     # Don't use which for this; we want to find it in the current dir.
     module.configure = Executable("./configure")
 
+    # Put spack compiler paths in module scope. (Some packages use it
+    # in setup_run_environment etc, so don't put it context == build)
+    link_dir = spack.paths.build_env_path
+    pkg_compiler = None
+    try:
+        pkg_compiler = pkg.compiler
+    except spack.compilers.NoCompilerForSpecError as e:
+        tty.debug(f"cannot set 'spack_cc': {str(e)}")
+
+    if pkg_compiler is not None:
+        module.spack_cc = os.path.join(link_dir, pkg_compiler.link_paths["cc"])
+        module.spack_cxx = os.path.join(link_dir, pkg_compiler.link_paths["cxx"])
+        module.spack_f77 = os.path.join(link_dir, pkg_compiler.link_paths["f77"])
+        module.spack_fc = os.path.join(link_dir, pkg_compiler.link_paths["fc"])
+    else:
+        module.spack_cc = None
+        module.spack_cxx = None
+        module.spack_f77 = None
+        module.spack_fc = None
+
     # Useful directories within the prefix are encapsulated in
     # a Prefix object.
     module.prefix = pkg.prefix
@@ -656,6 +916,7 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD):
         context == Context.TEST and pkg.test_requires_compiler
     )
     if need_compiler:
+        set_compiler_environment_variables(pkg, env_mods)
         set_wrapper_variables(pkg, env_mods)
 
     # Platform specific setup goes before package specific setup. This is for setting
@@ -667,11 +928,6 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD):
     env_mods.extend(setup_context.get_env_modifications())
     tty.debug("setup_package: collected all modifications from dependencies")
 
-    tty.debug("setup_package: adding compiler wrappers paths")
-    for x in env_mods.group_by_name()["SPACK_ENV_PATH"]:
-        assert isinstance(x, PrependPath), "unexpected setting used for SPACK_ENV_PATH"
-        env_mods.prepend_path("PATH", x.value)
-
     if context == Context.TEST:
         env_mods.prepend_path("PATH", ".")
     elif context == Context.BUILD and not dirty and not env_mods.is_unset("CPATH"):
@@ -685,6 +941,11 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD):
 
     # Load modules on an already clean environment, just before applying Spack's
     # own environment modifications. This ensures Spack controls CC/CXX/... variables.
+    if need_compiler:
+        tty.debug("setup_package: loading compiler modules")
+        for mod in pkg.compiler.modules:
+            load_module(mod)
+
     load_external_modules(pkg)
 
     # Make sure nothing's strange about the Spack environment.

lib/spack/spack/build_systems/aspell_dict.py

@@ -6,7 +6,9 @@
 import llnl.util.filesystem as fs
 
 import spack.directives
+import spack.spec
 import spack.util.executable
+import spack.util.prefix
 
 from .autotools import AutotoolsBuilder, AutotoolsPackage
 
@@ -17,19 +19,18 @@ class AspellBuilder(AutotoolsBuilder):
     to the Aspell extensions.
     """
 
-    def configure(self, pkg, spec, prefix):
+    def configure(
+        self,
+        pkg: "AspellDictPackage",  # type: ignore[override]
+        spec: spack.spec.Spec,
+        prefix: spack.util.prefix.Prefix,
+    ):
         aspell = spec["aspell"].prefix.bin.aspell
         prezip = spec["aspell"].prefix.bin.prezip
         destdir = prefix
 
-        sh = spack.util.executable.which("sh")
-        sh(
-            "./configure",
-            "--vars",
-            "ASPELL={0}".format(aspell),
-            "PREZIP={0}".format(prezip),
-            "DESTDIR={0}".format(destdir),
-        )
+        sh = spack.util.executable.Executable("/bin/sh")
+        sh("./configure", "--vars", f"ASPELL={aspell}", f"PREZIP={prezip}", f"DESTDIR={destdir}")
 
 
 # Aspell dictionaries install their bits into their prefix.lib

lib/spack/spack/build_systems/autotools.py

@@ -12,7 +12,6 @@
 
 import spack.build_environment
 import spack.builder
-import spack.compilers.libraries
 import spack.error
 import spack.package_base
 import spack.phase_callbacks
@@ -192,6 +191,177 @@ def archive_files(self) -> List[str]:
             files.append(self._removed_la_files_log)
         return files
 
+    @property
+    def configure_directory(self) -> str:
+        """Return the directory where 'configure' resides."""
+        return self.pkg.stage.source_path
+
+    @property
+    def configure_abs_path(self) -> str:
+        # Absolute path to configure
+        configure_abs_path = os.path.join(os.path.abspath(self.configure_directory), "configure")
+        return configure_abs_path
+
+    @property
+    def build_directory(self) -> str:
+        """Override to provide another place to build the package"""
+        # Handle the case where the configure directory is set to a non-absolute path
+        # Non-absolute paths are always relative to the staging source path
+        build_dir = self.configure_directory
+        if not os.path.isabs(build_dir):
+            build_dir = os.path.join(self.pkg.stage.source_path, build_dir)
+        return build_dir
+
+    @property
+    def autoreconf_search_path_args(self) -> List[str]:
+        """Search path includes for autoreconf. Add an -I flag for all `aclocal` dirs
+        of build deps, skips the default path of automake, move external include
+        flags to the back, since they might pull in unrelated m4 files shadowing
+        spack dependencies."""
+        return _autoreconf_search_path_args(self.spec)
+
+    def autoreconf(
+        self, pkg: AutotoolsPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
+        """Not needed usually, configure should be already there"""
+
+        # If configure exists nothing needs to be done
+        if os.path.exists(self.configure_abs_path):
+            return
+
+        # Else try to regenerate it, which requires a few build dependencies
+        ensure_build_dependencies_or_raise(
+            spec=spec,
+            dependencies=["autoconf", "automake", "libtool"],
+            error_msg="Cannot generate configure",
+        )
+
+        tty.msg("Configure script not found: trying to generate it")
+        tty.warn("*********************************************************")
+        tty.warn("* If the default procedure fails, consider implementing *")
+        tty.warn("*        a custom AUTORECONF phase in the package       *")
+        tty.warn("*********************************************************")
+        with fs.working_dir(self.configure_directory):
+            # This line is what is needed most of the time
+            # --install, --verbose, --force
+            autoreconf_args = ["-ivf"]
+            autoreconf_args += self.autoreconf_search_path_args
+            autoreconf_args += self.autoreconf_extra_args
+            self.pkg.module.autoreconf(*autoreconf_args)
+
+    def configure(
+        self, pkg: AutotoolsPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
+        """Run "configure", with the arguments specified by the builder and an
+        appropriately set prefix.
+        """
+        options = getattr(self.pkg, "configure_flag_args", [])
+        options += ["--prefix={0}".format(prefix)]
+        options += self.configure_args()
+
+        with fs.working_dir(self.build_directory, create=True):
+            pkg.module.configure(*options)
+
+    def build(
+        self, pkg: AutotoolsPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
+        """Run "make" on the build targets specified by the builder."""
+        # See https://autotools.io/automake/silent.html
+        params = ["V=1"]
+        params += self.build_targets
+        with fs.working_dir(self.build_directory):
+            pkg.module.make(*params)
+
+    def install(
+        self, pkg: AutotoolsPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
+        """Run "make" on the install targets specified by the builder."""
+        with fs.working_dir(self.build_directory):
+            pkg.module.make(*self.install_targets)
+
+    def check(self) -> None:
+        """Run "make" on the ``test`` and ``check`` targets, if found."""
+        with fs.working_dir(self.build_directory):
+            self.pkg._if_make_target_execute("test")
+            self.pkg._if_make_target_execute("check")
+
+    def installcheck(self) -> None:
+        """Run "make" on the ``installcheck`` target, if found."""
+        with fs.working_dir(self.build_directory):
+            self.pkg._if_make_target_execute("installcheck")
+
+    def setup_build_environment(self, env):
+        if self.spec.platform == "darwin" and macos_version() >= Version("11"):
+            # Many configure files rely on matching '10.*' for macOS version
+            # detection and fail to add flags if it shows as version 11.
+            env.set("MACOSX_DEPLOYMENT_TARGET", "10.16")
+
+    def with_or_without(
+        self,
+        name: str,
+        activation_value: Optional[Union[Callable, str]] = None,
+        variant: Optional[str] = None,
+    ) -> List[str]:
+        """Inspects a variant and returns the arguments that activate
+        or deactivate the selected feature(s) for the configure options.
+
+        This function works on all type of variants. For bool-valued variants
+        it will return by default ``--with-{name}`` or ``--without-{name}``.
+        For other kinds of variants it will cycle over the allowed values and
+        return either ``--with-{value}`` or ``--without-{value}``.
+
+        If activation_value is given, then for each possible value of the
+        variant, the option ``--with-{value}=activation_value(value)`` or
+        ``--without-{value}`` will be added depending on whether or not
+        ``variant=value`` is in the spec.
+
+        Args:
+            name: name of a valid multi-valued variant
+            activation_value: callable that accepts a single value and returns the parameter to be
+                used leading to an entry of the type ``--with-{name}={parameter}``.
+
+                The special value "prefix" can also be assigned and will return
+                ``spec[name].prefix`` as activation parameter.
+
+        Returns:
+            list of arguments to configure
+        """
+        return self._activate_or_not(name, "with", "without", activation_value, variant)
+
+    def enable_or_disable(
+        self,
+        name: str,
+        activation_value: Optional[Union[Callable, str]] = None,
+        variant: Optional[str] = None,
+    ) -> List[str]:
+        """Same as
+        :meth:`~spack.build_systems.autotools.AutotoolsBuilder.with_or_without`
+        but substitute ``with`` with ``enable`` and ``without`` with ``disable``.
+
+        Args:
+            name: name of a valid multi-valued variant
+            activation_value: if present accepts a single value and returns the parameter to be
+                used leading to an entry of the type ``--enable-{name}={parameter}``
+
+                The special value "prefix" can also be assigned and will return
+                ``spec[name].prefix`` as activation parameter.
+
+        Returns:
+            list of arguments to configure
+        """
+        return self._activate_or_not(name, "enable", "disable", activation_value, variant)
+
+    def configure_args(self) -> List[str]:
+        """Return the list of all the arguments that must be passed to configure,
+        except ``--prefix`` which will be pre-pended to the list.
+        """
+        return []
+
+    @spack.phase_callbacks.run_before("autoreconf")
+    def _delete_configure_to_force_update(self) -> None:
+        if self.force_autoreconf:
+            fs.force_remove(self.configure_abs_path)
+
     @spack.phase_callbacks.run_after("autoreconf")
     def _do_patch_config_files(self) -> None:
         """Some packages ship with older config.guess/config.sub files and need to
@@ -304,6 +474,24 @@ def runs_ok(script_abs_path):
             fs.copy(substitutes[name], abs_path)
             os.chmod(abs_path, mode)
 
+    @spack.phase_callbacks.run_after("autoreconf")
+    def _set_configure_or_die(self) -> None:
+        """Ensure the presence of a "configure" script, or raise. If the "configure"
+        is found, a module level attribute is set.
+
+        Raises:
+             RuntimeError: if the "configure" script is not found
+        """
+        # Check if the "configure" script is there. If not raise a RuntimeError.
+        if not os.path.exists(self.configure_abs_path):
+            msg = "configure script not found in {0}"
+            raise RuntimeError(msg.format(self.configure_directory))
+
+        # Monkey-patch the configure script in the corresponding module
+        globals_for_pkg = spack.build_environment.ModuleChangePropagator(self.pkg)
+        globals_for_pkg.configure = Executable(self.configure_abs_path)
+        globals_for_pkg.propagate_changes_to_mro()
+
     @spack.phase_callbacks.run_before("configure")
     def _patch_usr_bin_file(self) -> None:
         """On NixOS file is not available in /usr/bin/file. Patch configure
@@ -357,6 +545,13 @@ def _do_patch_libtool_configure(self) -> None:
             )
             # Support Libtool 2.4.2 and older:
             x.filter(regex=r'^(\s*test \$p = "-R")(; then\s*)$', repl=r'\1 || test x-l = x"$p"\2')
+            # Configure scripts generated with libtool < 2.5.4 have a faulty test for the
+            # -single_module linker flag. A deprecation warning makes it think the default is
+            # -multi_module, triggering it to use problematic linker flags (such as ld -r). The
+            # linker default is `-single_module` from (ancient) macOS 10.4, so override by setting
+            # `lt_cv_apple_cc_single_mod=yes`. See the fix in libtool commit
+            # 82f7f52123e4e7e50721049f7fa6f9b870e09c9d.
+            x.filter("lt_cv_apple_cc_single_mod=no", "lt_cv_apple_cc_single_mod=yes", string=True)
 
     @spack.phase_callbacks.run_after("configure")
     def _do_patch_libtool(self) -> None:
@@ -393,44 +588,33 @@ def _do_patch_libtool(self) -> None:
             markers[tag] = "LIBTOOL TAG CONFIG: {0}".format(tag.upper())
 
         # Replace empty linker flag prefixes:
-        if self.spec.satisfies("%nag"):
+        if self.pkg.compiler.name == "nag":
             # Nag is mixed with gcc and g++, which are recognized correctly.
             # Therefore, we change only Fortran values:
-            nag_pkg = self.spec["fortran"].package
             for tag in ["fc", "f77"]:
                 marker = markers[tag]
                 x.filter(
                     regex='^wl=""$',
-                    repl=f'wl="{nag_pkg.linker_arg}"',
-                    start_at=f"# ### BEGIN {marker}",
-                    stop_at=f"# ### END {marker}",
+                    repl='wl="{0}"'.format(self.pkg.compiler.linker_arg),
+                    start_at="# ### BEGIN {0}".format(marker),
+                    stop_at="# ### END {0}".format(marker),
                 )
         else:
-            compiler_spec = spack.compilers.libraries.compiler_spec(self.spec)
-            if compiler_spec:
-                x.filter(regex='^wl=""$', repl='wl="{0}"'.format(compiler_spec.package.linker_arg))
+            x.filter(regex='^wl=""$', repl='wl="{0}"'.format(self.pkg.compiler.linker_arg))
 
         # Replace empty PIC flag values:
-        for compiler, marker in markers.items():
-            if compiler == "cc":
-                language = "c"
-            elif compiler == "cxx":
-                language = "cxx"
-            else:
-                language = "fortran"
-
-            if language not in self.spec:
-                continue
-
+        for cc, marker in markers.items():
             x.filter(
                 regex='^pic_flag=""$',
-                repl=f'pic_flag="{self.spec[language].package.pic_flag}"',
-                start_at=f"# ### BEGIN {marker}",
-                stop_at=f"# ### END {marker}",
+                repl='pic_flag="{0}"'.format(
+                    getattr(self.pkg.compiler, "{0}_pic_flag".format(cc))
+                ),
+                start_at="# ### BEGIN {0}".format(marker),
+                stop_at="# ### END {0}".format(marker),
             )
 
         # Other compiler-specific patches:
-        if self.spec.satisfies("%fj"):
+        if self.pkg.compiler.name == "fj":
             x.filter(regex="-nostdlib", repl="", string=True)
             rehead = r"/\S*/"
             for o in [
@@ -443,7 +627,7 @@ def _do_patch_libtool(self) -> None:
                 r"crtendS\.o",
             ]:
                 x.filter(regex=(rehead + o), repl="")
-        elif self.spec.satisfies("%nag"):
+        elif self.pkg.compiler.name == "nag":
             for tag in ["fc", "f77"]:
                 marker = markers[tag]
                 start_at = "# ### BEGIN {0}".format(marker)
@@ -517,142 +701,27 @@ def _do_patch_libtool(self) -> None:
                 stop_at=stop_at,
             )
 
-    @property
-    def configure_directory(self) -> str:
-        """Return the directory where 'configure' resides."""
-        return self.pkg.stage.source_path
+    spack.phase_callbacks.run_after("build")(execute_build_time_tests)
+    spack.phase_callbacks.run_after("install")(execute_install_time_tests)
 
-    @property
-    def configure_abs_path(self) -> str:
-        # Absolute path to configure
-        configure_abs_path = os.path.join(os.path.abspath(self.configure_directory), "configure")
-        return configure_abs_path
-
-    @property
-    def build_directory(self) -> str:
-        """Override to provide another place to build the package"""
-        # Handle the case where the configure directory is set to a non-absolute path
-        # Non-absolute paths are always relative to the staging source path
-        build_dir = self.configure_directory
-        if not os.path.isabs(build_dir):
-            build_dir = os.path.join(self.pkg.stage.source_path, build_dir)
-        return build_dir
-
-    @spack.phase_callbacks.run_before("autoreconf")
-    def delete_configure_to_force_update(self) -> None:
-        if self.force_autoreconf:
-            fs.force_remove(self.configure_abs_path)
-
-    @property
-    def autoreconf_search_path_args(self) -> List[str]:
-        """Search path includes for autoreconf. Add an -I flag for all `aclocal` dirs
-        of build deps, skips the default path of automake, move external include
-        flags to the back, since they might pull in unrelated m4 files shadowing
-        spack dependencies."""
-        return _autoreconf_search_path_args(self.spec)
-
-    @spack.phase_callbacks.run_after("autoreconf")
-    def set_configure_or_die(self) -> None:
-        """Ensure the presence of a "configure" script, or raise. If the "configure"
-        is found, a module level attribute is set.
-
-        Raises:
-             RuntimeError: if the "configure" script is not found
+    @spack.phase_callbacks.run_after("install")
+    def _remove_libtool_archives(self) -> None:
+        """Remove all .la files in prefix sub-folders if the package sets
+        ``install_libtool_archives`` to be False.
         """
-        # Check if the "configure" script is there. If not raise a RuntimeError.
-        if not os.path.exists(self.configure_abs_path):
-            msg = "configure script not found in {0}"
-            raise RuntimeError(msg.format(self.configure_directory))
-
-        # Monkey-patch the configure script in the corresponding module
-        globals_for_pkg = spack.build_environment.ModuleChangePropagator(self.pkg)
-        globals_for_pkg.configure = Executable(self.configure_abs_path)
-        globals_for_pkg.propagate_changes_to_mro()
-
-    def configure_args(self) -> List[str]:
-        """Return the list of all the arguments that must be passed to configure,
-        except ``--prefix`` which will be pre-pended to the list.
-        """
-        return []
-
-    def autoreconf(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
-    ) -> None:
-        """Not needed usually, configure should be already there"""
-
-        # If configure exists nothing needs to be done
-        if os.path.exists(self.configure_abs_path):
+        # If .la files are to be installed there's nothing to do
+        if self.install_libtool_archives:
             return
 
-        # Else try to regenerate it, which requires a few build dependencies
-        ensure_build_dependencies_or_raise(
-            spec=spec,
-            dependencies=["autoconf", "automake", "libtool"],
-            error_msg="Cannot generate configure",
-        )
+        # Remove the files and create a log of what was removed
+        libtool_files = fs.find(str(self.pkg.prefix), "*.la", recursive=True)
+        with fs.safe_remove(*libtool_files):
+            fs.mkdirp(os.path.dirname(self._removed_la_files_log))
+            with open(self._removed_la_files_log, mode="w", encoding="utf-8") as f:
+                f.write("\n".join(libtool_files))
 
-        tty.msg("Configure script not found: trying to generate it")
-        tty.warn("*********************************************************")
-        tty.warn("* If the default procedure fails, consider implementing *")
-        tty.warn("*        a custom AUTORECONF phase in the package       *")
-        tty.warn("*********************************************************")
-        with fs.working_dir(self.configure_directory):
-            # This line is what is needed most of the time
-            # --install, --verbose, --force
-            autoreconf_args = ["-ivf"]
-            autoreconf_args += self.autoreconf_search_path_args
-            autoreconf_args += self.autoreconf_extra_args
-            self.pkg.module.autoreconf(*autoreconf_args)
-
-    def configure(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
-    ) -> None:
-        """Run "configure", with the arguments specified by the builder and an
-        appropriately set prefix.
-        """
-        options = getattr(self.pkg, "configure_flag_args", [])
-        options += ["--prefix={0}".format(prefix)]
-        options += self.configure_args()
-
-        with fs.working_dir(self.build_directory, create=True):
-            pkg.module.configure(*options)
-
-    def build(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
-    ) -> None:
-        """Run "make" on the build targets specified by the builder."""
-        # See https://autotools.io/automake/silent.html
-        params = ["V=1"]
-        params += self.build_targets
-        with fs.working_dir(self.build_directory):
-            pkg.module.make(*params)
-
-    def install(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
-    ) -> None:
-        """Run "make" on the install targets specified by the builder."""
-        with fs.working_dir(self.build_directory):
-            pkg.module.make(*self.install_targets)
-
-    spack.phase_callbacks.run_after("build")(execute_build_time_tests)
-
-    def check(self) -> None:
-        """Run "make" on the ``test`` and ``check`` targets, if found."""
-        with fs.working_dir(self.build_directory):
-            self.pkg._if_make_target_execute("test")
-            self.pkg._if_make_target_execute("check")
+    # On macOS, force rpaths for shared library IDs and remove duplicate rpaths
+    spack.phase_callbacks.run_after("install", when="platform=darwin")(apply_macos_rpath_fixups)
 
     def _activate_or_not(
         self,
@@ -774,93 +843,6 @@ def _default_generator(is_activated):
             args.append(line_generator(activated))
         return args
 
-    def with_or_without(
-        self,
-        name: str,
-        activation_value: Optional[Union[Callable, str]] = None,
-        variant: Optional[str] = None,
-    ) -> List[str]:
-        """Inspects a variant and returns the arguments that activate
-        or deactivate the selected feature(s) for the configure options.
-
-        This function works on all type of variants. For bool-valued variants
-        it will return by default ``--with-{name}`` or ``--without-{name}``.
-        For other kinds of variants it will cycle over the allowed values and
-        return either ``--with-{value}`` or ``--without-{value}``.
-
-        If activation_value is given, then for each possible value of the
-        variant, the option ``--with-{value}=activation_value(value)`` or
-        ``--without-{value}`` will be added depending on whether or not
-        ``variant=value`` is in the spec.
-
-        Args:
-            name: name of a valid multi-valued variant
-            activation_value: callable that accepts a single value and returns the parameter to be
-                used leading to an entry of the type ``--with-{name}={parameter}``.
-
-                The special value "prefix" can also be assigned and will return
-                ``spec[name].prefix`` as activation parameter.
-
-        Returns:
-            list of arguments to configure
-        """
-        return self._activate_or_not(name, "with", "without", activation_value, variant)
-
-    def enable_or_disable(
-        self,
-        name: str,
-        activation_value: Optional[Union[Callable, str]] = None,
-        variant: Optional[str] = None,
-    ) -> List[str]:
-        """Same as
-        :meth:`~spack.build_systems.autotools.AutotoolsBuilder.with_or_without`
-        but substitute ``with`` with ``enable`` and ``without`` with ``disable``.
-
-        Args:
-            name: name of a valid multi-valued variant
-            activation_value: if present accepts a single value and returns the parameter to be
-                used leading to an entry of the type ``--enable-{name}={parameter}``
-
-                The special value "prefix" can also be assigned and will return
-                ``spec[name].prefix`` as activation parameter.
-
-        Returns:
-            list of arguments to configure
-        """
-        return self._activate_or_not(name, "enable", "disable", activation_value, variant)
-
-    spack.phase_callbacks.run_after("install")(execute_install_time_tests)
-
-    def installcheck(self) -> None:
-        """Run "make" on the ``installcheck`` target, if found."""
-        with fs.working_dir(self.build_directory):
-            self.pkg._if_make_target_execute("installcheck")
-
-    @spack.phase_callbacks.run_after("install")
-    def remove_libtool_archives(self) -> None:
-        """Remove all .la files in prefix sub-folders if the package sets
-        ``install_libtool_archives`` to be False.
-        """
-        # If .la files are to be installed there's nothing to do
-        if self.install_libtool_archives:
-            return
-
-        # Remove the files and create a log of what was removed
-        libtool_files = fs.find(str(self.pkg.prefix), "*.la", recursive=True)
-        with fs.safe_remove(*libtool_files):
-            fs.mkdirp(os.path.dirname(self._removed_la_files_log))
-            with open(self._removed_la_files_log, mode="w", encoding="utf-8") as f:
-                f.write("\n".join(libtool_files))
-
-    def setup_build_environment(self, env):
-        if self.spec.platform == "darwin" and macos_version() >= Version("11"):
-            # Many configure files rely on matching '10.*' for macOS version
-            # detection and fail to add flags if it shows as version 11.
-            env.set("MACOSX_DEPLOYMENT_TARGET", "10.16")
-
-    # On macOS, force rpaths for shared library IDs and remove duplicate rpaths
-    spack.phase_callbacks.run_after("install", when="platform=darwin")(apply_macos_rpath_fixups)
-
 
 def _autoreconf_search_path_args(spec: spack.spec.Spec) -> List[str]:
     dirs_seen: Set[Tuple[int, int]] = set()

lib/spack/spack/build_systems/cached_cmake.py

@@ -10,7 +10,8 @@
 import llnl.util.tty as tty
 
 import spack.phase_callbacks
-from spack.directives import depends_on
+import spack.spec
+import spack.util.prefix
 
 from .cmake import CMakeBuilder, CMakePackage
 
@@ -68,7 +69,12 @@ class CachedCMakeBuilder(CMakeBuilder):
 
     @property
     def cache_name(self):
-        return f"{self.pkg.name}-{self.spec.architecture.platform}-{self.spec.dag_hash()}.cmake"
+        return "{0}-{1}-{2}@{3}.cmake".format(
+            self.pkg.name,
+            self.pkg.spec.architecture,
+            self.pkg.spec.compiler.name,
+            self.pkg.spec.compiler.version,
+        )
 
     @property
     def cache_path(self):
@@ -111,9 +117,7 @@ def initconfig_compiler_entries(self):
         # Fortran compiler is optional
         if "FC" in os.environ:
             spack_fc_entry = cmake_cache_path("CMAKE_Fortran_COMPILER", os.environ["FC"])
-            system_fc_entry = cmake_cache_path(
-                "CMAKE_Fortran_COMPILER", self.spec["fortran"].package.fortran
-            )
+            system_fc_entry = cmake_cache_path("CMAKE_Fortran_COMPILER", self.pkg.compiler.fc)
         else:
             spack_fc_entry = "# No Fortran compiler defined in spec"
             system_fc_entry = "# No Fortran compiler defined in spec"
@@ -129,8 +133,8 @@ def initconfig_compiler_entries(self):
             "  " + cmake_cache_path("CMAKE_CXX_COMPILER", os.environ["CXX"]),
             "  " + spack_fc_entry,
             "else()\n",
-            "  " + cmake_cache_path("CMAKE_C_COMPILER", self.spec["c"].package.cc),
-            "  " + cmake_cache_path("CMAKE_CXX_COMPILER", self.spec["cxx"].package.cxx),
+            "  " + cmake_cache_path("CMAKE_C_COMPILER", self.pkg.compiler.cc),
+            "  " + cmake_cache_path("CMAKE_CXX_COMPILER", self.pkg.compiler.cxx),
             "  " + system_fc_entry,
             "endif()\n",
         ]
@@ -291,6 +295,13 @@ def initconfig_hardware_entries(self):
                 entries.append(cmake_cache_string("AMDGPU_TARGETS", arch_str))
                 entries.append(cmake_cache_string("GPU_TARGETS", arch_str))
 
+            if spec.satisfies("%gcc"):
+                entries.append(
+                    cmake_cache_string(
+                        "CMAKE_HIP_FLAGS", f"--gcc-toolchain={self.pkg.compiler.prefix}"
+                    )
+                )
+
         return entries
 
     def std_initconfig_entries(self):
@@ -321,7 +332,9 @@ def initconfig_package_entries(self):
         """This method is to be overwritten by the package"""
         return []
 
-    def initconfig(self, pkg, spec, prefix):
+    def initconfig(
+        self, pkg: "CachedCMakePackage", spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
         cache_entries = (
             self.std_initconfig_entries()
             + self.initconfig_compiler_entries()
@@ -358,10 +371,6 @@ class CachedCMakePackage(CMakePackage):
 
     CMakeBuilder = CachedCMakeBuilder
 
-    # These dependencies are assumed in the builder
-    depends_on("c", type="build")
-    depends_on("cxx", type="build")
-
     def flag_handler(self, name, flags):
         if name in ("cflags", "cxxflags", "cppflags", "fflags"):
             return None, None, None  # handled in the cmake cache

lib/spack/spack/build_systems/cargo.py

@@ -7,6 +7,8 @@
 import spack.builder
 import spack.package_base
 import spack.phase_callbacks
+import spack.spec
+import spack.util.prefix
 from spack.directives import build_system, depends_on
 from spack.multimethod import when
 
@@ -81,12 +83,16 @@ def check_args(self):
     def setup_build_environment(self, env):
         env.set("CARGO_HOME", self.stage.path)
 
-    def build(self, pkg, spec, prefix):
+    def build(
+        self, pkg: CargoPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
         """Runs ``cargo install`` in the source directory"""
         with fs.working_dir(self.build_directory):
             pkg.module.cargo("install", "--root", "out", "--path", ".", *self.build_args)
 
-    def install(self, pkg, spec, prefix):
+    def install(
+        self, pkg: CargoPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
         """Copy build files into package prefix."""
         with fs.working_dir(self.build_directory):
             fs.install_tree("out", prefix)

lib/spack/spack/build_systems/cmake.py

@@ -454,10 +454,7 @@ def cmake_args(self) -> List[str]:
         return []
 
     def cmake(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
+        self, pkg: CMakePackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
     ) -> None:
         """Runs ``cmake`` in the build directory"""
 
@@ -474,10 +471,7 @@ def cmake(
             pkg.module.cmake(*options)
 
     def build(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
+        self, pkg: CMakePackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
     ) -> None:
         """Make the build targets"""
         with fs.working_dir(self.build_directory):
@@ -488,10 +482,7 @@ def build(
                 pkg.module.ninja(*self.build_targets)
 
     def install(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
+        self, pkg: CMakePackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
     ) -> None:
         """Make the install targets"""
         with fs.working_dir(self.build_directory):

lib/spack/spack/build_systems/compiler.py

@@ -6,13 +6,12 @@
 import pathlib
 import re
 import sys
-from typing import Dict, List, Optional, Sequence, Tuple, Union
+from typing import Dict, List, Sequence, Tuple, Union
 
 import llnl.util.tty as tty
-from llnl.util.lang import classproperty, memoized
+from llnl.util.lang import classproperty
 
-import spack
-import spack.compilers.error
+import spack.compiler
 import spack.package_base
 import spack.util.executable
 
@@ -44,9 +43,6 @@ class CompilerPackage(spack.package_base.PackageBase):
     #: Static definition of languages supported by this class
     compiler_languages: Sequence[str] = ["c", "cxx", "fortran"]
 
-    #: Relative path to compiler wrappers
-    link_paths: Dict[str, str] = {}
-
     def __init__(self, spec: "spack.spec.Spec"):
         super().__init__(spec)
         msg = f"Supported languages for {spec} are not a subset of possible supported languages"
@@ -81,14 +77,14 @@ def executables(cls) -> Sequence[str]:
         ]
 
     @classmethod
-    def determine_version(cls, exe: Path) -> str:
+    def determine_version(cls, exe: Path):
         version_argument = cls.compiler_version_argument
         if isinstance(version_argument, str):
             version_argument = (version_argument,)
 
         for va in version_argument:
             try:
-                output = compiler_output(exe, version_argument=va)
+                output = spack.compiler.get_compiler_version_output(exe, va)
                 match = re.search(cls.compiler_version_regex, output)
                 if match:
                     return ".".join(match.groups())
@@ -99,11 +95,10 @@ def determine_version(cls, exe: Path) -> str:
                     f"[{__file__}] Cannot detect a valid version for the executable "
                     f"{str(exe)}, for package '{cls.name}': {e}"
                 )
-        return ""
 
     @classmethod
     def compiler_bindir(cls, prefix: Path) -> Path:
-        """Overridable method for the location of the compiler bindir within the prefix"""
+        """Overridable method for the location of the compiler bindir within the preifx"""
         return os.path.join(prefix, "bin")
 
     @classmethod
@@ -147,109 +142,3 @@ def determine_compiler_paths(cls, exes: Sequence[Path]) -> Dict[str, Path]:
     def determine_variants(cls, exes: Sequence[Path], version_str: str) -> Tuple:
         # path determination is separated so it can be reused in subclasses
         return "", {"compilers": cls.determine_compiler_paths(exes=exes)}
-
-    #: Returns the argument needed to set the RPATH, or None if it does not exist
-    rpath_arg: Optional[str] = "-Wl,-rpath,"
-    #: Flag that needs to be used to pass an argument to the linker
-    linker_arg: str = "-Wl,"
-    #: Flag used to produce Position Independent Code
-    pic_flag: str = "-fPIC"
-    #: Flag used to get verbose output
-    verbose_flags: str = "-v"
-    #: Flag to activate OpenMP support
-    openmp_flag: str = "-fopenmp"
-
-    required_libs: List[str] = []
-
-    def standard_flag(self, *, language: str, standard: str) -> str:
-        """Returns the flag used to enforce a given standard for a language"""
-        if language not in self.supported_languages:
-            raise spack.compilers.error.UnsupportedCompilerFlag(
-                f"{self.spec} does not provide the '{language}' language"
-            )
-        try:
-            return self._standard_flag(language=language, standard=standard)
-        except (KeyError, RuntimeError) as e:
-            raise spack.compilers.error.UnsupportedCompilerFlag(
-                f"{self.spec} does not provide the '{language}' standard {standard}"
-            ) from e
-
-    def _standard_flag(self, *, language: str, standard: str) -> str:
-        raise NotImplementedError("Must be implemented by derived classes")
-
-    def archspec_name(self) -> str:
-        """Name that archspec uses to refer to this compiler"""
-        return self.spec.name
-
-    @property
-    def cc(self) -> Optional[str]:
-        assert self.spec.concrete, "cannot retrieve C compiler, spec is not concrete"
-        if self.spec.external:
-            return self.spec.extra_attributes["compilers"].get("c", None)
-        return self._cc_path()
-
-    def _cc_path(self) -> Optional[str]:
-        """Returns the path to the C compiler, if the package was installed by Spack"""
-        return None
-
-    @property
-    def cxx(self) -> Optional[str]:
-        assert self.spec.concrete, "cannot retrieve C++ compiler, spec is not concrete"
-        if self.spec.external:
-            return self.spec.extra_attributes["compilers"].get("cxx", None)
-        return self._cxx_path()
-
-    def _cxx_path(self) -> Optional[str]:
-        """Returns the path to the C++ compiler, if the package was installed by Spack"""
-        return None
-
-    @property
-    def fortran(self):
-        assert self.spec.concrete, "cannot retrieve Fortran compiler, spec is not concrete"
-        if self.spec.external:
-            return self.spec.extra_attributes["compilers"].get("fortran", None)
-        return self._fortran_path()
-
-    def _fortran_path(self) -> Optional[str]:
-        """Returns the path to the Fortran compiler, if the package was installed by Spack"""
-        return None
-
-
-@memoized
-def _compiler_output(
-    compiler_path: Path, *, version_argument: str, ignore_errors: Tuple[int, ...] = ()
-) -> str:
-    """Returns the output from the compiler invoked with the given version argument.
-
-    Args:
-        compiler_path: path of the compiler to be invoked
-        version_argument: the argument used to extract version information
-    """
-    compiler = spack.util.executable.Executable(compiler_path)
-    if not version_argument:
-        return compiler(
-            output=str, error=str, ignore_errors=ignore_errors, timeout=120, fail_on_error=True
-        )
-    return compiler(
-        version_argument,
-        output=str,
-        error=str,
-        ignore_errors=ignore_errors,
-        timeout=120,
-        fail_on_error=True,
-    )
-
-
-def compiler_output(
-    compiler_path: Path, *, version_argument: str, ignore_errors: Tuple[int, ...] = ()
-) -> str:
-    """Wrapper for _get_compiler_version_output()."""
-    # This ensures that we memoize compiler output by *absolute path*,
-    # not just executable name. If we don't do this, and the path changes
-    # (e.g., during testing), we can get incorrect results.
-    if not os.path.isabs(compiler_path):
-        compiler_path = spack.util.executable.which_string(compiler_path, required=True)
-
-    return _compiler_output(
-        compiler_path, version_argument=version_argument, ignore_errors=ignore_errors
-    )

lib/spack/spack/build_systems/generic.py

@@ -7,6 +7,8 @@
 import spack.directives
 import spack.package_base
 import spack.phase_callbacks
+import spack.spec
+import spack.util.prefix
 
 from ._checks import BuilderWithDefaults, apply_macos_rpath_fixups, execute_install_time_tests
 
@@ -48,3 +50,8 @@ class GenericBuilder(BuilderWithDefaults):
 
     # unconditionally perform any post-install phase tests
     spack.phase_callbacks.run_after("install")(execute_install_time_tests)
+
+    def install(
+        self, pkg: Package, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
+        raise NotImplementedError

lib/spack/spack/build_systems/go.py

@@ -7,6 +7,8 @@
 import spack.builder
 import spack.package_base
 import spack.phase_callbacks
+import spack.spec
+import spack.util.prefix
 from spack.directives import build_system, extends
 from spack.multimethod import when
 
@@ -88,12 +90,16 @@ def check_args(self):
         """Argument for ``go test`` during check phase"""
         return []
 
-    def build(self, pkg, spec, prefix):
+    def build(
+        self, pkg: GoPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
         """Runs ``go build`` in the source directory"""
         with fs.working_dir(self.build_directory):
             pkg.module.go("build", *self.build_args)
 
-    def install(self, pkg, spec, prefix):
+    def install(
+        self, pkg: GoPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
         """Install built binaries into prefix bin."""
         with fs.working_dir(self.build_directory):
             fs.mkdirp(prefix.bin)

lib/spack/spack/build_systems/lua.py

@@ -7,7 +7,9 @@
 
 import spack.builder
 import spack.package_base
+import spack.spec
 import spack.util.executable
+import spack.util.prefix
 from spack.directives import build_system, depends_on, extends
 from spack.multimethod import when
 
@@ -55,7 +57,9 @@ class LuaBuilder(spack.builder.Builder):
     #: Names associated with package attributes in the old build-system format
     legacy_attributes = ()
 
-    def unpack(self, pkg, spec, prefix):
+    def unpack(
+        self, pkg: LuaPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
         if os.path.splitext(pkg.stage.archive_file)[1] == ".rock":
             directory = pkg.luarocks("unpack", pkg.stage.archive_file, output=str)
             dirlines = directory.split("\n")
@@ -66,15 +70,16 @@ def unpack(self, pkg, spec, prefix):
     def _generate_tree_line(name, prefix):
         return """{{ name = "{name}", root = "{prefix}" }};""".format(name=name, prefix=prefix)
 
-    def generate_luarocks_config(self, pkg, spec, prefix):
+    def generate_luarocks_config(
+        self, pkg: LuaPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
         spec = self.pkg.spec
         table_entries = []
         for d in spec.traverse(deptype=("build", "run")):
             if d.package.extends(self.pkg.extendee_spec):
                 table_entries.append(self._generate_tree_line(d.name, d.prefix))
 
-        path = self._luarocks_config_path()
-        with open(path, "w", encoding="utf-8") as config:
+        with open(self._luarocks_config_path(), "w", encoding="utf-8") as config:
             config.write(
                 """
                 deps_mode="all"
@@ -85,23 +90,26 @@ def generate_luarocks_config(self, pkg, spec, prefix):
                     "\n".join(table_entries)
                 )
             )
-        return path
 
-    def preprocess(self, pkg, spec, prefix):
+    def preprocess(
+        self, pkg: LuaPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
         """Override this to preprocess source before building with luarocks"""
         pass
 
     def luarocks_args(self):
         return []
 
-    def install(self, pkg, spec, prefix):
+    def install(
+        self, pkg: LuaPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
         rock = "."
         specs = find(".", "*.rockspec", recursive=False)
         if specs:
             rock = specs[0]
         rocks_args = self.luarocks_args()
         rocks_args.append(rock)
-        self.pkg.luarocks("--tree=" + prefix, "make", *rocks_args)
+        pkg.luarocks("--tree=" + prefix, "make", *rocks_args)
 
     def _luarocks_config_path(self):
         return os.path.join(self.pkg.stage.source_path, "spack_luarocks.lua")

lib/spack/spack/build_systems/makefile.py

@@ -98,29 +98,20 @@ def build_directory(self) -> str:
         return self.pkg.stage.source_path
 
     def edit(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
+        self, pkg: MakefilePackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
     ) -> None:
         """Edit the Makefile before calling make. The default is a no-op."""
         pass
 
     def build(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
+        self, pkg: MakefilePackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
     ) -> None:
         """Run "make" on the build targets specified by the builder."""
         with fs.working_dir(self.build_directory):
             pkg.module.make(*self.build_targets)
 
     def install(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
+        self, pkg: MakefilePackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
     ) -> None:
         """Run "make" on the install targets specified by the builder."""
         with fs.working_dir(self.build_directory):

lib/spack/spack/build_systems/maven.py

@@ -5,6 +5,8 @@
 
 import spack.builder
 import spack.package_base
+import spack.spec
+import spack.util.prefix
 from spack.directives import build_system, depends_on
 from spack.multimethod import when
 from spack.util.executable import which
@@ -58,16 +60,20 @@ def build_args(self):
         """List of args to pass to build phase."""
         return []
 
-    def build(self, pkg, spec, prefix):
+    def build(
+        self, pkg: MavenPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
         """Compile code and package into a JAR file."""
         with fs.working_dir(self.build_directory):
-            mvn = which("mvn")
+            mvn = which("mvn", required=True)
             if self.pkg.run_tests:
                 mvn("verify", *self.build_args())
             else:
                 mvn("package", "-DskipTests", *self.build_args())
 
-    def install(self, pkg, spec, prefix):
+    def install(
+        self, pkg: MavenPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
         """Copy to installation prefix."""
         with fs.working_dir(self.build_directory):
             fs.install_tree(".", prefix)

lib/spack/spack/build_systems/meson.py

@@ -188,10 +188,7 @@ def meson_args(self) -> List[str]:
         return []
 
     def meson(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
+        self, pkg: MesonPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
     ) -> None:
         """Run ``meson`` in the build directory"""
         options = []
@@ -204,10 +201,7 @@ def meson(
             pkg.module.meson(*options)
 
     def build(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
+        self, pkg: MesonPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
     ) -> None:
         """Make the build targets"""
         options = ["-v"]
@@ -216,10 +210,7 @@ def build(
             pkg.module.ninja(*options)
 
     def install(
-        self,
-        pkg: spack.package_base.PackageBase,
-        spec: spack.spec.Spec,
-        prefix: spack.util.prefix.Prefix,
+        self, pkg: MesonPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
     ) -> None:
         """Make the install targets"""
         with fs.working_dir(self.build_directory):