netcdf-c: add comment to PR with CMake option rename

Starting from amrex-25.03, FFT is enabled by default in spack build.

.github/workflows/requirements/style/requirements.txt

@@ -1,7 +1,7 @@
 black==25.1.0
 clingo==5.7.1
 flake8==7.1.2
-isort==6.0.0
-mypy==1.11.2
+isort==6.0.1
+mypy==1.15.0
 types-six==1.17.0.20241205
 vermin==1.6.0

.gitignore

@@ -201,7 +201,6 @@ tramp
 
 # Org-mode
 .org-id-locations
-*_archive
 
 # flymake-mode
 *_flymake.*

etc/spack/defaults/concretizer.yaml

@@ -54,9 +54,15 @@ concretizer:
       # Regular packages
       cmake: 2
       gmake: 2
+      python: 2
+      python-venv: 2
       py-cython: 2
       py-flit-core: 2
+      py-pip: 2
       py-setuptools: 2
+      py-wheel: 2
+      xcb-proto: 2
+      # Compilers
       gcc: 2
       llvm: 2
   # Option to specify compatibility between operating systems for reuse of compilers and packages

lib/spack/docs/basic_usage.rst

@@ -1761,19 +1761,24 @@ Verifying installations
 The ``spack verify`` command can be used to verify the validity of
 Spack-installed packages any time after installation.
 
+
+^^^^^^^^^^^^^^^^^^^^^^^^^
+``spack verify manifest``
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
 At installation time, Spack creates a manifest of every file in the
 installation prefix. For links, Spack tracks the mode, ownership, and
 destination. For directories, Spack tracks the mode, and
 ownership. For files, Spack tracks the mode, ownership, modification
-time, hash, and size. The Spack verify command will check, for every
-file in each package, whether any of those attributes have changed. It
-will also check for newly added files or deleted files from the
-installation prefix. Spack can either check all installed packages
+time, hash, and size. The ``spack verify manifest`` command will check,
+for every file in each package, whether any of those attributes have
+changed. It will also check for newly added files or deleted files from
+the installation prefix. Spack can either check all installed packages
 using the `-a,--all` or accept specs listed on the command line to
 verify.
 
-The ``spack verify`` command can also verify for individual files that
-they haven't been altered since installation time. If the given file
+The ``spack verify manifest`` command can also verify for individual files
+that they haven't been altered since installation time. If the given file
 is not in a Spack installation prefix, Spack will report that it is
 not owned by any package. To check individual files instead of specs,
 use the ``-f,--files`` option.
@@ -1788,6 +1793,22 @@ check only local packages (as opposed to those used transparently from
 ``upstream`` spack instances) and the ``-j,--json`` option to output
 machine-readable json data for any errors.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+``spack verify libraries``
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``spack verify libraries`` command can be used to verify that packages
+do not have accidental system dependencies. This command scans the install
+prefixes of packages for executables and shared libraries, and resolves
+their needed libraries in their RPATHs. When needed libraries cannot be
+located, an error is reported. This typically indicates that a package
+was linked against a system library, instead of a library provided by
+a Spack package.
+
+This verification can also be enabled as a post-install hook by setting
+``config:shared_linking:missing_library_policy`` to ``error`` or ``warn``
+in :ref:`config.yaml <config-yaml>`.
+
 -----------------------
 Filesystem requirements
 -----------------------

lib/spack/docs/conf.py

@@ -223,6 +223,10 @@ def setup(sphinx):
     ("py:class", "spack.compiler.CompilerCache"),
     # TypeVar that is not handled correctly
     ("py:class", "llnl.util.lang.T"),
+    ("py:class", "llnl.util.lang.KT"),
+    ("py:class", "llnl.util.lang.VT"),
+    ("py:obj", "llnl.util.lang.KT"),
+    ("py:obj", "llnl.util.lang.VT"),
 ]
 
 # The reST default role (used for this markup: `text`) to use for all documents.

lib/spack/docs/config_yaml.rst

@@ -125,6 +125,8 @@ are stored in ``$spack/var/spack/cache``.  These are stored indefinitely
 by default. Can be purged with :ref:`spack clean --downloads
 <cmd-spack-clean>`.
 
+.. _Misc Cache:
+
 --------------------
 ``misc_cache``
 --------------------
@@ -334,3 +336,52 @@ create a new alias called ``inst`` that will always call ``install -v``:
 
    aliases:
      inst: install -v
+
+-------------------------------
+``concretization_cache:enable``
+-------------------------------
+
+When set to ``true``, Spack will utilize a cache of solver outputs from
+successful concretization runs. When enabled, Spack will check the concretization
+cache prior to running the solver. If a previous request to solve a given
+problem is present in the cache, Spack will load the concrete specs and other
+solver data from the cache rather than running the solver. Specs not previously
+concretized will be added to the cache on a successful solve. The cache additionally
+holds solver statistics, so commands like ``spack solve`` will still return information
+about the run that produced a given solver result.
+
+This cache is a subcache of the :ref:`Misc Cache` and as such will be cleaned when the Misc
+Cache is cleaned.
+
+When ``false`` or ommitted, all concretization requests will be performed from scatch
+
+----------------------------
+``concretization_cache:url``
+----------------------------
+
+Path to the location where Spack will root the concretization cache. Currently this only supports
+paths on the local filesystem.
+
+Default location is under the :ref:`Misc Cache` at: ``$misc_cache/concretization``
+
+------------------------------------
+``concretization_cache:entry_limit``
+------------------------------------
+
+Sets a limit on the number of concretization results that Spack will cache. The limit is evaluated
+after each concretization run; if Spack has stored more results than the limit allows, the
+oldest concretization results are pruned until 10% of the limit has been removed.
+
+Setting this value to 0 disables the automatic pruning. It is expected users will be
+responsible for maintaining this cache.
+
+-----------------------------------
+``concretization_cache:size_limit``
+-----------------------------------
+
+Sets a limit on the size of the concretization cache in bytes. The limit is evaluated
+after each concretization run; if Spack has stored more results than the limit allows, the
+oldest concretization results are pruned until 10% of the limit has been removed.
+
+Setting this value to 0 disables the automatic pruning. It is expected users will be
+responsible for maintaining this cache.

lib/spack/docs/pipelines.rst

@@ -820,6 +820,69 @@ presence of a ``SPACK_CDASH_AUTH_TOKEN`` environment variable during the
 build group on CDash called "Release Testing" (that group will be created if
 it didn't already exist).
 
+.. _ci_artifacts:
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+CI Artifacts Directory Layout
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When running the CI build using the command ``spack ci rebuild`` a number of directories are created for
+storing data generated during the CI job. The default root directory for artifacts is ``job_scratch_root``.
+This can be overridden by passing the argument ``--artifacts-root`` to the ``spack ci generate`` command
+or by setting the ``SPACK_ARTIFACTS_ROOT`` environment variable in the build job scripts.
+
+The top level directories under the artifact root are ``concrete_environment``, ``logs``, ``reproduction``,
+``tests``, and ``user_data``. Spack does not restrict what is written to any of these directories nor does
+it require user specified files be written to any specific directory.
+
+------------------------
+``concrete_environment``
+------------------------
+
+The directory ``concrete_environment`` is used to communicate the ci generate processed ``spack.yaml`` and
+the concrete ``spack.lock`` for the CI environment.
+
+--------
+``logs``
+--------
+
+The directory ``logs`` contains the spack build log, ``spack-build-out.txt``, and the spack build environment
+modification file, ``spack-build-mod-env.txt``. Additionally all files specified by the packages ``Builder``
+property ``archive_files`` are also copied here (ie. ``CMakeCache.txt`` in ``CMakeBuilder``).
+
+----------------
+``reproduction``
+----------------
+
+The directory ``reproduction`` is used to store the files needed by the ``spack reproduce-build`` command.
+This includes ``repro.json``, copies of all of the files in ``concrete_environment``, the concrete spec
+JSON file for the current spec being built, and all of the files written in the artifacts root directory.
+
+The ``repro.json`` file is not versioned and is only designed to work with the version of spack CI was run with.
+An example of what a ``repro.json`` may look like is here.
+
+.. code:: json
+
+  {
+    "job_name": "adios2@2.9.2 /feaevuj %gcc@11.4.0 arch=linux-ubuntu20.04-x86_64_v3 E4S ROCm External",
+    "job_spec_json": "adios2.json",
+    "ci_project_dir": "/builds/spack/spack"
+  }
+
+---------
+``tests``
+---------
+
+The directory ``tests`` is used to store output from running ``spack test <job spec>``. This may or may not have
+data in it depending on the package that was built and the availability of tests.
+
+-------------
+``user_data``
+-------------
+
+The directory ``user_data`` is used to store everything else that shouldn't be copied to the ``reproduction`` direcotory.
+Users may use this to store additional logs or metrics or other types of files generated by the build job.
+
 -------------------------------------
 Using a custom spack in your pipeline
 -------------------------------------

lib/spack/docs/requirements.txt

@@ -1,13 +1,13 @@
-sphinx==8.1.3
+sphinx==8.2.3
 sphinxcontrib-programoutput==0.18
 sphinx_design==0.6.1
 sphinx-rtd-theme==3.0.2
-python-levenshtein==0.26.1
+python-levenshtein==0.27.1
 docutils==0.21.2
 pygments==2.19.1
 urllib3==2.3.0
-pytest==8.3.4
-isort==6.0.0
+pytest==8.3.5
+isort==6.0.1
 black==25.1.0
 flake8==7.1.2
 mypy==1.11.1

lib/spack/llnl/util/filesystem.py

@@ -7,6 +7,7 @@
 import fnmatch
 import glob
 import hashlib
+import io
 import itertools
 import numbers
 import os
@@ -20,6 +21,7 @@
 from contextlib import contextmanager
 from itertools import accumulate
 from typing import (
+    IO,
     Callable,
     Deque,
     Dict,
@@ -2454,26 +2456,69 @@ class WindowsSimulatedRPath:
     and vis versa.
     """
 
-    def __init__(self, package, link_install_prefix=True):
+    def __init__(
+        self,
+        package,
+        base_modification_prefix: Optional[Union[str, pathlib.Path]] = None,
+        link_install_prefix: bool = True,
+    ):
         """
         Args:
             package (spack.package_base.PackageBase): Package requiring links
+            base_modification_prefix (str|pathlib.Path): Path representation indicating
+                the root directory in which to establish the simulated rpath, ie where the
+                symlinks that comprise the "rpath" behavior will be installed.
+
+                Note: This is a mutually exclusive option with `link_install_prefix` using
+                both is an error.
+
+                Default: None
             link_install_prefix (bool): Link against package's own install or stage root.
                 Packages that run their own executables during build and require rpaths to
-                the build directory during build time require this option. Default: install
+                the build directory during build time require this option.
+
+                Default: install
                 root
+
+                Note: This is a mutually exclusive option with `base_modification_prefix`, using
+                both is an error.
         """
         self.pkg = package
-        self._addl_rpaths = set()
+        self._addl_rpaths: set[str] = set()
+        if link_install_prefix and base_modification_prefix:
+            raise RuntimeError(
+                "Invalid combination of arguments given to WindowsSimulated RPath.\n"
+                "Select either `link_install_prefix` to create an install prefix rpath"
+                " or specify a `base_modification_prefix` for any other link type. "
+                "Specifying both arguments is invalid."
+            )
+        if not (link_install_prefix or base_modification_prefix):
+            raise RuntimeError(
+                "Insufficient arguments given to WindowsSimulatedRpath.\n"
+                "WindowsSimulatedRPath requires one of link_install_prefix"
+                " or base_modification_prefix to be specified."
+                " Neither was provided."
+            )
+
         self.link_install_prefix = link_install_prefix
-        self._additional_library_dependents = set()
+        if base_modification_prefix:
+            self.base_modification_prefix = pathlib.Path(base_modification_prefix)
+        else:
+            self.base_modification_prefix = pathlib.Path(self.pkg.prefix)
+        self._additional_library_dependents: set[pathlib.Path] = set()
+        if not self.link_install_prefix:
+            tty.debug(f"Generating rpath for non install context: {base_modification_prefix}")
 
     @property
     def library_dependents(self):
         """
         Set of directories where package binaries/libraries are located.
         """
-        return set([pathlib.Path(self.pkg.prefix.bin)]) | self._additional_library_dependents
+        base_pths = set()
+        if self.link_install_prefix:
+            base_pths.add(pathlib.Path(self.pkg.prefix.bin))
+        base_pths |= self._additional_library_dependents
+        return base_pths
 
     def add_library_dependent(self, *dest):
         """
@@ -2489,6 +2534,12 @@ def add_library_dependent(self, *dest):
                 new_pth = pathlib.Path(pth).parent
             else:
                 new_pth = pathlib.Path(pth)
+            path_is_in_prefix = new_pth.is_relative_to(self.base_modification_prefix)
+            if not path_is_in_prefix:
+                raise RuntimeError(
+                    f"Attempting to generate rpath symlink out of rpath context:\
+{str(self.base_modification_prefix)}"
+                )
             self._additional_library_dependents.add(new_pth)
 
     @property
@@ -2577,6 +2628,33 @@ def establish_link(self):
                 self._link(library, lib_dir)
 
 
+def make_package_test_rpath(pkg, test_dir: Union[str, pathlib.Path]):
+    """Establishes a temp Windows simulated rpath for the pkg in the testing directory
+    so an executable can test the libraries/executables with proper access
+    to dependent dlls
+
+    Note: this is a no-op on all other platforms besides Windows
+
+    Args:
+        pkg (spack.package_base.PackageBase): the package for which the rpath should be computed
+        test_dir: the testing directory in which we should construct an rpath
+    """
+    # link_install_prefix as false ensures we're not linking into the install prefix
+    mini_rpath = WindowsSimulatedRPath(pkg, link_install_prefix=False)
+    # add the testing directory as a location to install rpath symlinks
+    mini_rpath.add_library_dependent(test_dir)
+
+    # check for whether build_directory is available, if not
+    # assume the stage root is the build dir
+    build_dir_attr = getattr(pkg, "build_directory", None)
+    build_directory = build_dir_attr if build_dir_attr else pkg.stage.path
+    # add the build dir & build dir bin
+    mini_rpath.add_rpath(os.path.join(build_directory, "bin"))
+    mini_rpath.add_rpath(os.path.join(build_directory))
+    # construct rpath
+    mini_rpath.establish_link()
+
+
 @system_path_filter
 @memoized
 def can_access_dir(path):
@@ -2805,6 +2883,20 @@ def keep_modification_time(*filenames):
             os.utime(f, (os.path.getatime(f), mtime))
 
 
+@contextmanager
+def temporary_file_position(stream):
+    orig_pos = stream.tell()
+    yield
+    stream.seek(orig_pos)
+
+
+@contextmanager
+def current_file_position(stream: IO[str], loc: int, relative_to=io.SEEK_CUR):
+    with temporary_file_position(stream):
+        stream.seek(loc, relative_to)
+        yield
+
+
 @contextmanager
 def temporary_dir(
     suffix: Optional[str] = None, prefix: Optional[str] = None, dir: Optional[str] = None

lib/spack/llnl/util/lang.py

@@ -14,7 +14,7 @@
 import typing
 import warnings
 from datetime import datetime, timedelta
-from typing import Callable, Dict, Iterable, List, Tuple, TypeVar
+from typing import Callable, Dict, Iterable, List, Mapping, Optional, Tuple, TypeVar
 
 # Ignore emacs backups when listing modules
 ignore_modules = r"^\.#|~$"
@@ -1080,3 +1080,88 @@ def __set__(self, instance, value):
 
     def factory(self, instance, owner):
         raise NotImplementedError("must be implemented by derived classes")
+
+
+KT = TypeVar("KT")
+VT = TypeVar("VT")
+
+
+class PriorityOrderedMapping(Mapping[KT, VT]):
+    """Mapping that iterates over key according to an integer priority. If the priority is
+    the same for two keys, insertion order is what matters.
+
+    The priority is set when the key/value pair is added. If not set, the highest current priority
+    is used.
+    """
+
+    _data: Dict[KT, VT]
+    _priorities: List[Tuple[int, KT]]
+
+    def __init__(self) -> None:
+        self._data = {}
+        # Tuple of (priority, key)
+        self._priorities = []
+
+    def __getitem__(self, key: KT) -> VT:
+        return self._data[key]
+
+    def __len__(self) -> int:
+        return len(self._data)
+
+    def __iter__(self):
+        yield from (key for _, key in self._priorities)
+
+    def __reversed__(self):
+        yield from (key for _, key in reversed(self._priorities))
+
+    def reversed_keys(self):
+        """Iterates over keys from the highest priority, to the lowest."""
+        return reversed(self)
+
+    def reversed_values(self):
+        """Iterates over values from the highest priority, to the lowest."""
+        yield from (self._data[key] for _, key in reversed(self._priorities))
+
+    def _highest_priority(self) -> int:
+        if not self._priorities:
+            return 0
+        result, _ = self._priorities[-1]
+        return result
+
+    def add(self, key: KT, *, value: VT, priority: Optional[int] = None) -> None:
+        """Adds a key/value pair to the mapping, with a specific priority.
+
+        If the priority is None, then it is assumed to be the highest priority value currently
+        in the container.
+
+        Raises:
+              ValueError: when the same priority is already in the mapping
+        """
+        if priority is None:
+            priority = self._highest_priority()
+
+        if key in self._data:
+            self.remove(key)
+
+        self._priorities.append((priority, key))
+        # We rely on sort being stable
+        self._priorities.sort(key=lambda x: x[0])
+        self._data[key] = value
+        assert len(self._data) == len(self._priorities)
+
+    def remove(self, key: KT) -> VT:
+        """Removes a key from the mapping.
+
+        Returns:
+            The value associated with the key being removed
+
+        Raises:
+            KeyError: if the key is not in the mapping
+        """
+        if key not in self._data:
+            raise KeyError(f"cannot find {key}")
+
+        popped_item = self._data.pop(key)
+        self._priorities = [(p, k) for p, k in self._priorities if k != key]
+        assert len(self._data) == len(self._priorities)
+        return popped_item

lib/spack/spack/__init__.py

@@ -13,6 +13,18 @@
 __version__ = "1.0.0.dev0"
 spack_version = __version__
 
+#: The current Package API version implemented by this version of Spack. The Package API defines
+#: the Python interface for packages as well as the layout of package repositories. The minor
+#: version is incremented when the package API is extended in a backwards-compatible way. The major
+#: version is incremented upon breaking changes. This version is changed independently from the
+#: Spack version.
+package_api_version = (1, 0)
+
+#: The minimum Package API version that this version of Spack is compatible with. This should
+#: always be a tuple of the form ``(major, 0)``, since compatibility with vX.Y implies
+#: compatibility with vX.0.
+min_package_api_version = (1, 0)
+
 
 def __try_int(v):
     try:
@@ -79,4 +91,6 @@ def get_short_version() -> str:
     "get_version",
     "get_spack_commit",
     "get_short_version",
+    "package_api_version",
+    "min_package_api_version",
 ]

lib/spack/spack/bootstrap/core.py

@@ -292,7 +292,12 @@ def try_import(self, module: str, abstract_spec_str: str) -> bool:
 
         # Install the spec that should make the module importable
         with spack.config.override(self.mirror_scope):
-            PackageInstaller([concrete_spec.package], fail_fast=True).install()
+            PackageInstaller(
+                [concrete_spec.package],
+                fail_fast=True,
+                package_use_cache=False,
+                dependencies_use_cache=False,
+            ).install()
 
         if _try_import_from_store(module, query_spec=concrete_spec, query_info=info):
             self.last_search = info
@@ -362,6 +367,7 @@ def ensure_module_importable_or_raise(module: str, abstract_spec: Optional[str]
     for current_config in bootstrapping_sources():
         if not source_is_enabled(current_config):
             continue
+
         with exception_handler.forward(current_config["name"], Exception):
             if create_bootstrapper(current_config).try_import(module, abstract_spec):
                 return

lib/spack/spack/build_systems/cached_cmake.py

@@ -12,6 +12,7 @@
 import spack.phase_callbacks
 import spack.spec
 import spack.util.prefix
+from spack.directives import depends_on
 
 from .cmake import CMakeBuilder, CMakePackage
 
@@ -371,6 +372,10 @@ class CachedCMakePackage(CMakePackage):
 
     CMakeBuilder = CachedCMakeBuilder
 
+    # These dependencies are assumed in the builder
+    depends_on("c", type="build")
+    depends_on("cxx", type="build")
+
     def flag_handler(self, name, flags):
         if name in ("cflags", "cxxflags", "cppflags", "fflags"):
             return None, None, None  # handled in the cmake cache

lib/spack/spack/build_systems/cargo.py

@@ -70,10 +70,16 @@ def build_directory(self):
         """Return the directory containing the main Cargo.toml."""
         return self.pkg.stage.source_path
 
+    @property
+    def std_build_args(self):
+        """Standard arguments for ``cargo build`` provided as a property for
+        convenience of package writers."""
+        return ["-j", str(self.pkg.module.make_jobs)]
+
     @property
     def build_args(self):
         """Arguments for ``cargo build``."""
-        return ["-j", str(self.pkg.module.make_jobs)]
+        return []
 
     @property
     def check_args(self):
@@ -88,7 +94,9 @@ def build(
     ) -> None:
         """Runs ``cargo install`` in the source directory"""
         with fs.working_dir(self.build_directory):
-            pkg.module.cargo("install", "--root", "out", "--path", ".", *self.build_args)
+            pkg.module.cargo(
+                "install", "--root", "out", "--path", ".", *self.std_build_args, *self.build_args
+            )
 
     def install(
         self, pkg: CargoPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix

lib/spack/spack/build_systems/oneapi.py

@@ -142,7 +142,7 @@ def setup_run_environment(self, env):
            $ source {prefix}/{component}/{version}/env/vars.sh
         """
         # Only if environment modifications are desired (default is +envmods)
-        if "~envmods" not in self.spec:
+        if "+envmods" in self.spec:
             env.extend(
                 EnvironmentModifications.from_sourcing_file(
                     self.component_prefix.env.join("vars.sh"), *self.env_script_args

lib/spack/spack/ci/__init__.py

@@ -616,7 +616,7 @@ def copy_test_logs_to_artifacts(test_stage, job_test_dir):
     copy_files_to_artifacts(os.path.join(test_stage, "*", "*.txt"), job_test_dir)
 
 
-def download_and_extract_artifacts(url, work_dir):
+def download_and_extract_artifacts(url, work_dir) -> str:
     """Look for gitlab artifacts.zip at the given url, and attempt to download
         and extract the contents into the given work_dir
 
@@ -624,6 +624,10 @@ def download_and_extract_artifacts(url, work_dir):
 
         url (str): Complete url to artifacts.zip file
         work_dir (str): Path to destination where artifacts should be extracted
+
+    Output:
+
+        Artifacts root path relative to the archive root
     """
     tty.msg(f"Fetching artifacts from: {url}")
 
@@ -641,13 +645,25 @@ def download_and_extract_artifacts(url, work_dir):
         response = urlopen(request, timeout=SPACK_CDASH_TIMEOUT)
         with open(artifacts_zip_path, "wb") as out_file:
             shutil.copyfileobj(response, out_file)
+
+        with zipfile.ZipFile(artifacts_zip_path) as zip_file:
+            zip_file.extractall(work_dir)
+            # Get the artifact root
+            artifact_root = ""
+            for f in zip_file.filelist:
+                if "spack.lock" in f.filename:
+                    artifact_root = os.path.dirname(os.path.dirname(f.filename))
+                    break
     except OSError as e:
         raise SpackError(f"Error fetching artifacts: {e}")
+    finally:
+        try:
+            os.remove(artifacts_zip_path)
+        except FileNotFoundError:
+            # If the file doesn't exist we are already raising
+            pass
 
-    with zipfile.ZipFile(artifacts_zip_path) as zip_file:
-        zip_file.extractall(work_dir)
-
-    os.remove(artifacts_zip_path)
+    return artifact_root
 
 
 def get_spack_info():
@@ -761,7 +777,7 @@ def setup_spack_repro_version(repro_dir, checkout_commit, merge_commit=None):
     return True
 
 
-def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
+def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime, use_local_head):
     """Given a url to gitlab artifacts.zip from a failed 'spack ci rebuild' job,
     attempt to setup an environment in which the failure can be reproduced
     locally.  This entails the following:
@@ -775,8 +791,11 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
     commands to run to reproduce the build once inside the container.
     """
     work_dir = os.path.realpath(work_dir)
+    if os.path.exists(work_dir) and os.listdir(work_dir):
+        raise SpackError(f"Cannot run reproducer in non-emptry working dir:\n  {work_dir}")
+
     platform_script_ext = "ps1" if IS_WINDOWS else "sh"
-    download_and_extract_artifacts(url, work_dir)
+    artifact_root = download_and_extract_artifacts(url, work_dir)
 
     gpg_path = None
     if gpg_url:
@@ -838,6 +857,9 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
     with open(repro_file, encoding="utf-8") as fd:
         repro_details = json.load(fd)
 
+    spec_file = fs.find(work_dir, repro_details["job_spec_json"])[0]
+    reproducer_spec = spack.spec.Spec.from_specfile(spec_file)
+
     repro_dir = os.path.dirname(repro_file)
     rel_repro_dir = repro_dir.replace(work_dir, "").lstrip(os.path.sep)
 
@@ -898,17 +920,20 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
     commit_regex = re.compile(r"commit\s+([^\s]+)")
     merge_commit_regex = re.compile(r"Merge\s+([^\s]+)\s+into\s+([^\s]+)")
 
-    # Try the more specific merge commit regex first
-    m = merge_commit_regex.search(spack_info)
-    if m:
-        # This was a merge commit and we captured the parents
-        commit_1 = m.group(1)
-        commit_2 = m.group(2)
+    if use_local_head:
+        commit_1 = "HEAD"
     else:
-        # Not a merge commit, just get the commit sha
-        m = commit_regex.search(spack_info)
+        # Try the more specific merge commit regex first
+        m = merge_commit_regex.search(spack_info)
         if m:
+            # This was a merge commit and we captured the parents
             commit_1 = m.group(1)
+            commit_2 = m.group(2)
+        else:
+            # Not a merge commit, just get the commit sha
+            m = commit_regex.search(spack_info)
+            if m:
+                commit_1 = m.group(1)
 
     setup_result = False
     if commit_1:
@@ -983,6 +1008,8 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
             "entrypoint", entrypoint_script, work_dir, run=False, exit_on_failure=False
         )
 
+        # Attempt to create a unique name for the reproducer container
+        container_suffix = "_" + reproducer_spec.dag_hash() if reproducer_spec else ""
         docker_command = [
             runtime,
             "run",
@@ -990,14 +1017,14 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
             "-t",
             "--rm",
             "--name",
-            "spack_reproducer",
+            f"spack_reproducer{container_suffix}",
             "-v",
             ":".join([work_dir, mounted_workdir, "Z"]),
             "-v",
             ":".join(
                 [
-                    os.path.join(work_dir, "jobs_scratch_dir"),
-                    os.path.join(mount_as_dir, "jobs_scratch_dir"),
+                    os.path.join(work_dir, artifact_root),
+                    os.path.join(mount_as_dir, artifact_root),
                     "Z",
                 ]
             ),

lib/spack/spack/cmd/ci.py

@@ -176,6 +176,11 @@ def setup_parser(subparser):
     reproduce.add_argument(
         "-s", "--autostart", help="Run docker reproducer automatically", action="store_true"
     )
+    reproduce.add_argument(
+        "--use-local-head",
+        help="Use the HEAD of the local Spack instead of reproducing a commit",
+        action="store_true",
+    )
     gpg_group = reproduce.add_mutually_exclusive_group(required=False)
     gpg_group.add_argument(
         "--gpg-file", help="Path to public GPG key for validating binary cache installs"
@@ -608,7 +613,12 @@ def ci_reproduce(args):
         gpg_key_url = None
 
     return spack_ci.reproduce_ci_job(
-        args.job_url, args.working_dir, args.autostart, gpg_key_url, args.runtime
+        args.job_url,
+        args.working_dir,
+        args.autostart,
+        gpg_key_url,
+        args.runtime,
+        args.use_local_head,
     )
 
 

lib/spack/spack/cmd/common/arguments.py

@@ -528,7 +528,6 @@ def __call__(self, parser, namespace, values, option_string):
         # the const from the constructor or a value from the CLI.
         # Note that this is only called if the argument is actually
         # specified on the command line.
-        spack.config.CONFIG.ensure_scope_ordering()
         spack.config.set(self.config_path, self.const, scope="command_line")
 
 

lib/spack/spack/cmd/unit_test.py

@@ -216,7 +216,7 @@ def unit_test(parser, args, unknown_args):
     # Ensure clingo is available before switching to the
     # mock configuration used by unit tests
     with spack.bootstrap.ensure_bootstrap_configuration():
-        spack.bootstrap.ensure_core_dependencies()
+        spack.bootstrap.ensure_clingo_importable_or_raise()
         if pytest is None:
             spack.bootstrap.ensure_environment_dependencies()
             import pytest

lib/spack/spack/cmd/verify.py

@@ -2,35 +2,48 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import argparse
+import io
+from typing import List, Optional
 
 import llnl.util.tty as tty
+from llnl.string import plural
+from llnl.util.filesystem import visit_directory_tree
 
 import spack.cmd
 import spack.environment as ev
+import spack.spec
 import spack.store
 import spack.verify
+import spack.verify_libraries
+from spack.cmd.common import arguments
 
-description = "check that all spack packages are on disk as installed"
+description = "verify spack installations on disk"
 section = "admin"
 level = "long"
 
+MANIFEST_SUBPARSER: Optional[argparse.ArgumentParser] = None
 
-def setup_parser(subparser):
-    setup_parser.parser = subparser
 
-    subparser.add_argument(
+def setup_parser(subparser: argparse.ArgumentParser):
+    global MANIFEST_SUBPARSER
+    sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="verify_command")
+
+    MANIFEST_SUBPARSER = sp.add_parser(
+        "manifest", help=verify_manifest.__doc__, description=verify_manifest.__doc__
+    )
+    MANIFEST_SUBPARSER.add_argument(
         "-l", "--local", action="store_true", help="verify only locally installed packages"
     )
-    subparser.add_argument(
+    MANIFEST_SUBPARSER.add_argument(
         "-j", "--json", action="store_true", help="ouptut json-formatted errors"
     )
-    subparser.add_argument("-a", "--all", action="store_true", help="verify all packages")
-    subparser.add_argument(
+    MANIFEST_SUBPARSER.add_argument("-a", "--all", action="store_true", help="verify all packages")
+    MANIFEST_SUBPARSER.add_argument(
         "specs_or_files", nargs=argparse.REMAINDER, help="specs or files to verify"
     )
 
-    type = subparser.add_mutually_exclusive_group()
-    type.add_argument(
+    manifest_sp_type = MANIFEST_SUBPARSER.add_mutually_exclusive_group()
+    manifest_sp_type.add_argument(
         "-s",
         "--specs",
         action="store_const",
@@ -39,7 +52,7 @@ def setup_parser(subparser):
         default="specs",
         help="treat entries as specs (default)",
     )
-    type.add_argument(
+    manifest_sp_type.add_argument(
         "-f",
         "--files",
         action="store_const",
@@ -49,14 +62,67 @@ def setup_parser(subparser):
         help="treat entries as absolute filenames\n\ncannot be used with '-a'",
     )
 
+    libraries_subparser = sp.add_parser(
+        "libraries", help=verify_libraries.__doc__, description=verify_libraries.__doc__
+    )
+
+    arguments.add_common_arguments(libraries_subparser, ["constraint"])
+
 
 def verify(parser, args):
+    cmd = args.verify_command
+    if cmd == "libraries":
+        return verify_libraries(args)
+    elif cmd == "manifest":
+        return verify_manifest(args)
+    parser.error("invalid verify subcommand")
+
+
+def verify_libraries(args):
+    """verify that shared libraries of install packages can be located in rpaths (Linux only)"""
+    specs_from_db = [s for s in args.specs(installed=True) if not s.external]
+
+    tty.info(f"Checking {len(specs_from_db)} packages for shared library resolution")
+
+    errors = 0
+    for spec in specs_from_db:
+        try:
+            pkg = spec.package
+        except Exception:
+            tty.warn(f"Skipping {spec.cformat('{name}{@version}{/hash}')} due to missing package")
+        error_msg = _verify_libraries(spec, pkg.unresolved_libraries)
+        if error_msg is not None:
+            errors += 1
+            tty.error(error_msg)
+
+    if errors:
+        tty.error(f"Cannot resolve shared libraries in {plural(errors, 'package')}")
+        return 1
+
+
+def _verify_libraries(spec: spack.spec.Spec, unresolved_libraries: List[str]) -> Optional[str]:
+    """Go over the prefix of the installed spec and verify its shared libraries can be resolved."""
+    visitor = spack.verify_libraries.ResolveSharedElfLibDepsVisitor(
+        [*spack.verify_libraries.ALLOW_UNRESOLVED, *unresolved_libraries]
+    )
+    visit_directory_tree(spec.prefix, visitor)
+
+    if not visitor.problems:
+        return None
+
+    output = io.StringIO()
+    visitor.write(output, indent=4, brief=True)
+    message = output.getvalue().rstrip()
+    return f"{spec.cformat('{name}{@version}{/hash}')}: {spec.prefix}:\n{message}"
+
+
+def verify_manifest(args):
+    """verify that install directories have not been modified since installation"""
     local = args.local
 
     if args.type == "files":
         if args.all:
-            setup_parser.parser.print_help()
-            return 1
+            MANIFEST_SUBPARSER.error("cannot use --all with --files")
 
         for file in args.specs_or_files:
             results = spack.verify.check_file_manifest(file)
@@ -87,8 +153,7 @@ def verify(parser, args):
         env = ev.active_environment()
         specs = list(map(lambda x: spack.cmd.disambiguate_spec(x, env, local=local), spec_args))
     else:
-        setup_parser.parser.print_help()
-        return 1
+        MANIFEST_SUBPARSER.error("use --all or specify specs to verify")
 
     for spec in specs:
         tty.debug("Verifying package %s")

lib/spack/spack/config.py

@@ -66,6 +66,8 @@
 import spack.util.web as web_util
 from spack.util.cpus import cpus_available
 
+from .enums import ConfigScopePriority
+
 #: Dict from section names -> schema for that section
 SECTION_SCHEMAS: Dict[str, Any] = {
     "compilers": spack.schema.compilers.schema,
@@ -408,26 +410,18 @@ def _method(self, *args, **kwargs):
     return _method
 
 
-class Configuration:
-    """A full Spack configuration, from a hierarchy of config files.
+ScopeWithOptionalPriority = Union[ConfigScope, Tuple[int, ConfigScope]]
+ScopeWithPriority = Tuple[int, ConfigScope]
 
-    This class makes it easy to add a new scope on top of an existing one.
-    """
+
+class Configuration:
+    """A hierarchical configuration, merging a number of scopes at different priorities."""
 
     # convert to typing.OrderedDict when we drop 3.6, or OrderedDict when we reach 3.9
-    scopes: Dict[str, ConfigScope]
+    scopes: lang.PriorityOrderedMapping[str, ConfigScope]
 
-    def __init__(self, *scopes: ConfigScope) -> None:
-        """Initialize a configuration with an initial list of scopes.
-
-        Args:
-            scopes: list of scopes to add to this
-                Configuration, ordered from lowest to highest precedence
-
-        """
-        self.scopes = collections.OrderedDict()
-        for scope in scopes:
-            self.push_scope(scope)
+    def __init__(self) -> None:
+        self.scopes = lang.PriorityOrderedMapping()
         self.format_updates: Dict[str, List[ConfigScope]] = collections.defaultdict(list)
 
     def ensure_unwrapped(self) -> "Configuration":
@@ -435,36 +429,31 @@ def ensure_unwrapped(self) -> "Configuration":
         return self
 
     def highest(self) -> ConfigScope:
-        """Scope with highest precedence"""
-        return next(reversed(self.scopes.values()))  # type: ignore
+        """Scope with the highest precedence"""
+        return next(self.scopes.reversed_values())  # type: ignore
 
     @_config_mutator
-    def ensure_scope_ordering(self):
-        """Ensure that scope order matches documented precedent"""
-        # FIXME: We also need to consider that custom configurations and other orderings
-        # may not be preserved correctly
-        if "command_line" in self.scopes:
-            # TODO (when dropping python 3.6): self.scopes.move_to_end
-            self.scopes["command_line"] = self.remove_scope("command_line")
+    def push_scope(self, scope: ConfigScope, priority: Optional[int] = None) -> None:
+        """Adds a scope to the Configuration, at a given priority.
 
-    @_config_mutator
-    def push_scope(self, scope: ConfigScope) -> None:
-        """Add a higher precedence scope to the Configuration."""
-        tty.debug(f"[CONFIGURATION: PUSH SCOPE]: {str(scope)}", level=2)
-        self.scopes[scope.name] = scope
+        If a priority is not given, it is assumed to be the current highest priority.
 
-    @_config_mutator
-    def pop_scope(self) -> ConfigScope:
-        """Remove the highest precedence scope and return it."""
-        name, scope = self.scopes.popitem(last=True)  # type: ignore[call-arg]
-        tty.debug(f"[CONFIGURATION: POP SCOPE]: {str(scope)}", level=2)
-        return scope
+        Args:
+            scope: scope to be added
+            priority: priority of the scope
+        """
+        tty.debug(f"[CONFIGURATION: PUSH SCOPE]: {str(scope)}, priority={priority}", level=2)
+        self.scopes.add(scope.name, value=scope, priority=priority)
 
     @_config_mutator
     def remove_scope(self, scope_name: str) -> Optional[ConfigScope]:
-        """Remove scope by name; has no effect when ``scope_name`` does not exist"""
-        scope = self.scopes.pop(scope_name, None)
-        tty.debug(f"[CONFIGURATION: POP SCOPE]: {str(scope)}", level=2)
+        """Removes a scope by name, and returns it. If the scope does not exist, returns None."""
+        try:
+            scope = self.scopes.remove(scope_name)
+            tty.debug(f"[CONFIGURATION: POP SCOPE]: {str(scope)}", level=2)
+        except KeyError as e:
+            tty.debug(f"[CONFIGURATION: POP SCOPE]: {e}", level=2)
+            return None
         return scope
 
     @property
@@ -473,15 +462,13 @@ def writable_scopes(self) -> Generator[ConfigScope, None, None]:
         return (s for s in self.scopes.values() if s.writable)
 
     def highest_precedence_scope(self) -> ConfigScope:
-        """Writable scope with highest precedence."""
-        return next(s for s in reversed(self.scopes.values()) if s.writable)  # type: ignore
+        """Writable scope with the highest precedence."""
+        return next(s for s in self.scopes.reversed_values() if s.writable)
 
     def highest_precedence_non_platform_scope(self) -> ConfigScope:
-        """Writable non-platform scope with highest precedence"""
+        """Writable non-platform scope with the highest precedence"""
         return next(
-            s
-            for s in reversed(self.scopes.values())  # type: ignore
-            if s.writable and not s.is_platform_dependent
+            s for s in self.scopes.reversed_values() if s.writable and not s.is_platform_dependent
         )
 
     def matching_scopes(self, reg_expr) -> List[ConfigScope]:
@@ -748,7 +735,7 @@ def override(
     """
     if isinstance(path_or_scope, ConfigScope):
         overrides = path_or_scope
-        CONFIG.push_scope(path_or_scope)
+        CONFIG.push_scope(path_or_scope, priority=None)
     else:
         base_name = _OVERRIDES_BASE_NAME
         # Ensure the new override gets a unique scope name
@@ -762,7 +749,7 @@ def override(
                 break
 
         overrides = InternalConfigScope(scope_name)
-        CONFIG.push_scope(overrides)
+        CONFIG.push_scope(overrides, priority=None)
         CONFIG.set(path_or_scope, value, scope=scope_name)
 
     try:
@@ -772,13 +759,15 @@ def override(
         assert scope is overrides
 
 
-def _add_platform_scope(cfg: Configuration, name: str, path: str, writable: bool = True) -> None:
+def _add_platform_scope(
+    cfg: Configuration, name: str, path: str, priority: ConfigScopePriority, writable: bool = True
+) -> None:
     """Add a platform-specific subdirectory for the current platform."""
     platform = spack.platforms.host().name
     scope = DirectoryConfigScope(
         f"{name}/{platform}", os.path.join(path, platform), writable=writable
     )
-    cfg.push_scope(scope)
+    cfg.push_scope(scope, priority=priority)
 
 
 def config_paths_from_entry_points() -> List[Tuple[str, str]]:
@@ -813,11 +802,10 @@ def create() -> Configuration:
     it. It is bundled inside a function so that configuration can be
     initialized lazily.
     """
-    cfg = Configuration()
-
     # first do the builtin, hardcoded defaults
-    builtin = InternalConfigScope("_builtin", CONFIG_DEFAULTS)
-    cfg.push_scope(builtin)
+    cfg = create_from(
+        (ConfigScopePriority.BUILTIN, InternalConfigScope("_builtin", CONFIG_DEFAULTS))
+    )
 
     # Builtin paths to configuration files in Spack
     configuration_paths = [
@@ -847,10 +835,9 @@ def create() -> Configuration:
 
     # add each scope and its platform-specific directory
     for name, path in configuration_paths:
-        cfg.push_scope(DirectoryConfigScope(name, path))
-
-        # Each scope can have per-platfom overrides in subdirectories
-        _add_platform_scope(cfg, name, path)
+        cfg.push_scope(DirectoryConfigScope(name, path), priority=ConfigScopePriority.CONFIG_FILES)
+        # Each scope can have per-platform overrides in subdirectories
+        _add_platform_scope(cfg, name, path, priority=ConfigScopePriority.CONFIG_FILES)
 
     return cfg
 
@@ -955,7 +942,7 @@ def set(path: str, value: Any, scope: Optional[str] = None) -> None:
     return CONFIG.set(path, value, scope)
 
 
-def scopes() -> Dict[str, ConfigScope]:
+def scopes() -> lang.PriorityOrderedMapping[str, ConfigScope]:
     """Convenience function to get list of configuration scopes."""
     return CONFIG.scopes
 
@@ -1409,7 +1396,7 @@ def ensure_latest_format_fn(section: str) -> Callable[[YamlConfigDict], bool]:
 
 @contextlib.contextmanager
 def use_configuration(
-    *scopes_or_paths: Union[ConfigScope, str]
+    *scopes_or_paths: Union[ScopeWithOptionalPriority, str]
 ) -> Generator[Configuration, None, None]:
     """Use the configuration scopes passed as arguments within the context manager.
 
@@ -1424,7 +1411,7 @@ def use_configuration(
     global CONFIG
 
     # Normalize input and construct a Configuration object
-    configuration = _config_from(scopes_or_paths)
+    configuration = create_from(*scopes_or_paths)
     CONFIG.clear_caches(), configuration.clear_caches()
 
     saved_config, CONFIG = CONFIG, configuration
@@ -1435,23 +1422,44 @@ def use_configuration(
         CONFIG = saved_config
 
 
+def _normalize_input(entry: Union[ScopeWithOptionalPriority, str]) -> ScopeWithPriority:
+    if isinstance(entry, tuple):
+        return entry
+
+    default_priority = ConfigScopePriority.CONFIG_FILES
+    if isinstance(entry, ConfigScope):
+        return default_priority, entry
+
+    # Otherwise we need to construct it
+    path = os.path.normpath(entry)
+    assert os.path.isdir(path), f'"{path}" must be a directory'
+    name = os.path.basename(path)
+    return default_priority, DirectoryConfigScope(name, path)
+
+
 @lang.memoized
-def _config_from(scopes_or_paths: List[Union[ConfigScope, str]]) -> Configuration:
-    scopes = []
-    for scope_or_path in scopes_or_paths:
-        # If we have a config scope we are already done
-        if isinstance(scope_or_path, ConfigScope):
-            scopes.append(scope_or_path)
-            continue
+def create_from(*scopes_or_paths: Union[ScopeWithOptionalPriority, str]) -> Configuration:
+    """Creates a configuration object from the scopes passed in input.
 
-        # Otherwise we need to construct it
-        path = os.path.normpath(scope_or_path)
-        assert os.path.isdir(path), f'"{path}" must be a directory'
-        name = os.path.basename(path)
-        scopes.append(DirectoryConfigScope(name, path))
+    Args:
+        *scopes_or_paths: either a tuple of (priority, ConfigScope), or a ConfigScope, or a string
+            If priority is not given, it is assumed to be ConfigScopePriority.CONFIG_FILES. If a
+            string is given, a DirectoryConfigScope is created from it.
 
-    configuration = Configuration(*scopes)
-    return configuration
+    Examples:
+
+        >>> builtin_scope = InternalConfigScope("_builtin", {"config": {"build_jobs": 1}})
+        >>> cl_scope = InternalConfigScope("command_line", {"config": {"build_jobs": 10}})
+        >>> cfg = create_from(
+        ...     (ConfigScopePriority.COMMAND_LINE, cl_scope),
+        ...     (ConfigScopePriority.BUILTIN, builtin_scope)
+        ... )
+    """
+    scopes_with_priority = [_normalize_input(x) for x in scopes_or_paths]
+    result = Configuration()
+    for priority, scope in scopes_with_priority:
+        result.push_scope(scope, priority=priority)
+    return result
 
 
 def raw_github_gitlab_url(url: str) -> str:

lib/spack/spack/detection/path.py

@@ -243,7 +243,7 @@ def prefix_from_path(self, *, path: str) -> str:
         raise NotImplementedError("must be implemented by derived classes")
 
     def detect_specs(
-        self, *, pkg: Type["spack.package_base.PackageBase"], paths: List[str]
+        self, *, pkg: Type["spack.package_base.PackageBase"], paths: Iterable[str]
     ) -> List["spack.spec.Spec"]:
         """Given a list of files matching the search patterns, returns a list of detected specs.
 

lib/spack/spack/directory_layout.py

@@ -25,7 +25,7 @@
 }
 
 
-def _check_concrete(spec):
+def _check_concrete(spec: "spack.spec.Spec") -> None:
     """If the spec is not concrete, raise a ValueError"""
     if not spec.concrete:
         raise ValueError("Specs passed to a DirectoryLayout must be concrete!")
@@ -51,7 +51,7 @@ def specs_from_metadata_dirs(root: str) -> List["spack.spec.Spec"]:
         spec = _get_spec(prefix)
 
         if spec:
-            spec.prefix = prefix
+            spec.set_prefix(prefix)
             specs.append(spec)
             continue
 
@@ -84,7 +84,7 @@ class DirectoryLayout:
 
     def __init__(
         self,
-        root,
+        root: str,
         *,
         projections: Optional[Dict[str, str]] = None,
         hash_length: Optional[int] = None,
@@ -120,17 +120,17 @@ def __init__(
         self.manifest_file_name = "install_manifest.json"
 
     @property
-    def hidden_file_regexes(self):
+    def hidden_file_regexes(self) -> Tuple[str]:
         return ("^{0}$".format(re.escape(self.metadata_dir)),)
 
-    def relative_path_for_spec(self, spec):
+    def relative_path_for_spec(self, spec: "spack.spec.Spec") -> str:
         _check_concrete(spec)
 
         projection = spack.projections.get_projection(self.projections, spec)
         path = spec.format_path(projection)
         return str(Path(path))
 
-    def write_spec(self, spec, path):
+    def write_spec(self, spec: "spack.spec.Spec", path: str) -> None:
         """Write a spec out to a file."""
         _check_concrete(spec)
         with open(path, "w", encoding="utf-8") as f:
@@ -138,7 +138,7 @@ def write_spec(self, spec, path):
             # the full provenance, so it's availabe if we want it later
             spec.to_json(f, hash=ht.dag_hash)
 
-    def write_host_environment(self, spec):
+    def write_host_environment(self, spec: "spack.spec.Spec") -> None:
         """The host environment is a json file with os, kernel, and spack
         versioning. We use it in the case that an analysis later needs to
         easily access this information.
@@ -148,7 +148,7 @@ def write_host_environment(self, spec):
         with open(env_file, "w", encoding="utf-8") as fd:
             sjson.dump(environ, fd)
 
-    def read_spec(self, path):
+    def read_spec(self, path: str) -> "spack.spec.Spec":
         """Read the contents of a file and parse them as a spec"""
         try:
             with open(path, encoding="utf-8") as f:
@@ -159,26 +159,28 @@ def read_spec(self, path):
                     # Too late for conversion; spec_file_path() already called.
                     spec = spack.spec.Spec.from_yaml(f)
                 else:
-                    raise SpecReadError(
-                        "Did not recognize spec file extension:" " {0}".format(extension)
-                    )
+                    raise SpecReadError(f"Did not recognize spec file extension: {extension}")
         except Exception as e:
             if spack.config.get("config:debug"):
                 raise
-            raise SpecReadError("Unable to read file: %s" % path, "Cause: " + str(e))
+            raise SpecReadError(f"Unable to read file: {path}", f"Cause: {e}")
 
         # Specs read from actual installations are always concrete
         spec._mark_concrete()
         return spec
 
-    def spec_file_path(self, spec):
+    def spec_file_path(self, spec: "spack.spec.Spec") -> str:
         """Gets full path to spec file"""
         _check_concrete(spec)
         yaml_path = os.path.join(self.metadata_path(spec), self._spec_file_name_yaml)
         json_path = os.path.join(self.metadata_path(spec), self.spec_file_name)
         return yaml_path if os.path.exists(yaml_path) else json_path
 
-    def deprecated_file_path(self, deprecated_spec, deprecator_spec=None):
+    def deprecated_file_path(
+        self,
+        deprecated_spec: "spack.spec.Spec",
+        deprecator_spec: Optional["spack.spec.Spec"] = None,
+    ) -> str:
         """Gets full path to spec file for deprecated spec
 
         If the deprecator_spec is provided, use that. Otherwise, assume
@@ -212,16 +214,16 @@ def deprecated_file_path(self, deprecated_spec, deprecator_spec=None):
 
         return yaml_path if os.path.exists(yaml_path) else json_path
 
-    def metadata_path(self, spec):
+    def metadata_path(self, spec: "spack.spec.Spec") -> str:
         return os.path.join(spec.prefix, self.metadata_dir)
 
-    def env_metadata_path(self, spec):
+    def env_metadata_path(self, spec: "spack.spec.Spec") -> str:
         return os.path.join(self.metadata_path(spec), "install_environment.json")
 
-    def build_packages_path(self, spec):
+    def build_packages_path(self, spec: "spack.spec.Spec") -> str:
         return os.path.join(self.metadata_path(spec), self.packages_dir)
 
-    def create_install_directory(self, spec):
+    def create_install_directory(self, spec: "spack.spec.Spec") -> None:
         _check_concrete(spec)
 
         # Create install directory with properly configured permissions
@@ -239,7 +241,7 @@ def create_install_directory(self, spec):
 
         self.write_spec(spec, self.spec_file_path(spec))
 
-    def ensure_installed(self, spec):
+    def ensure_installed(self, spec: "spack.spec.Spec") -> None:
         """
         Throws InconsistentInstallDirectoryError if:
         1. spec prefix does not exist
@@ -266,7 +268,7 @@ def ensure_installed(self, spec):
                 "Spec file in %s does not match hash!" % spec_file_path
             )
 
-    def path_for_spec(self, spec):
+    def path_for_spec(self, spec: "spack.spec.Spec") -> str:
         """Return absolute path from the root to a directory for the spec."""
         _check_concrete(spec)
 
@@ -277,23 +279,13 @@ def path_for_spec(self, spec):
         assert not path.startswith(self.root)
         return os.path.join(self.root, path)
 
-    def remove_install_directory(self, spec, deprecated=False):
+    def remove_install_directory(self, spec: "spack.spec.Spec", deprecated: bool = False) -> None:
         """Removes a prefix and any empty parent directories from the root.
         Raised RemoveFailedError if something goes wrong.
         """
         path = self.path_for_spec(spec)
         assert path.startswith(self.root)
 
-        # Windows readonly files cannot be removed by Python
-        # directly, change permissions before attempting to remove
-        if sys.platform == "win32":
-            kwargs = {
-                "ignore_errors": False,
-                "onerror": fs.readonly_file_handler(ignore_errors=False),
-            }
-        else:
-            kwargs = {}  # the default value for ignore_errors is false
-
         if deprecated:
             if os.path.exists(path):
                 try:
@@ -304,7 +296,16 @@ def remove_install_directory(self, spec, deprecated=False):
                     raise RemoveFailedError(spec, path, e) from e
         elif os.path.exists(path):
             try:
-                shutil.rmtree(path, **kwargs)
+                if sys.platform == "win32":
+                    # Windows readonly files cannot be removed by Python
+                    # directly, change permissions before attempting to remove
+                    shutil.rmtree(
+                        path,
+                        ignore_errors=False,
+                        onerror=fs.readonly_file_handler(ignore_errors=False),
+                    )
+                else:
+                    shutil.rmtree(path)
             except OSError as e:
                 raise RemoveFailedError(spec, path, e) from e
 

lib/spack/spack/enums.py

@@ -12,3 +12,13 @@ class InstallRecordStatus(enum.Flag):
     DEPRECATED = enum.auto()
     MISSING = enum.auto()
     ANY = INSTALLED | DEPRECATED | MISSING
+
+
+class ConfigScopePriority(enum.IntEnum):
+    """Priorities of the different kind of config scopes used by Spack"""
+
+    BUILTIN = 0
+    CONFIG_FILES = 1
+    CUSTOM = 2
+    ENVIRONMENT = 3
+    COMMAND_LINE = 4

lib/spack/spack/environment/environment.py

@@ -51,6 +51,8 @@
 from spack.spec_list import SpecList
 from spack.util.path import substitute_path_variables
 
+from ..enums import ConfigScopePriority
+
 SpecPair = spack.concretize.SpecPair
 
 #: environment variable used to indicate the active environment
@@ -387,6 +389,7 @@ def create_in_dir(
                 # dev paths in this environment to refer to their original
                 # locations.
                 _rewrite_relative_dev_paths_on_relocation(env, init_file_dir)
+                _rewrite_relative_repos_paths_on_relocation(env, init_file_dir)
 
     return env
 
@@ -403,8 +406,8 @@ def _rewrite_relative_dev_paths_on_relocation(env, init_file_dir):
             dev_path = substitute_path_variables(entry["path"])
             expanded_path = spack.util.path.canonicalize_path(dev_path, default_wd=init_file_dir)
 
-            # Skip if the expanded path is the same (e.g. when absolute)
-            if dev_path == expanded_path:
+            # Skip if the substituted and expanded path is the same (e.g. when absolute)
+            if entry["path"] == expanded_path:
                 continue
 
             tty.debug("Expanding develop path for {0} to {1}".format(name, expanded_path))
@@ -419,6 +422,34 @@ def _rewrite_relative_dev_paths_on_relocation(env, init_file_dir):
         env._re_read()
 
 
+def _rewrite_relative_repos_paths_on_relocation(env, init_file_dir):
+    """When initializing the environment from a manifest file and we plan
+    to store the environment in a different directory, we have to rewrite
+    relative repo paths to absolute ones and expand environment variables."""
+    with env:
+        repos_specs = spack.config.get("repos", default={}, scope=env.scope_name)
+        if not repos_specs:
+            return
+        for i, entry in enumerate(repos_specs):
+            repo_path = substitute_path_variables(entry)
+            expanded_path = spack.util.path.canonicalize_path(repo_path, default_wd=init_file_dir)
+
+            # Skip if the substituted and expanded path is the same (e.g. when absolute)
+            if entry == expanded_path:
+                continue
+
+            tty.debug("Expanding repo path for {0} to {1}".format(entry, expanded_path))
+
+            repos_specs[i] = expanded_path
+
+        spack.config.set("repos", repos_specs, scope=env.scope_name)
+
+        env.repos_specs = None
+        # If we changed the environment's spack.yaml scope, that will not be reflected
+        # in the manifest that we read
+        env._re_read()
+
+
 def environment_dir_from_name(name: str, exists_ok: bool = True) -> str:
     """Returns the directory associated with a named environment.
 
@@ -2392,6 +2423,8 @@ def invalidate_repository_cache(self):
 
     def __enter__(self):
         self._previous_active = _active_environment
+        if self._previous_active:
+            deactivate()
         activate(self)
         return self
 
@@ -3067,14 +3100,12 @@ def env_config_scopes(self) -> List[spack.config.ConfigScope]:
     def prepare_config_scope(self) -> None:
         """Add the manifest's scopes to the global configuration search path."""
         for scope in self.env_config_scopes:
-            spack.config.CONFIG.push_scope(scope)
-        spack.config.CONFIG.ensure_scope_ordering()
+            spack.config.CONFIG.push_scope(scope, priority=ConfigScopePriority.ENVIRONMENT)
 
     def deactivate_config_scope(self) -> None:
         """Remove any of the manifest's scopes from the global config path."""
         for scope in self.env_config_scopes:
             spack.config.CONFIG.remove_scope(scope.name)
-        spack.config.CONFIG.ensure_scope_ordering()
 
     @contextlib.contextmanager
     def use_config(self):

lib/spack/spack/filesystem_view.py

@@ -10,7 +10,7 @@
 import stat
 import sys
 import tempfile
-from typing import Callable, Dict, Optional
+from typing import Callable, Dict, List, Optional
 
 from typing_extensions import Literal
 
@@ -78,7 +78,7 @@ def view_copy(
 
     # Order of this dict is somewhat irrelevant
     prefix_to_projection = {
-        s.prefix: view.get_projection_for_spec(s)
+        str(s.prefix): view.get_projection_for_spec(s)
         for s in spec.traverse(root=True, order="breadth")
         if not s.external
     }
@@ -185,7 +185,7 @@ def __init__(
     def link(self, src: str, dst: str, spec: Optional[spack.spec.Spec] = None) -> None:
         self._link(src, dst, self, spec)
 
-    def add_specs(self, *specs, **kwargs):
+    def add_specs(self, *specs: spack.spec.Spec, **kwargs) -> None:
         """
         Add given specs to view.
 
@@ -200,19 +200,19 @@ def add_specs(self, *specs, **kwargs):
         """
         raise NotImplementedError
 
-    def add_standalone(self, spec):
+    def add_standalone(self, spec: spack.spec.Spec) -> bool:
         """
         Add (link) a standalone package into this view.
         """
         raise NotImplementedError
 
-    def check_added(self, spec):
+    def check_added(self, spec: spack.spec.Spec) -> bool:
         """
         Check if the given concrete spec is active in this view.
         """
         raise NotImplementedError
 
-    def remove_specs(self, *specs, **kwargs):
+    def remove_specs(self, *specs: spack.spec.Spec, **kwargs) -> None:
         """
         Removes given specs from view.
 
@@ -231,25 +231,25 @@ def remove_specs(self, *specs, **kwargs):
         """
         raise NotImplementedError
 
-    def remove_standalone(self, spec):
+    def remove_standalone(self, spec: spack.spec.Spec) -> None:
         """
         Remove (unlink) a standalone package from this view.
         """
         raise NotImplementedError
 
-    def get_projection_for_spec(self, spec):
+    def get_projection_for_spec(self, spec: spack.spec.Spec) -> str:
         """
         Get the projection in this view for a spec.
         """
         raise NotImplementedError
 
-    def get_all_specs(self):
+    def get_all_specs(self) -> List[spack.spec.Spec]:
         """
         Get all specs currently active in this view.
         """
         raise NotImplementedError
 
-    def get_spec(self, spec):
+    def get_spec(self, spec: spack.spec.Spec) -> Optional[spack.spec.Spec]:
         """
         Return the actual spec linked in this view (i.e. do not look it up
         in the database by name).
@@ -263,7 +263,7 @@ def get_spec(self, spec):
         """
         raise NotImplementedError
 
-    def print_status(self, *specs, **kwargs):
+    def print_status(self, *specs: spack.spec.Spec, **kwargs) -> None:
         """
         Print a short summary about the given specs, detailing whether..
             * ..they are active in the view.
@@ -694,7 +694,7 @@ def _sanity_check_view_projection(self, specs):
                 raise ConflictingSpecsError(current_spec, conflicting_spec)
             seen[metadata_dir] = current_spec
 
-    def add_specs(self, *specs: spack.spec.Spec) -> None:
+    def add_specs(self, *specs, **kwargs) -> None:
         """Link a root-to-leaf topologically ordered list of specs into the view."""
         assert all((s.concrete for s in specs))
         if len(specs) == 0:
@@ -831,7 +831,7 @@ def get_projection_for_spec(self, spec):
 #####################
 # utility functions #
 #####################
-def get_spec_from_file(filename):
+def get_spec_from_file(filename) -> Optional[spack.spec.Spec]:
     try:
         with open(filename, "r", encoding="utf-8") as f:
             return spack.spec.Spec.from_yaml(f)

lib/spack/spack/hooks/resolve_shared_libraries.py

@@ -2,198 +2,14 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import fnmatch
 import io
-import os
-import re
-from typing import Dict, List, Union
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import BaseDirectoryVisitor, visit_directory_tree
-from llnl.util.lang import stable_partition
+from llnl.util.filesystem import visit_directory_tree
 
 import spack.config
 import spack.error
-import spack.util.elf as elf
-
-#: Patterns for names of libraries that are allowed to be unresolved when *just* looking at RPATHs
-#: added by Spack. These are libraries outside of Spack's control, and assumed to be located in
-#: default search paths of the dynamic linker.
-ALLOW_UNRESOLVED = [
-    # kernel
-    "linux-vdso.so.*",
-    "libselinux.so.*",
-    # musl libc
-    "ld-musl-*.so.*",
-    # glibc
-    "ld-linux*.so.*",
-    "ld64.so.*",
-    "libanl.so.*",
-    "libc.so.*",
-    "libdl.so.*",
-    "libm.so.*",
-    "libmemusage.so.*",
-    "libmvec.so.*",
-    "libnsl.so.*",
-    "libnss_compat.so.*",
-    "libnss_db.so.*",
-    "libnss_dns.so.*",
-    "libnss_files.so.*",
-    "libnss_hesiod.so.*",
-    "libpcprofile.so.*",
-    "libpthread.so.*",
-    "libresolv.so.*",
-    "librt.so.*",
-    "libSegFault.so.*",
-    "libthread_db.so.*",
-    "libutil.so.*",
-    # gcc -- this is required even with gcc-runtime, because e.g. libstdc++ depends on libgcc_s,
-    # but the binaries we copy from the compiler don't have an $ORIGIN rpath.
-    "libasan.so.*",
-    "libatomic.so.*",
-    "libcc1.so.*",
-    "libgcc_s.so.*",
-    "libgfortran.so.*",
-    "libgomp.so.*",
-    "libitm.so.*",
-    "liblsan.so.*",
-    "libquadmath.so.*",
-    "libssp.so.*",
-    "libstdc++.so.*",
-    "libtsan.so.*",
-    "libubsan.so.*",
-    # systemd
-    "libudev.so.*",
-    # cuda driver
-    "libcuda.so.*",
-]
-
-
-def is_compatible(parent: elf.ElfFile, child: elf.ElfFile) -> bool:
-    return (
-        child.elf_hdr.e_type == elf.ELF_CONSTANTS.ET_DYN
-        and parent.is_little_endian == child.is_little_endian
-        and parent.is_64_bit == child.is_64_bit
-        and parent.elf_hdr.e_machine == child.elf_hdr.e_machine
-    )
-
-
-def candidate_matches(current_elf: elf.ElfFile, candidate_path: bytes) -> bool:
-    try:
-        with open(candidate_path, "rb") as g:
-            return is_compatible(current_elf, elf.parse_elf(g))
-    except (OSError, elf.ElfParsingError):
-        return False
-
-
-class Problem:
-    def __init__(
-        self, resolved: Dict[bytes, bytes], unresolved: List[bytes], relative_rpaths: List[bytes]
-    ) -> None:
-        self.resolved = resolved
-        self.unresolved = unresolved
-        self.relative_rpaths = relative_rpaths
-
-
-class ResolveSharedElfLibDepsVisitor(BaseDirectoryVisitor):
-    def __init__(self, allow_unresolved_patterns: List[str]) -> None:
-        self.problems: Dict[str, Problem] = {}
-        self._allow_unresolved_regex = re.compile(
-            "|".join(fnmatch.translate(x) for x in allow_unresolved_patterns)
-        )
-
-    def allow_unresolved(self, needed: bytes) -> bool:
-        try:
-            name = needed.decode("utf-8")
-        except UnicodeDecodeError:
-            return False
-        return bool(self._allow_unresolved_regex.match(name))
-
-    def visit_file(self, root: str, rel_path: str, depth: int) -> None:
-        # We work with byte strings for paths.
-        path = os.path.join(root, rel_path).encode("utf-8")
-
-        # For $ORIGIN interpolation: should not have trailing dir seperator.
-        origin = os.path.dirname(path)
-
-        # Retrieve the needed libs + rpaths.
-        try:
-            with open(path, "rb") as f:
-                parsed_elf = elf.parse_elf(f, interpreter=False, dynamic_section=True)
-        except (OSError, elf.ElfParsingError):
-            # Not dealing with an invalid ELF file.
-            return
-
-        # If there's no needed libs all is good
-        if not parsed_elf.has_needed:
-            return
-
-        # Get the needed libs and rpaths (notice: byte strings)
-        # Don't force an encoding cause paths are just a bag of bytes.
-        needed_libs = parsed_elf.dt_needed_strs
-
-        rpaths = parsed_elf.dt_rpath_str.split(b":") if parsed_elf.has_rpath else []
-
-        # We only interpolate $ORIGIN, not $LIB and $PLATFORM, they're not really
-        # supported in general. Also remove empty paths.
-        rpaths = [x.replace(b"$ORIGIN", origin) for x in rpaths if x]
-
-        # Do not allow relative rpaths (they are relative to the current working directory)
-        rpaths, relative_rpaths = stable_partition(rpaths, os.path.isabs)
-
-        # If there's a / in the needed lib, it's opened directly, otherwise it needs
-        # a search.
-        direct_libs, search_libs = stable_partition(needed_libs, lambda x: b"/" in x)
-
-        # Do not allow relative paths in direct libs (they are relative to the current working
-        # directory)
-        direct_libs, unresolved = stable_partition(direct_libs, os.path.isabs)
-
-        resolved: Dict[bytes, bytes] = {}
-
-        for lib in search_libs:
-            if self.allow_unresolved(lib):
-                continue
-            for rpath in rpaths:
-                candidate = os.path.join(rpath, lib)
-                if candidate_matches(parsed_elf, candidate):
-                    resolved[lib] = candidate
-                    break
-            else:
-                unresolved.append(lib)
-
-        # Check if directly opened libs are compatible
-        for lib in direct_libs:
-            if candidate_matches(parsed_elf, lib):
-                resolved[lib] = lib
-            else:
-                unresolved.append(lib)
-
-        if unresolved or relative_rpaths:
-            self.problems[rel_path] = Problem(resolved, unresolved, relative_rpaths)
-
-    def visit_symlinked_file(self, root: str, rel_path: str, depth: int) -> None:
-        pass
-
-    def before_visit_dir(self, root: str, rel_path: str, depth: int) -> bool:
-        # There can be binaries in .spack/test which shouldn't be checked.
-        if rel_path == ".spack":
-            return False
-        return True
-
-    def before_visit_symlinked_dir(self, root: str, rel_path: str, depth: int) -> bool:
-        return False
-
-
-class CannotLocateSharedLibraries(spack.error.SpackError):
-    pass
-
-
-def maybe_decode(byte_str: bytes) -> Union[str, bytes]:
-    try:
-        return byte_str.decode("utf-8")
-    except UnicodeDecodeError:
-        return byte_str
+import spack.verify_libraries
 
 
 def post_install(spec, explicit):
@@ -204,36 +20,23 @@ def post_install(spec, explicit):
     if policy == "ignore" or spec.external or spec.platform not in ("linux", "freebsd"):
         return
 
-    visitor = ResolveSharedElfLibDepsVisitor(
-        [*ALLOW_UNRESOLVED, *spec.package.unresolved_libraries]
+    visitor = spack.verify_libraries.ResolveSharedElfLibDepsVisitor(
+        [*spack.verify_libraries.ALLOW_UNRESOLVED, *spec.package.unresolved_libraries]
     )
     visit_directory_tree(spec.prefix, visitor)
 
-    # All good?
     if not visitor.problems:
         return
 
-    # For now just list the issues (print it in ldd style, except we don't recurse)
-    output = io.StringIO()
-    output.write("not all executables and libraries can resolve their dependencies:\n")
-    for path, problem in visitor.problems.items():
-        output.write(path)
-        output.write("\n")
-        for needed, full_path in problem.resolved.items():
-            output.write("        ")
-            if needed == full_path:
-                output.write(maybe_decode(needed))
-            else:
-                output.write(f"{maybe_decode(needed)} => {maybe_decode(full_path)}")
-            output.write("\n")
-        for not_found in problem.unresolved:
-            output.write(f"        {maybe_decode(not_found)} => not found\n")
-        for relative_rpath in problem.relative_rpaths:
-            output.write(f"        {maybe_decode(relative_rpath)} => relative rpath\n")
-
+    output = io.StringIO("not all executables and libraries can resolve their dependencies:\n")
+    visitor.write(output)
     message = output.getvalue().strip()
 
     if policy == "error":
         raise CannotLocateSharedLibraries(message)
 
     tty.warn(message)
+
+
+class CannotLocateSharedLibraries(spack.error.SpackError):
+    pass

lib/spack/spack/main.py

@@ -47,6 +47,8 @@
 import spack.util.environment
 import spack.util.lock
 
+from .enums import ConfigScopePriority
+
 #: names of profile statistics
 stat_names = pstats.Stats.sort_arg_dict_default
 
@@ -872,14 +874,19 @@ def add_command_line_scopes(
         scopes = ev.environment_path_scopes(name, path)
         if scopes is None:
             if os.path.isdir(path):  # directory with config files
-                cfg.push_scope(spack.config.DirectoryConfigScope(name, path, writable=False))
-                spack.config._add_platform_scope(cfg, name, path, writable=False)
+                cfg.push_scope(
+                    spack.config.DirectoryConfigScope(name, path, writable=False),
+                    priority=ConfigScopePriority.CUSTOM,
+                )
+                spack.config._add_platform_scope(
+                    cfg, name, path, priority=ConfigScopePriority.CUSTOM, writable=False
+                )
                 continue
             else:
                 raise spack.error.ConfigError(f"Invalid configuration scope: {path}")
 
         for scope in scopes:
-            cfg.push_scope(scope)
+            cfg.push_scope(scope, priority=ConfigScopePriority.CUSTOM)
 
 
 def _main(argv=None):
@@ -952,7 +959,9 @@ def _main(argv=None):
     # Push scopes from the command line last
     if args.config_scopes:
         add_command_line_scopes(spack.config.CONFIG, args.config_scopes)
-    spack.config.CONFIG.push_scope(spack.config.InternalConfigScope("command_line"))
+    spack.config.CONFIG.push_scope(
+        spack.config.InternalConfigScope("command_line"), priority=ConfigScopePriority.COMMAND_LINE
+    )
     setup_main_options(args)
 
     # ------------------------------------------------------------------------
@@ -998,6 +1007,7 @@ def finish_parse_and_run(parser, cmd_name, main_args, env_format_error):
     args, unknown = parser.parse_known_args(main_args.command)
     # we need to inherit verbose since the install command checks for it
     args.verbose = main_args.verbose
+    args.lines = main_args.lines
 
     # Now that we know what command this is and what its args are, determine
     # whether we can continue with a bad environment and raise if not.

lib/spack/spack/modules/common.py

@@ -330,18 +330,17 @@ class BaseConfiguration:
     default_projections = {"all": "{name}/{version}-{compiler.name}-{compiler.version}"}
 
     def __init__(self, spec: spack.spec.Spec, module_set_name: str, explicit: bool) -> None:
-        # Module where type(self) is defined
-        m = inspect.getmodule(self)
-        assert m is not None  # make mypy happy
-        self.module = m
         # Spec for which we want to generate a module file
         self.spec = spec
         self.name = module_set_name
         self.explicit = explicit
-        # Dictionary of configuration options that should be applied
-        # to the spec
+        # Dictionary of configuration options that should be applied to the spec
         self.conf = merge_config_rules(self.module.configuration(self.name), self.spec)
 
+    @property
+    def module(self):
+        return inspect.getmodule(self)
+
     @property
     def projections(self):
         """Projection from specs to module names"""
@@ -775,10 +774,6 @@ def __init__(
     ) -> None:
         self.spec = spec
 
-        # This class is meant to be derived. Get the module of the
-        # actual writer.
-        self.module = inspect.getmodule(self)
-        assert self.module is not None  # make mypy happy
         m = self.module
 
         # Create the triplet of configuration/layout/context
@@ -816,6 +811,10 @@ def __init__(
             name = type(self).__name__
             raise ModulercHeaderNotDefined(msg.format(name))
 
+    @property
+    def module(self):
+        return inspect.getmodule(self)
+
     def _get_template(self):
         """Gets the template that will be rendered for this spec."""
         # Get templates and put them in the order of importance:

lib/spack/spack/package_base.py

@@ -125,9 +125,10 @@ def windows_establish_runtime_linkage(self):
         # Spack should in general not modify things it has not installed
         # we can reasonably expect externals to have their link interface properly established
         if sys.platform == "win32" and not self.spec.external:
-            self.win_rpath.add_library_dependent(*self.win_add_library_dependent())
-            self.win_rpath.add_rpath(*self.win_add_rpath())
-            self.win_rpath.establish_link()
+            win_rpath = fsys.WindowsSimulatedRPath(self)
+            win_rpath.add_library_dependent(*self.win_add_library_dependent())
+            win_rpath.add_rpath(*self.win_add_rpath())
+            win_rpath.establish_link()
 
 
 #: Registers which are the detectable packages, by repo and package name
@@ -742,7 +743,6 @@ def __init__(self, spec):
         # Set up timing variables
         self._fetch_time = 0.0
 
-        self.win_rpath = fsys.WindowsSimulatedRPath(self)
         super().__init__()
 
     def __getitem__(self, key: str) -> "PackageBase":

lib/spack/spack/patch.py

@@ -83,6 +83,7 @@ def __init__(
         level: int,
         working_dir: str,
         reverse: bool = False,
+        ordering_key: Optional[Tuple[str, int]] = None,
     ) -> None:
         """Initialize a new Patch instance.
 
@@ -92,6 +93,7 @@ def __init__(
             level: patch level
             working_dir: relative path *within* the stage to change to
             reverse: reverse the patch
+            ordering_key: key used to ensure patches are applied in a consistent order
         """
         # validate level (must be an integer >= 0)
         if not isinstance(level, int) or not level >= 0:
@@ -105,6 +107,13 @@ def __init__(
         self.working_dir = working_dir
         self.reverse = reverse
 
+        # The ordering key is passed when executing package.py directives, and is only relevant
+        # after a solve to build concrete specs with consistently ordered patches. For concrete
+        # specs read from a file, we add patches in the order of its patches variants and the
+        # ordering_key is irrelevant. In that case, use a default value so we don't need to branch
+        # on whether ordering_key is None where it's used, just to make static analysis happy.
+        self.ordering_key: Tuple[str, int] = ordering_key or ("", 0)
+
     def apply(self, stage: "spack.stage.Stage") -> None:
         """Apply a patch to source in a stage.
 
@@ -202,9 +211,8 @@ def __init__(
             msg += "package %s.%s does not exist." % (pkg.namespace, pkg.name)
             raise ValueError(msg)
 
-        super().__init__(pkg, abs_path, level, working_dir, reverse)
+        super().__init__(pkg, abs_path, level, working_dir, reverse, ordering_key)
         self.path = abs_path
-        self.ordering_key = ordering_key
 
     @property
     def sha256(self) -> str:
@@ -266,13 +274,11 @@ def __init__(
             archive_sha256: sha256 sum of the *archive*, if the patch is compressed
                 (only required for compressed URL patches)
         """
-        super().__init__(pkg, url, level, working_dir, reverse)
+        super().__init__(pkg, url, level, working_dir, reverse, ordering_key)
 
         self.url = url
         self._stage: Optional["spack.stage.Stage"] = None
 
-        self.ordering_key = ordering_key
-
         if allowed_archive(self.url) and not archive_sha256:
             raise spack.error.PatchDirectiveError(
                 "Compressed patches require 'archive_sha256' "

lib/spack/spack/paths.py

@@ -108,6 +108,8 @@ def _get_user_cache_path():
 #: transient caches for Spack data (virtual cache, patch sha256 lookup, etc.)
 default_misc_cache_path = os.path.join(user_cache_path, "cache")
 
+#: concretization cache for Spack concretizations
+default_conc_cache_path = os.path.join(default_misc_cache_path, "concretization")
 
 # Below paths pull configuration from the host environment.
 #

lib/spack/spack/repo.py

@@ -32,6 +32,7 @@
 import llnl.util.tty as tty
 from llnl.util.filesystem import working_dir
 
+import spack
 import spack.caches
 import spack.config
 import spack.error
@@ -49,6 +50,8 @@
 #: Package modules are imported as spack.pkg.<repo-namespace>.<pkg-name>
 ROOT_PYTHON_NAMESPACE = "spack.pkg"
 
+_API_REGEX = re.compile(r"^v(\d+)\.(\d+)$")
+
 
 def python_package_for_repo(namespace):
     """Returns the full namespace of a repository, given its relative one
@@ -909,19 +912,52 @@ def __reduce__(self):
         return RepoPath.unmarshal, self.marshal()
 
 
+def _parse_package_api_version(
+    config: Dict[str, Any],
+    min_api: Tuple[int, int] = spack.min_package_api_version,
+    max_api: Tuple[int, int] = spack.package_api_version,
+) -> Tuple[int, int]:
+    api = config.get("api")
+    if api is None:
+        package_api = (1, 0)
+    else:
+        if not isinstance(api, str):
+            raise BadRepoError(f"Invalid Package API version '{api}'. Must be of the form vX.Y")
+        api_match = _API_REGEX.match(api)
+        if api_match is None:
+            raise BadRepoError(f"Invalid Package API version '{api}'. Must be of the form vX.Y")
+        package_api = (int(api_match.group(1)), int(api_match.group(2)))
+
+    if min_api <= package_api <= max_api:
+        return package_api
+
+    min_str = ".".join(str(i) for i in min_api)
+    max_str = ".".join(str(i) for i in max_api)
+    curr_str = ".".join(str(i) for i in package_api)
+    raise BadRepoError(
+        f"Package API v{curr_str} is not supported by this version of Spack ("
+        f"must be between v{min_str} and v{max_str})"
+    )
+
+
 class Repo:
     """Class representing a package repository in the filesystem.
 
-    Each package repository must have a top-level configuration file
-    called `repo.yaml`.
+    Each package repository must have a top-level configuration file called `repo.yaml`.
 
-    Currently, `repo.yaml` must define:
+    It contains the following keys:
 
     `namespace`:
         A Python namespace where the repository's packages should live.
 
     `subdirectory`:
         An optional subdirectory name where packages are placed
+
+    `api`:
+        A string of the form vX.Y that indicates the Package API version. The default is "v1.0".
+        For the repo to be compatible with the current version of Spack, the version must be
+        greater than or equal to :py:data:`spack.min_package_api_version` and less than or equal to
+        :py:data:`spack.package_api_version`.
     """
 
     def __init__(
@@ -958,7 +994,7 @@ def check(condition, msg):
             f"{os.path.join(root, repo_config_name)} must define a namespace.",
         )
 
-        self.namespace = config["namespace"]
+        self.namespace: str = config["namespace"]
         check(
             re.match(r"[a-zA-Z][a-zA-Z0-9_.]+", self.namespace),
             f"Invalid namespace '{self.namespace}' in repo '{self.root}'. "
@@ -971,12 +1007,14 @@ def check(condition, msg):
         # Keep name components around for checking prefixes.
         self._names = self.full_namespace.split(".")
 
-        packages_dir = config.get("subdirectory", packages_dir_name)
+        packages_dir: str = config.get("subdirectory", packages_dir_name)
         self.packages_path = os.path.join(self.root, packages_dir)
         check(
             os.path.isdir(self.packages_path), f"No directory '{packages_dir}' found in '{root}'"
         )
 
+        self.package_api = _parse_package_api_version(config)
+
         # Class attribute overrides by package name
         self.overrides = overrides or {}
 
@@ -1026,7 +1064,7 @@ def is_prefix(self, fullname: str) -> bool:
         parts = fullname.split(".")
         return self._names[: len(parts)] == parts
 
-    def _read_config(self) -> Dict[str, str]:
+    def _read_config(self) -> Dict[str, Any]:
         """Check for a YAML config file in this db's root directory."""
         try:
             with open(self.config_file, encoding="utf-8") as reponame_file:

lib/spack/spack/reporters/cdash.py

@@ -177,7 +177,7 @@ def build_report_for_package(self, report_dir, package, duration):
         # something went wrong pre-cdash "configure" phase b/c we have an exception and only
         # "update" was encounterd.
         # dump the report in the configure line so teams can see what the issue is
-        if len(phases_encountered) == 1 and package["exception"]:
+        if len(phases_encountered) == 1 and package.get("exception"):
             # TODO this mapping is not ideal since these are pre-configure errors
             # we need to determine if a more appropriate cdash phase can be utilized
             # for now we will add a message to the log explaining this

lib/spack/spack/schema/__init__.py

@@ -7,8 +7,7 @@
 import warnings
 
 import jsonschema
-
-import llnl.util.lang
+import jsonschema.validators
 
 from spack.error import SpecSyntaxError
 
@@ -18,59 +17,59 @@ class DeprecationMessage(typing.NamedTuple):
     error: bool
 
 
-# jsonschema is imported lazily as it is heavy to import
-# and increases the start-up time
-def _make_validator():
-    def _validate_spec(validator, is_spec, instance, schema):
-        """Check if the attributes on instance are valid specs."""
-        import spack.spec_parser
+def _validate_spec(validator, is_spec, instance, schema):
+    """Check if all additional keys are valid specs."""
+    import spack.spec_parser
 
-        if not validator.is_type(instance, "object"):
-            return
+    if not validator.is_type(instance, "object"):
+        return
 
-        for spec_str in instance:
-            try:
-                spack.spec_parser.parse(spec_str)
-            except SpecSyntaxError:
-                yield jsonschema.ValidationError(f"the key '{spec_str}' is not a valid spec")
+    properties = schema.get("properties") or {}
 
-    def _deprecated_properties(validator, deprecated, instance, schema):
-        if not (validator.is_type(instance, "object") or validator.is_type(instance, "array")):
-            return
-
-        if not deprecated:
-            return
-
-        deprecations = {
-            name: DeprecationMessage(message=x["message"], error=x["error"])
-            for x in deprecated
-            for name in x["names"]
-        }
-
-        # Get a list of the deprecated properties, return if there is none
-        issues = [entry for entry in instance if entry in deprecations]
-        if not issues:
-            return
-
-        # Process issues
-        errors = []
-        for name in issues:
-            msg = deprecations[name].message.format(name=name)
-            if deprecations[name].error:
-                errors.append(msg)
-            else:
-                warnings.warn(msg)
-
-        if errors:
-            yield jsonschema.ValidationError("\n".join(errors))
-
-    return jsonschema.validators.extend(
-        jsonschema.Draft7Validator,
-        {"validate_spec": _validate_spec, "deprecatedProperties": _deprecated_properties},
-    )
+    for spec_str in instance:
+        if spec_str in properties:
+            continue
+        try:
+            spack.spec_parser.parse(spec_str)
+        except SpecSyntaxError:
+            yield jsonschema.ValidationError(f"the key '{spec_str}' is not a valid spec")
 
 
-Validator = llnl.util.lang.Singleton(_make_validator)
+def _deprecated_properties(validator, deprecated, instance, schema):
+    if not (validator.is_type(instance, "object") or validator.is_type(instance, "array")):
+        return
+
+    if not deprecated:
+        return
+
+    deprecations = {
+        name: DeprecationMessage(message=x["message"], error=x["error"])
+        for x in deprecated
+        for name in x["names"]
+    }
+
+    # Get a list of the deprecated properties, return if there is none
+    issues = [entry for entry in instance if entry in deprecations]
+    if not issues:
+        return
+
+    # Process issues
+    errors = []
+    for name in issues:
+        msg = deprecations[name].message.format(name=name)
+        if deprecations[name].error:
+            errors.append(msg)
+        else:
+            warnings.warn(msg)
+
+    if errors:
+        yield jsonschema.ValidationError("\n".join(errors))
+
+
+Validator = jsonschema.validators.extend(
+    jsonschema.Draft7Validator,
+    {"additionalKeysAreSpecs": _validate_spec, "deprecatedProperties": _deprecated_properties},
+)
 
 
 def _append(string: str) -> bool:

lib/spack/spack/schema/config.py

@@ -58,6 +58,15 @@
                     {"type": "string"},  # deprecated
                 ]
             },
+            "concretization_cache": {
+                "type": "object",
+                "properties": {
+                    "enable": {"type": "boolean"},
+                    "url": {"type": "string"},
+                    "entry_limit": {"type": "integer", "minimum": 0},
+                    "size_limit": {"type": "integer", "minimum": 0},
+                },
+            },
             "install_hash_length": {"type": "integer", "minimum": 1},
             "install_path_scheme": {"type": "string"},  # deprecated
             "build_stage": {

lib/spack/spack/schema/modules.py

@@ -39,7 +39,7 @@
         "load": array_of_strings,
         "suffixes": {
             "type": "object",
-            "validate_spec": True,
+            "additionalKeysAreSpecs": True,
             "additionalProperties": {"type": "string"},  # key
         },
         "environment": spack.schema.environment.definition,
@@ -48,40 +48,44 @@
 
 projections_scheme = spack.schema.projections.properties["projections"]
 
-module_type_configuration: Dict = {
+common_props = {
+    "verbose": {"type": "boolean", "default": False},
+    "hash_length": {"type": "integer", "minimum": 0, "default": 7},
+    "include": array_of_strings,
+    "exclude": array_of_strings,
+    "exclude_implicits": {"type": "boolean", "default": False},
+    "defaults": array_of_strings,
+    "hide_implicits": {"type": "boolean", "default": False},
+    "naming_scheme": {"type": "string"},
+    "projections": projections_scheme,
+    "all": module_file_configuration,
+}
+
+tcl_configuration = {
     "type": "object",
     "default": {},
-    "validate_spec": True,
-    "properties": {
-        "verbose": {"type": "boolean", "default": False},
-        "hash_length": {"type": "integer", "minimum": 0, "default": 7},
-        "include": array_of_strings,
-        "exclude": array_of_strings,
-        "exclude_implicits": {"type": "boolean", "default": False},
-        "defaults": array_of_strings,
-        "hide_implicits": {"type": "boolean", "default": False},
-        "naming_scheme": {"type": "string"},
-        "projections": projections_scheme,
-        "all": module_file_configuration,
-    },
+    "additionalKeysAreSpecs": True,
+    "properties": {**common_props},
     "additionalProperties": module_file_configuration,
 }
 
-tcl_configuration = module_type_configuration.copy()
-
-lmod_configuration = module_type_configuration.copy()
-lmod_configuration["properties"].update(
-    {
+lmod_configuration = {
+    "type": "object",
+    "default": {},
+    "additionalKeysAreSpecs": True,
+    "properties": {
+        **common_props,
         "core_compilers": array_of_strings,
         "hierarchy": array_of_strings,
         "core_specs": array_of_strings,
         "filter_hierarchy_specs": {
             "type": "object",
-            "validate_spec": True,
+            "additionalKeysAreSpecs": True,
             "additionalProperties": array_of_strings,
         },
-    }
-)
+    },
+    "additionalProperties": module_file_configuration,
+}
 
 module_config_properties = {
     "use_view": {"anyOf": [{"type": "string"}, {"type": "boolean"}]},

lib/spack/spack/solver/asp.py

@@ -5,9 +5,12 @@
 import collections.abc
 import copy
 import enum
+import errno
 import functools
+import hashlib
 import io
 import itertools
+import json
 import os
 import pathlib
 import pprint
@@ -17,12 +20,25 @@
 import typing
 import warnings
 from contextlib import contextmanager
-from typing import Callable, Dict, Iterator, List, NamedTuple, Optional, Set, Tuple, Type, Union
+from typing import (
+    IO,
+    Callable,
+    Dict,
+    Iterator,
+    List,
+    NamedTuple,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+    Union,
+)
 
 import archspec.cpu
 
 import llnl.util.lang
 import llnl.util.tty as tty
+from llnl.util.filesystem import current_file_position
 from llnl.util.lang import elide_list
 
 import spack
@@ -34,21 +50,27 @@
 import spack.deptypes as dt
 import spack.environment as ev
 import spack.error
+import spack.hash_types as ht
 import spack.package_base
 import spack.package_prefs
+import spack.patch
+import spack.paths
 import spack.platforms
 import spack.repo
 import spack.solver.splicing
 import spack.spec
 import spack.store
 import spack.util.crypto
+import spack.util.hash
 import spack.util.libc
+import spack.util.module_cmd as md
 import spack.util.path
 import spack.util.timer
 import spack.variant as vt
 import spack.version as vn
 import spack.version.git_ref_lookup
 from spack import traverse
+from spack.util.file_cache import FileCache
 
 from .core import (
     AspFunction,
@@ -536,6 +558,365 @@ def format_unsolved(unsolved_specs):
                 msg += "\n\t(No candidate specs from solver)"
         return msg
 
+    def to_dict(self, test: bool = False) -> dict:
+        """Produces dict representation of Result object
+
+        Does not include anything related to unsatisfiability as we
+        are only interested in storing satisfiable results
+        """
+        serial_node_arg = (
+            lambda node_dict: f"""{{"id": "{node_dict.id}", "pkg": "{node_dict.pkg}"}}"""
+        )
+        spec_hash_type = ht.process_hash if test else ht.dag_hash
+        ret = dict()
+        ret["asp"] = self.asp
+        ret["criteria"] = self.criteria
+        ret["optimal"] = self.optimal
+        ret["warnings"] = self.warnings
+        ret["nmodels"] = self.nmodels
+        ret["abstract_specs"] = [str(x) for x in self.abstract_specs]
+        ret["satisfiable"] = self.satisfiable
+        serial_answers = []
+        for answer in self.answers:
+            serial_answer = answer[:2]
+            serial_answer_dict = {}
+            for node, spec in answer[2].items():
+                serial_answer_dict[serial_node_arg(node)] = spec.to_dict(hash=spec_hash_type)
+            serial_answer = serial_answer + (serial_answer_dict,)
+            serial_answers.append(serial_answer)
+        ret["answers"] = serial_answers
+        ret["specs_by_input"] = {}
+        input_specs = {} if not self.specs_by_input else self.specs_by_input
+        for input, spec in input_specs.items():
+            ret["specs_by_input"][str(input)] = spec.to_dict(hash=spec_hash_type)
+        return ret
+
+    @staticmethod
+    def from_dict(obj: dict):
+        """Returns Result object from compatible dictionary"""
+
+        def _dict_to_node_argument(dict):
+            id = dict["id"]
+            pkg = dict["pkg"]
+            return NodeArgument(id=id, pkg=pkg)
+
+        def _str_to_spec(spec_str):
+            return spack.spec.Spec(spec_str)
+
+        def _dict_to_spec(spec_dict):
+            loaded_spec = spack.spec.Spec.from_dict(spec_dict)
+            _ensure_external_path_if_external(loaded_spec)
+            spack.spec.Spec.ensure_no_deprecated(loaded_spec)
+            return loaded_spec
+
+        asp = obj.get("asp")
+        spec_list = obj.get("abstract_specs")
+        if not spec_list:
+            raise RuntimeError("Invalid json for concretization Result object")
+        if spec_list:
+            spec_list = [_str_to_spec(x) for x in spec_list]
+        result = Result(spec_list, asp)
+        result.criteria = obj.get("criteria")
+        result.optimal = obj.get("optimal")
+        result.warnings = obj.get("warnings")
+        result.nmodels = obj.get("nmodels")
+        result.satisfiable = obj.get("satisfiable")
+        result._unsolved_specs = []
+        answers = []
+        for answer in obj.get("answers", []):
+            loaded_answer = answer[:2]
+            answer_node_dict = {}
+            for node, spec in answer[2].items():
+                answer_node_dict[_dict_to_node_argument(json.loads(node))] = _dict_to_spec(spec)
+            loaded_answer.append(answer_node_dict)
+            answers.append(tuple(loaded_answer))
+        result.answers = answers
+        result._concrete_specs_by_input = {}
+        result._concrete_specs = []
+        for input, spec in obj.get("specs_by_input", {}).items():
+            result._concrete_specs_by_input[_str_to_spec(input)] = _dict_to_spec(spec)
+            result._concrete_specs.append(_dict_to_spec(spec))
+        return result
+
+
+class ConcretizationCache:
+    """Store for Spack concretization results and statistics
+
+    Serializes solver result objects and statistics to json and stores
+    at a given endpoint in a cache associated by the sha256 of the
+    asp problem and the involved control files.
+    """
+
+    def __init__(self, root: Union[str, None] = None):
+        if not root:
+            root = spack.config.get(
+                "config:concretization_cache:url", spack.paths.default_conc_cache_path
+            )
+        self.root = pathlib.Path(spack.util.path.canonicalize_path(root))
+        self._fc = FileCache(self.root)
+        self._cache_manifest = ".cache_manifest"
+        self._manifest_queue: List[Tuple[pathlib.Path, int]] = []
+
+    def cleanup(self):
+        """Prunes the concretization cache according to configured size and entry
+        count limits. Cleanup is done in FIFO ordering."""
+        # TODO: determine a better default
+        entry_limit = spack.config.get("config:concretization_cache:entry_limit", 1000)
+        bytes_limit = spack.config.get("config:concretization_cache:size_limit", 3e8)
+        # lock the entire buildcache as we're removing a lot of data from the
+        # manifest and cache itself
+        with self._fc.read_transaction(self._cache_manifest) as f:
+            count, cache_bytes = self._extract_cache_metadata(f)
+            if not count or not cache_bytes:
+                return
+            entry_count = int(count)
+            manifest_bytes = int(cache_bytes)
+            # move beyond the metadata entry
+            f.readline()
+            if entry_count > entry_limit and entry_limit > 0:
+                with self._fc.write_transaction(self._cache_manifest) as (old, new):
+                    # prune the oldest 10% or until we have removed 10% of
+                    # total bytes starting from oldest entry
+                    # TODO: make this configurable?
+                    prune_count = entry_limit // 10
+                    lines_to_prune = f.readlines(prune_count)
+                    for i, line in enumerate(lines_to_prune):
+                        sha, cache_entry_bytes = self._parse_manifest_entry(line)
+                        if sha and cache_entry_bytes:
+                            cache_path = self._cache_path_from_hash(sha)
+                            if self._fc.remove(cache_path):
+                                entry_count -= 1
+                                manifest_bytes -= int(cache_entry_bytes)
+                        else:
+                            tty.warn(
+                                f"Invalid concretization cache entry: '{line}' on line: {i+1}"
+                            )
+                    self._write_manifest(f, entry_count, manifest_bytes)
+
+            elif manifest_bytes > bytes_limit and bytes_limit > 0:
+                with self._fc.write_transaction(self._cache_manifest) as (old, new):
+                    # take 10% of current size off
+                    prune_amount = bytes_limit // 10
+                    total_pruned = 0
+                    i = 0
+                    while total_pruned < prune_amount:
+                        sha, manifest_cache_bytes = self._parse_manifest_entry(f.readline())
+                        if sha and manifest_cache_bytes:
+                            entry_bytes = int(manifest_cache_bytes)
+                            cache_path = self.root / sha[:2] / sha
+                            if self._safe_remove(cache_path):
+                                entry_count -= 1
+                                entry_bytes -= entry_bytes
+                                total_pruned += entry_bytes
+                        else:
+                            tty.warn(
+                                "Invalid concretization cache entry "
+                                f"'{sha} {manifest_cache_bytes}' on line: {i}"
+                            )
+                        i += 1
+                    self._write_manifest(f, entry_count, manifest_bytes)
+            for cache_dir in self.root.iterdir():
+                if cache_dir.is_dir() and not any(cache_dir.iterdir()):
+                    self._safe_remove(cache_dir)
+
+    def cache_entries(self):
+        """Generator producing cache entries"""
+        for cache_dir in self.root.iterdir():
+            # ensure component is cache entry directory
+            # not metadata file
+            if cache_dir.is_dir():
+                for cache_entry in cache_dir.iterdir():
+                    if not cache_entry.is_dir():
+                        yield cache_entry
+                    else:
+                        raise RuntimeError(
+                            "Improperly formed concretization cache. "
+                            f"Directory {cache_entry.name} is improperly located "
+                            "within the concretization cache."
+                        )
+
+    def _parse_manifest_entry(self, line):
+        """Returns parsed manifest entry lines
+        with handling for invalid reads."""
+        if line:
+            cache_values = line.strip("\n").split(" ")
+            if len(cache_values) < 2:
+                tty.warn(f"Invalid cache entry at {line}")
+                return None, None
+        return None, None
+
+    def _write_manifest(self, manifest_file, entry_count, entry_bytes):
+        """Writes new concretization cache manifest file.
+
+        Arguments:
+            manifest_file: IO stream opened for readin
+                            and writing wrapping the manifest file
+                            with cursor at calltime set to location
+                            where manifest should be truncated
+            entry_count: new total entry count
+            entry_bytes: new total entry bytes count
+
+        """
+        persisted_entries = manifest_file.readlines()
+        manifest_file.truncate(0)
+        manifest_file.write(f"{entry_count} {entry_bytes}\n")
+        manifest_file.writelines(persisted_entries)
+
+    def _results_from_cache(self, cache_entry_buffer: IO[str]) -> Union[Result, None]:
+        """Returns a Results object from the concretizer cache
+
+        Reads the cache hit and uses `Result`'s own deserializer
+        to produce a new Result object
+        """
+
+        with current_file_position(cache_entry_buffer, 0):
+            cache_str = cache_entry_buffer.read()
+            # TODO: Should this be an error if None?
+            # Same for _stats_from_cache
+            if cache_str:
+                cache_entry = json.loads(cache_str)
+                result_json = cache_entry["results"]
+                return Result.from_dict(result_json)
+        return None
+
+    def _stats_from_cache(self, cache_entry_buffer: IO[str]) -> Union[List, None]:
+        """Returns concretization statistic from the
+        concretization associated with the cache.
+
+        Deserialzes the the json representation of the
+        statistics covering the cached concretization run
+        and returns the Python data structures
+        """
+        with current_file_position(cache_entry_buffer, 0):
+            cache_str = cache_entry_buffer.read()
+            if cache_str:
+                return json.loads(cache_str)["statistics"]
+        return None
+
+    def _extract_cache_metadata(self, cache_stream: IO[str]):
+        """Extracts and returns cache entry count and bytes count from head of manifest
+        file"""
+        # make sure we're always reading from the beginning of the stream
+        # concretization cache manifest data lives at the top of the file
+        with current_file_position(cache_stream, 0):
+            return self._parse_manifest_entry(cache_stream.readline())
+
+    def _prefix_digest(self, problem: str) -> Tuple[str, str]:
+        """Return the first two characters of, and the full, sha256 of the given asp problem"""
+        prob_digest = hashlib.sha256(problem.encode()).hexdigest()
+        prefix = prob_digest[:2]
+        return prefix, prob_digest
+
+    def _cache_path_from_problem(self, problem: str) -> pathlib.Path:
+        """Returns a Path object representing the path to the cache
+        entry for the given problem"""
+        prefix, digest = self._prefix_digest(problem)
+        return pathlib.Path(prefix) / digest
+
+    def _cache_path_from_hash(self, hash: str) -> pathlib.Path:
+        """Returns a Path object representing the cache entry
+        corresponding to the given sha256 hash"""
+        return pathlib.Path(hash[:2]) / hash
+
+    def _lock_prefix_from_cache_path(self, cache_path: str):
+        """Returns the bit location corresponding to a given cache entry path
+        for file locking"""
+        return spack.util.hash.base32_prefix_bits(
+            spack.util.hash.b32_hash(cache_path), spack.util.crypto.bit_length(sys.maxsize)
+        )
+
+    def flush_manifest(self):
+        """Updates the concretization cache manifest file after a cache write operation
+        Updates the current byte count and entry counts and writes to the head of the
+        manifest file"""
+        manifest_file = self.root / self._cache_manifest
+        manifest_file.touch(exist_ok=True)
+        with open(manifest_file, "r+", encoding="utf-8") as f:
+            # check if manifest is empty
+            count, cache_bytes = self._extract_cache_metadata(f)
+            if not count or not cache_bytes:
+                # cache is unintialized
+                count = 0
+                cache_bytes = 0
+            f.seek(0, io.SEEK_END)
+            for manifest_update in self._manifest_queue:
+                entry_path, entry_bytes = manifest_update
+                count += 1
+                cache_bytes += entry_bytes
+                f.write(f"{entry_path.name} {entry_bytes}")
+            f.seek(0, io.SEEK_SET)
+            new_stats = f"{int(count)+1} {int(cache_bytes)}\n"
+            f.write(new_stats)
+
+    def _register_cache_update(self, cache_path: pathlib.Path, bytes_written: int):
+        """Adds manifest entry to update queue for later updates to the manifest"""
+        self._manifest_queue.append((cache_path, bytes_written))
+
+    def _safe_remove(self, cache_dir: pathlib.Path):
+        """Removes cache entries with handling for the case where the entry has been
+        removed already or there are multiple cache entries in a directory"""
+        try:
+            if cache_dir.is_dir():
+                cache_dir.rmdir()
+            else:
+                cache_dir.unlink()
+            return True
+        except FileNotFoundError:
+            # This is acceptable, removal is idempotent
+            pass
+        except OSError as e:
+            if e.errno == errno.ENOTEMPTY:
+                # there exists another cache entry in this directory, don't clean yet
+                pass
+        return False
+
+    def store(self, problem: str, result: Result, statistics: List, test: bool = False):
+        """Creates entry in concretization cache for problem if none exists,
+        storing the concretization Result object and statistics in the cache
+        as serialized json joined as a single file.
+
+        Hash membership is computed based on the sha256 of the provided asp
+        problem.
+        """
+        cache_path = self._cache_path_from_problem(problem)
+        if self._fc.init_entry(cache_path):
+            # if an entry for this conc hash exists already, we're don't want
+            # to overwrite, just exit
+            tty.debug(f"Cache entry {cache_path} exists, will not be overwritten")
+            return
+        with self._fc.write_transaction(cache_path) as (old, new):
+            if old:
+                # Entry for this conc hash exists already, do not overwrite
+                tty.debug(f"Cache entry {cache_path} exists, will not be overwritten")
+                return
+            cache_dict = {"results": result.to_dict(test=test), "statistics": statistics}
+            bytes_written = new.write(json.dumps(cache_dict))
+            self._register_cache_update(cache_path, bytes_written)
+
+    def fetch(self, problem: str) -> Union[Tuple[Result, List], Tuple[None, None]]:
+        """Returns the concretization cache result for a lookup based on the given problem.
+
+        Checks the concretization cache for the given problem, and either returns the
+        Python objects cached on disk representing the concretization results and statistics
+        or returns none if no cache entry was found.
+        """
+        cache_path = self._cache_path_from_problem(problem)
+        result, statistics = None, None
+        with self._fc.read_transaction(cache_path) as f:
+            if f:
+                result = self._results_from_cache(f)
+                statistics = self._stats_from_cache(f)
+        if result and statistics:
+            tty.debug(f"Concretization cache hit at {str(cache_path)}")
+            return result, statistics
+        tty.debug(f"Concretization cache miss at {str(cache_path)}")
+        return None, None
+
+
+CONC_CACHE: ConcretizationCache = llnl.util.lang.Singleton(
+    lambda: ConcretizationCache()
+)  # type: ignore
+
 
 def _normalize_packages_yaml(packages_yaml):
     normalized_yaml = copy.copy(packages_yaml)
@@ -804,6 +1185,15 @@ def solve(self, setup, specs, reuse=None, output=None, control=None, allow_depre
         if sys.platform == "win32":
             tty.debug("Ensuring basic dependencies {win-sdk, wgl} available")
             spack.bootstrap.core.ensure_winsdk_external_or_raise()
+        control_files = ["concretize.lp", "heuristic.lp", "display.lp"]
+        if not setup.concretize_everything:
+            control_files.append("when_possible.lp")
+        if using_libc_compatibility():
+            control_files.append("libc_compatibility.lp")
+        else:
+            control_files.append("os_compatibility.lp")
+        if setup.enable_splicing:
+            control_files.append("splices.lp")
 
         timer.start("setup")
         asp_problem = setup.setup(specs, reuse=reuse, allow_deprecated=allow_deprecated)
@@ -813,123 +1203,133 @@ def solve(self, setup, specs, reuse=None, output=None, control=None, allow_depre
             return Result(specs), None, None
         timer.stop("setup")
 
-        timer.start("load")
-        # Add the problem instance
-        self.control.add("base", [], asp_problem)
-        # Load the file itself
+        timer.start("cache-check")
+        timer.start("ordering")
+        # ensure deterministic output
+        problem_repr = "\n".join(sorted(asp_problem.split("\n")))
+        timer.stop("ordering")
         parent_dir = os.path.dirname(__file__)
-        self.control.load(os.path.join(parent_dir, "concretize.lp"))
-        self.control.load(os.path.join(parent_dir, "heuristic.lp"))
-        self.control.load(os.path.join(parent_dir, "display.lp"))
-        if not setup.concretize_everything:
-            self.control.load(os.path.join(parent_dir, "when_possible.lp"))
+        full_path = lambda x: os.path.join(parent_dir, x)
+        abs_control_files = [full_path(x) for x in control_files]
+        for ctrl_file in abs_control_files:
+            with open(ctrl_file, "r+", encoding="utf-8") as f:
+                problem_repr += "\n" + f.read()
 
-        # Binary compatibility is based on libc on Linux, and on the os tag elsewhere
-        if using_libc_compatibility():
-            self.control.load(os.path.join(parent_dir, "libc_compatibility.lp"))
-        else:
-            self.control.load(os.path.join(parent_dir, "os_compatibility.lp"))
-        if setup.enable_splicing:
-            self.control.load(os.path.join(parent_dir, "splices.lp"))
+        result = None
+        conc_cache_enabled = spack.config.get("config:concretization_cache:enable", True)
+        if conc_cache_enabled:
+            result, concretization_stats = CONC_CACHE.fetch(problem_repr)
 
-        timer.stop("load")
+        timer.stop("cache-check")
+        if not result:
+            timer.start("load")
+            # Add the problem instance
+            self.control.add("base", [], asp_problem)
+            # Load the files
+            [self.control.load(lp) for lp in abs_control_files]
+            timer.stop("load")
 
-        # Grounding is the first step in the solve -- it turns our facts
-        # and first-order logic rules into propositional logic.
-        timer.start("ground")
-        self.control.ground([("base", [])])
-        timer.stop("ground")
+            # Grounding is the first step in the solve -- it turns our facts
+            # and first-order logic rules into propositional logic.
+            timer.start("ground")
+            self.control.ground([("base", [])])
+            timer.stop("ground")
 
-        # With a grounded program, we can run the solve.
-        models = []  # stable models if things go well
-        cores = []  # unsatisfiable cores if they do not
+            # With a grounded program, we can run the solve.
+            models = []  # stable models if things go well
+            cores = []  # unsatisfiable cores if they do not
 
-        def on_model(model):
-            models.append((model.cost, model.symbols(shown=True, terms=True)))
+            def on_model(model):
+                models.append((model.cost, model.symbols(shown=True, terms=True)))
 
-        solve_kwargs = {
-            "assumptions": setup.assumptions,
-            "on_model": on_model,
-            "on_core": cores.append,
-        }
+            solve_kwargs = {
+                "assumptions": setup.assumptions,
+                "on_model": on_model,
+                "on_core": cores.append,
+            }
 
-        if clingo_cffi():
-            solve_kwargs["on_unsat"] = cores.append
+            if clingo_cffi():
+                solve_kwargs["on_unsat"] = cores.append
 
-        timer.start("solve")
-        time_limit = spack.config.CONFIG.get("concretizer:timeout", -1)
-        error_on_timeout = spack.config.CONFIG.get("concretizer:error_on_timeout", True)
-        # Spack uses 0 to set no time limit, clingo API uses -1
-        if time_limit == 0:
-            time_limit = -1
-        with self.control.solve(**solve_kwargs, async_=True) as handle:
-            finished = handle.wait(time_limit)
-            if not finished:
-                specs_str = ", ".join(llnl.util.lang.elide_list([str(s) for s in specs], 4))
-                header = f"Spack is taking more than {time_limit} seconds to solve for {specs_str}"
-                if error_on_timeout:
-                    raise UnsatisfiableSpecError(f"{header}, stopping concretization")
-                warnings.warn(f"{header}, using the best configuration found so far")
-                handle.cancel()
+            timer.start("solve")
+            time_limit = spack.config.CONFIG.get("concretizer:timeout", -1)
+            error_on_timeout = spack.config.CONFIG.get("concretizer:error_on_timeout", True)
+            # Spack uses 0 to set no time limit, clingo API uses -1
+            if time_limit == 0:
+                time_limit = -1
+            with self.control.solve(**solve_kwargs, async_=True) as handle:
+                finished = handle.wait(time_limit)
+                if not finished:
+                    specs_str = ", ".join(llnl.util.lang.elide_list([str(s) for s in specs], 4))
+                    header = (
+                        f"Spack is taking more than {time_limit} seconds to solve for {specs_str}"
+                    )
+                    if error_on_timeout:
+                        raise UnsatisfiableSpecError(f"{header}, stopping concretization")
+                    warnings.warn(f"{header}, using the best configuration found so far")
+                    handle.cancel()
 
-            solve_result = handle.get()
-        timer.stop("solve")
+                solve_result = handle.get()
+            timer.stop("solve")
 
-        # once done, construct the solve result
-        result = Result(specs)
-        result.satisfiable = solve_result.satisfiable
+            # once done, construct the solve result
+            result = Result(specs)
+            result.satisfiable = solve_result.satisfiable
 
-        if result.satisfiable:
-            timer.start("construct_specs")
-            # get the best model
-            builder = SpecBuilder(specs, hash_lookup=setup.reusable_and_possible)
-            min_cost, best_model = min(models)
+            if result.satisfiable:
+                timer.start("construct_specs")
+                # get the best model
+                builder = SpecBuilder(specs, hash_lookup=setup.reusable_and_possible)
+                min_cost, best_model = min(models)
 
-            # first check for errors
-            error_handler = ErrorHandler(best_model, specs)
-            error_handler.raise_if_errors()
+                # first check for errors
+                error_handler = ErrorHandler(best_model, specs)
+                error_handler.raise_if_errors()
 
-            # build specs from spec attributes in the model
-            spec_attrs = [(name, tuple(rest)) for name, *rest in extract_args(best_model, "attr")]
-            answers = builder.build_specs(spec_attrs)
+                # build specs from spec attributes in the model
+                spec_attrs = [
+                    (name, tuple(rest)) for name, *rest in extract_args(best_model, "attr")
+                ]
+                answers = builder.build_specs(spec_attrs)
 
-            # add best spec to the results
-            result.answers.append((list(min_cost), 0, answers))
+                # add best spec to the results
+                result.answers.append((list(min_cost), 0, answers))
 
-            # get optimization criteria
-            criteria_args = extract_args(best_model, "opt_criterion")
-            result.criteria = build_criteria_names(min_cost, criteria_args)
+                # get optimization criteria
+                criteria_args = extract_args(best_model, "opt_criterion")
+                result.criteria = build_criteria_names(min_cost, criteria_args)
 
-            # record the number of models the solver considered
-            result.nmodels = len(models)
+                # record the number of models the solver considered
+                result.nmodels = len(models)
 
-            # record the possible dependencies in the solve
-            result.possible_dependencies = setup.pkgs
-            timer.stop("construct_specs")
-            timer.stop()
-        elif cores:
-            result.control = self.control
-            result.cores.extend(cores)
+                # record the possible dependencies in the solve
+                result.possible_dependencies = setup.pkgs
+                timer.stop("construct_specs")
+                timer.stop()
+            elif cores:
+                result.control = self.control
+                result.cores.extend(cores)
 
+            result.raise_if_unsat()
+
+            if result.satisfiable and result.unsolved_specs and setup.concretize_everything:
+                unsolved_str = Result.format_unsolved(result.unsolved_specs)
+                raise InternalConcretizerError(
+                    "Internal Spack error: the solver completed but produced specs"
+                    " that do not satisfy the request. Please report a bug at "
+                    f"https://github.com/spack/spack/issues\n\t{unsolved_str}"
+                )
+            if conc_cache_enabled:
+                CONC_CACHE.store(problem_repr, result, self.control.statistics, test=setup.tests)
+            concretization_stats = self.control.statistics
         if output.timers:
             timer.write_tty()
             print()
 
         if output.stats:
             print("Statistics:")
-            pprint.pprint(self.control.statistics)
-
-        result.raise_if_unsat()
-
-        if result.satisfiable and result.unsolved_specs and setup.concretize_everything:
-            unsolved_str = Result.format_unsolved(result.unsolved_specs)
-            raise InternalConcretizerError(
-                "Internal Spack error: the solver completed but produced specs"
-                " that do not satisfy the request. Please report a bug at "
-                f"https://github.com/spack/spack/issues\n\t{unsolved_str}"
-            )
-
-        return result, timer, self.control.statistics
+            pprint.pprint(concretization_stats)
+        return result, timer, concretization_stats
 
 
 class ConcreteSpecsByHash(collections.abc.Mapping):
@@ -1371,7 +1771,7 @@ def effect_rules(self):
             return
 
         self.gen.h2("Imposed requirements")
-        for name in self._effect_cache:
+        for name in sorted(self._effect_cache):
             cache = self._effect_cache[name]
             for (spec_str, _), (effect_id, requirements) in cache.items():
                 self.gen.fact(fn.pkg_fact(name, fn.effect_id(effect_id)))
@@ -1424,8 +1824,8 @@ def define_variant(
 
         elif isinstance(values, vt.DisjointSetsOfValues):
             union = set()
-            for sid, s in enumerate(values.sets):
-                for value in s:
+            for sid, s in enumerate(sorted(values.sets)):
+                for value in sorted(s):
                     pkg_fact(fn.variant_value_from_disjoint_sets(vid, value, sid))
                 union.update(s)
             values = union
@@ -1606,7 +2006,7 @@ def package_provider_rules(self, pkg):
             self.gen.fact(fn.pkg_fact(pkg.name, fn.possible_provider(vpkg_name)))
 
         for when, provided in pkg.provided.items():
-            for vpkg in provided:
+            for vpkg in sorted(provided):
                 if vpkg.name not in self.possible_virtuals:
                     continue
 
@@ -1621,8 +2021,8 @@ def package_provider_rules(self, pkg):
             condition_id = self.condition(
                 when, required_name=pkg.name, msg="Virtuals are provided together"
             )
-            for set_id, virtuals_together in enumerate(sets_of_virtuals):
-                for name in virtuals_together:
+            for set_id, virtuals_together in enumerate(sorted(sets_of_virtuals)):
+                for name in sorted(virtuals_together):
                     self.gen.fact(
                         fn.pkg_fact(pkg.name, fn.provided_together(condition_id, set_id, name))
                     )
@@ -1656,13 +2056,16 @@ def track_dependencies(input_spec, requirements):
                     return requirements + [fn.attr("track_dependencies", input_spec.name)]
 
                 def dependency_holds(input_spec, requirements):
-                    return remove_node(input_spec, requirements) + [
+                    result = remove_node(input_spec, requirements) + [
                         fn.attr(
                             "dependency_holds", pkg.name, input_spec.name, dt.flag_to_string(t)
                         )
                         for t in dt.ALL_FLAGS
                         if t & depflag
                     ]
+                    if input_spec.name not in pkg.extendees:
+                        return result
+                    return result + [fn.attr("extends", pkg.name, input_spec.name)]
 
                 context = ConditionContext()
                 context.source = ConstraintOrigin.append_type_suffix(
@@ -1729,7 +2132,7 @@ def package_splice_rules(self, pkg):
                     for map in pkg.variants.values():
                         for k in map:
                             filt_match_variants.add(k)
-                    filt_match_variants = list(filt_match_variants)
+                    filt_match_variants = sorted(filt_match_variants)
                     variant_constraints = self._gen_match_variant_splice_constraints(
                         pkg, cond, spec_to_splice, hash_var, splice_node, filt_match_variants
                     )
@@ -2259,7 +2662,7 @@ def define_package_versions_and_validate_preferences(
     ):
         """Declare any versions in specs not declared in packages."""
         packages_yaml = spack.config.get("packages")
-        for pkg_name in possible_pkgs:
+        for pkg_name in sorted(possible_pkgs):
             pkg_cls = self.pkg_class(pkg_name)
 
             # All the versions from the corresponding package.py file. Since concepts
@@ -2587,7 +2990,7 @@ def define_variant_values(self):
         """
         # Tell the concretizer about possible values from specs seen in spec_clauses().
         # We might want to order these facts by pkg and name if we are debugging.
-        for pkg_name, variant_def_id, value in self.variant_values_from_specs:
+        for pkg_name, variant_def_id, value in sorted(self.variant_values_from_specs):
             try:
                 vid = self.variant_ids_by_def_id[variant_def_id]
             except KeyError:
@@ -2625,6 +3028,8 @@ def concrete_specs(self):
             # Declare as possible parts of specs that are not in package.py
             # - Add versions to possible versions
             # - Add OS to possible OS's
+
+            # is traverse deterministic?
             for dep in spec.traverse():
                 self.possible_versions[dep.name].add(dep.version)
                 if isinstance(dep.version, vn.GitVersion):
@@ -2862,7 +3267,7 @@ def define_runtime_constraints(self):
         recorder.consume_facts()
 
     def literal_specs(self, specs):
-        for spec in specs:
+        for spec in sorted(specs):
             self.gen.h2("Spec: %s" % str(spec))
             condition_id = next(self._id_counter)
             trigger_id = next(self._id_counter)
@@ -3363,7 +3768,7 @@ def consume_facts(self):
             # on the available compilers)
             self._setup.pkg_version_rules(runtime_pkg)
 
-        for imposed_spec, when_spec in self.runtime_conditions:
+        for imposed_spec, when_spec in sorted(self.runtime_conditions):
             msg = f"{when_spec} requires {imposed_spec} at runtime"
             _ = self._setup.condition(when_spec, imposed_spec=imposed_spec, msg=msg)
 
@@ -3702,11 +4107,11 @@ def build_specs(self, function_tuples):
         roots = [spec.root for spec in self._specs.values()]
         roots = dict((id(r), r) for r in roots)
         for root in roots.values():
-            spack.spec.Spec.inject_patches_variant(root)
+            _inject_patches_variant(root)
 
         # Add external paths to specs with just external modules
         for s in self._specs.values():
-            spack.spec.Spec.ensure_external_path_if_external(s)
+            _ensure_external_path_if_external(s)
 
         for s in self._specs.values():
             _develop_specs_from_env(s, ev.active_environment())
@@ -3778,6 +4183,92 @@ def execute_explicit_splices(self):
         return specs
 
 
+def _inject_patches_variant(root: spack.spec.Spec) -> None:
+    # This dictionary will store object IDs rather than Specs as keys
+    # since the Spec __hash__ will change as patches are added to them
+    spec_to_patches: Dict[int, Set[spack.patch.Patch]] = {}
+    for s in root.traverse():
+        # After concretizing, assign namespaces to anything left.
+        # Note that this doesn't count as a "change".  The repository
+        # configuration is constant throughout a spack run, and
+        # normalize and concretize evaluate Packages using Repo.get(),
+        # which respects precedence.  So, a namespace assignment isn't
+        # changing how a package name would have been interpreted and
+        # we can do it as late as possible to allow as much
+        # compatibility across repositories as possible.
+        if s.namespace is None:
+            s.namespace = spack.repo.PATH.repo_for_pkg(s.name).namespace
+
+        if s.concrete:
+            continue
+
+        # Add any patches from the package to the spec.
+        node_patches = {
+            patch
+            for cond, patch_list in spack.repo.PATH.get_pkg_class(s.fullname).patches.items()
+            if s.satisfies(cond)
+            for patch in patch_list
+        }
+        if node_patches:
+            spec_to_patches[id(s)] = node_patches
+
+    # Also record all patches required on dependencies by depends_on(..., patch=...)
+    for dspec in root.traverse_edges(deptype=dt.ALL, cover="edges", root=False):
+        if dspec.spec.concrete:
+            continue
+
+        pkg_deps = spack.repo.PATH.get_pkg_class(dspec.parent.fullname).dependencies
+
+        edge_patches: List[spack.patch.Patch] = []
+        for cond, deps_by_name in pkg_deps.items():
+            if not dspec.parent.satisfies(cond):
+                continue
+
+            dependency = deps_by_name.get(dspec.spec.name)
+            if not dependency:
+                continue
+
+            for pcond, patch_list in dependency.patches.items():
+                if dspec.spec.satisfies(pcond):
+                    edge_patches.extend(patch_list)
+
+        if edge_patches:
+            spec_to_patches.setdefault(id(dspec.spec), set()).update(edge_patches)
+
+    for spec in root.traverse():
+        if id(spec) not in spec_to_patches:
+            continue
+
+        patches = list(spec_to_patches[id(spec)])
+        variant: vt.MultiValuedVariant = spec.variants.setdefault(
+            "patches", vt.MultiValuedVariant("patches", ())
+        )
+        variant.value = tuple(p.sha256 for p in patches)
+        # FIXME: Monkey patches variant to store patches order
+        ordered_hashes = [(*p.ordering_key, p.sha256) for p in patches if p.ordering_key]
+        ordered_hashes.sort()
+        tty.debug(
+            f"Ordered hashes [{spec.name}]: "
+            + ", ".join("/".join(str(e) for e in t) for t in ordered_hashes)
+        )
+        setattr(
+            variant, "_patches_in_order_of_appearance", [sha256 for _, _, sha256 in ordered_hashes]
+        )
+
+
+def _ensure_external_path_if_external(spec: spack.spec.Spec) -> None:
+    if not spec.external_modules or spec.external_path:
+        return
+
+    # Get the path from the module the package can override the default
+    # (this is mostly needed for Cray)
+    pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)
+    package = pkg_cls(spec)
+    spec.external_path = getattr(package, "external_prefix", None) or md.path_from_modules(
+        spec.external_modules
+    )
+
+
 def _develop_specs_from_env(spec, env):
     dev_info = env.dev_specs.get(spec.name, {}) if env else {}
     if not dev_info:
@@ -4134,6 +4625,9 @@ def solve_with_stats(
         reusable_specs.extend(self.selector.reusable_specs(specs))
         setup = SpackSolverSetup(tests=tests)
         output = OutputConfiguration(timers=timers, stats=stats, out=out, setup_only=setup_only)
+
+        CONC_CACHE.flush_manifest()
+        CONC_CACHE.cleanup()
         return self.driver.solve(
             setup, specs, reuse=reusable_specs, output=output, allow_deprecated=allow_deprecated
         )
@@ -4203,6 +4697,9 @@ def solve_in_rounds(
             for spec in result.specs:
                 reusable_specs.extend(spec.traverse())
 
+        CONC_CACHE.flush_manifest()
+        CONC_CACHE.cleanup()
+
 
 class UnsatisfiableSpecError(spack.error.UnsatisfiableSpecError):
     """There was an issue with the spec that was requested (i.e. a user error)."""

lib/spack/spack/solver/concretize.lp

@@ -524,6 +524,16 @@ error(10, "'{0}' is not a valid dependency for any package in the DAG", Package)
   :- attr("node", node(ID, Package)),
      not needed(node(ID, Package)).
 
+
+% Extensions depending on each other must all extend the same node (e.g. all Python packages
+% depending on each other must depend on the same Python interpreter)
+error(100, "{0} and {1} must depend on the same {2}", ExtensionParent, ExtensionChild, ExtendeePackage)
+  :- depends_on(ExtensionParent, ExtensionChild),
+     attr("extends", ExtensionParent, ExtendeePackage),
+     depends_on(ExtensionParent, node(X, ExtendeePackage)),
+     depends_on(ExtensionChild,  node(Y, ExtendeePackage)),
+     X != Y.
+
 #defined dependency_type/2.
 
 %-----------------------------------------------------------------------------

lib/spack/spack/spec.py

@@ -52,6 +52,7 @@
 import enum
 import io
 import itertools
+import json
 import os
 import pathlib
 import platform
@@ -98,7 +99,6 @@
 import spack.traverse
 import spack.util.executable
 import spack.util.hash
-import spack.util.module_cmd as md
 import spack.util.prefix
 import spack.util.spack_json as sjson
 import spack.util.spack_yaml as syaml
@@ -798,7 +798,7 @@ def update_deptypes(self, depflag: dt.DepFlag) -> bool:
         self.depflag = new
         return True
 
-    def update_virtuals(self, virtuals: Tuple[str, ...]) -> bool:
+    def update_virtuals(self, virtuals: Iterable[str]) -> bool:
         """Update the list of provided virtuals"""
         old = self.virtuals
         self.virtuals = tuple(sorted(set(virtuals).union(self.virtuals)))
@@ -2118,20 +2118,20 @@ def cshort_spec(self):
         return self.cformat(spec_format)
 
     @property
-    def prefix(self):
+    def prefix(self) -> spack.util.prefix.Prefix:
         if not self._concrete:
-            raise spack.error.SpecError("Spec is not concrete: " + str(self))
+            raise spack.error.SpecError(f"Spec is not concrete: {self}")
 
         if self._prefix is None:
-            upstream, record = spack.store.STORE.db.query_by_spec_hash(self.dag_hash())
+            _, record = spack.store.STORE.db.query_by_spec_hash(self.dag_hash())
             if record and record.path:
-                self.prefix = record.path
+                self.set_prefix(record.path)
             else:
-                self.prefix = spack.store.STORE.layout.path_for_spec(self)
+                self.set_prefix(spack.store.STORE.layout.path_for_spec(self))
+        assert self._prefix is not None
         return self._prefix
 
-    @prefix.setter
-    def prefix(self, value):
+    def set_prefix(self, value: str) -> None:
         self._prefix = spack.util.prefix.Prefix(llnl.path.convert_to_platform_path(value))
 
     def spec_hash(self, hash):
@@ -2145,7 +2145,9 @@ def spec_hash(self, hash):
         if hash.override is not None:
             return hash.override(self)
         node_dict = self.to_node_dict(hash=hash)
-        json_text = sjson.dump(node_dict)
+        json_text = json.dumps(
+            node_dict, ensure_ascii=True, indent=None, separators=(",", ":"), sort_keys=False
+        )
         # This implements "frankenhashes", preserving the last 7 characters of the
         # original hash when splicing so that we can avoid relocation issues
         out = spack.util.hash.b32_hash(json_text)
@@ -2735,7 +2737,7 @@ def spec_and_dependency_types(
         return spec_builder(spec_dict)
 
     @staticmethod
-    def from_dict(data):
+    def from_dict(data) -> "Spec":
         """Construct a spec from JSON/YAML.
 
         Args:
@@ -2758,7 +2760,7 @@ def from_dict(data):
         return spec
 
     @staticmethod
-    def from_yaml(stream):
+    def from_yaml(stream) -> "Spec":
         """Construct a spec from YAML.
 
         Args:
@@ -2768,7 +2770,7 @@ def from_yaml(stream):
         return Spec.from_dict(data)
 
     @staticmethod
-    def from_json(stream):
+    def from_json(stream) -> "Spec":
         """Construct a spec from JSON.
 
         Args:
@@ -2778,7 +2780,7 @@ def from_json(stream):
             data = sjson.load(stream)
             return Spec.from_dict(data)
         except Exception as e:
-            raise sjson.SpackJSONError("error parsing JSON spec:", str(e)) from e
+            raise sjson.SpackJSONError("error parsing JSON spec:", e) from e
 
     @staticmethod
     def extract_json_from_clearsig(data):
@@ -2842,94 +2844,6 @@ def _patches_assigned(self):
 
         return True
 
-    @staticmethod
-    def inject_patches_variant(root):
-        # This dictionary will store object IDs rather than Specs as keys
-        # since the Spec __hash__ will change as patches are added to them
-        spec_to_patches = {}
-        for s in root.traverse():
-            # After concretizing, assign namespaces to anything left.
-            # Note that this doesn't count as a "change".  The repository
-            # configuration is constant throughout a spack run, and
-            # normalize and concretize evaluate Packages using Repo.get(),
-            # which respects precedence.  So, a namespace assignment isn't
-            # changing how a package name would have been interpreted and
-            # we can do it as late as possible to allow as much
-            # compatibility across repositories as possible.
-            if s.namespace is None:
-                s.namespace = spack.repo.PATH.repo_for_pkg(s.name).namespace
-
-            if s.concrete:
-                continue
-
-            # Add any patches from the package to the spec.
-            patches = set()
-            for cond, patch_list in spack.repo.PATH.get_pkg_class(s.fullname).patches.items():
-                if s.satisfies(cond):
-                    for patch in patch_list:
-                        patches.add(patch)
-            if patches:
-                spec_to_patches[id(s)] = patches
-
-        # Also record all patches required on dependencies by
-        # depends_on(..., patch=...)
-        for dspec in root.traverse_edges(deptype=all, cover="edges", root=False):
-            if dspec.spec.concrete:
-                continue
-
-            pkg_deps = spack.repo.PATH.get_pkg_class(dspec.parent.fullname).dependencies
-
-            patches = []
-            for cond, deps_by_name in pkg_deps.items():
-                if not dspec.parent.satisfies(cond):
-                    continue
-
-                dependency = deps_by_name.get(dspec.spec.name)
-                if not dependency:
-                    continue
-
-                for pcond, patch_list in dependency.patches.items():
-                    if dspec.spec.satisfies(pcond):
-                        patches.extend(patch_list)
-
-            if patches:
-                all_patches = spec_to_patches.setdefault(id(dspec.spec), set())
-                for patch in patches:
-                    all_patches.add(patch)
-
-        for spec in root.traverse():
-            if id(spec) not in spec_to_patches:
-                continue
-
-            patches = list(lang.dedupe(spec_to_patches[id(spec)]))
-            mvar = spec.variants.setdefault("patches", vt.MultiValuedVariant("patches", ()))
-            mvar.value = tuple(p.sha256 for p in patches)
-            # FIXME: Monkey patches mvar to store patches order
-            full_order_keys = list(tuple(p.ordering_key) + (p.sha256,) for p in patches)
-            ordered_hashes = sorted(full_order_keys)
-            tty.debug(
-                "Ordered hashes [{0}]: ".format(spec.name)
-                + ", ".join("/".join(str(e) for e in t) for t in ordered_hashes)
-            )
-            mvar._patches_in_order_of_appearance = list(t[-1] for t in ordered_hashes)
-
-    @staticmethod
-    def ensure_external_path_if_external(external_spec):
-        if external_spec.external_modules and not external_spec.external_path:
-            compiler = spack.compilers.compiler_for_spec(
-                external_spec.compiler, external_spec.architecture
-            )
-            for mod in compiler.modules:
-                md.load_module(mod)
-
-            # Get the path from the module the package can override the default
-            # (this is mostly needed for Cray)
-            pkg_cls = spack.repo.PATH.get_pkg_class(external_spec.name)
-            package = pkg_cls(external_spec)
-            external_spec.external_path = getattr(
-                package, "external_prefix", md.path_from_modules(external_spec.external_modules)
-            )
-
     @staticmethod
     def ensure_no_deprecated(root):
         """Raise if a deprecated spec is in the dag.
@@ -4701,17 +4615,6 @@ def constrain(self, other: "VariantMap") -> bool:
 
         return changed
 
-    @property
-    def concrete(self):
-        """Returns True if the spec is concrete in terms of variants.
-
-        Returns:
-            bool: True or False
-        """
-        return self.spec._concrete or all(
-            v in self for v in spack.repo.PATH.get_pkg_class(self.spec.fullname).variant_names()
-        )
-
     def copy(self) -> "VariantMap":
         clone = VariantMap(self.spec)
         for name, variant in self.items():
@@ -4838,33 +4741,51 @@ def merge_abstract_anonymous_specs(*abstract_specs: Spec):
     return merged_spec
 
 
-def reconstruct_virtuals_on_edges(spec):
-    """Reconstruct virtuals on edges. Used to read from old DB and reindex.
+def reconstruct_virtuals_on_edges(spec: Spec) -> None:
+    """Reconstruct virtuals on edges. Used to read from old DB and reindex."""
+    virtuals_needed: Dict[str, Set[str]] = {}
+    virtuals_provided: Dict[str, Set[str]] = {}
+    for edge in spec.traverse_edges(cover="edges", root=False):
+        parent_key = edge.parent.dag_hash()
+        if parent_key not in virtuals_needed:
+            # Construct which virtuals are needed by parent
+            virtuals_needed[parent_key] = set()
+            try:
+                parent_pkg = edge.parent.package
+            except Exception as e:
+                warnings.warn(
+                    f"cannot reconstruct virtual dependencies on {edge.parent.name}: {e}"
+                )
+                continue
 
-    Args:
-        spec: spec on which we want to reconstruct virtuals
-    """
-    # Collect all possible virtuals
-    possible_virtuals = set()
-    for node in spec.traverse():
-        try:
-            possible_virtuals.update(
-                {x for x in node.package.dependencies if spack.repo.PATH.is_virtual(x)}
+            virtuals_needed[parent_key].update(
+                name
+                for name, when_deps in parent_pkg.dependencies_by_name(when=True).items()
+                if spack.repo.PATH.is_virtual(name)
+                and any(edge.parent.satisfies(x) for x in when_deps)
             )
-        except Exception as e:
-            warnings.warn(f"cannot reconstruct virtual dependencies on package {node.name}: {e}")
+
+        if not virtuals_needed[parent_key]:
             continue
 
-    # Assume all incoming edges to provider are marked with virtuals=
-    for vspec in possible_virtuals:
-        try:
-            provider = spec[vspec]
-        except KeyError:
-            # Virtual not in the DAG
+        child_key = edge.spec.dag_hash()
+        if child_key not in virtuals_provided:
+            virtuals_provided[child_key] = set()
+            try:
+                child_pkg = edge.spec.package
+            except Exception as e:
+                warnings.warn(
+                    f"cannot reconstruct virtual dependencies on {edge.parent.name}: {e}"
+                )
+                continue
+            virtuals_provided[child_key].update(x.name for x in child_pkg.virtuals_provided)
+
+        if not virtuals_provided[child_key]:
             continue
 
-        for edge in provider.edges_from_dependents():
-            edge.update_virtuals([vspec])
+        virtuals_to_add = virtuals_needed[parent_key] & virtuals_provided[child_key]
+        if virtuals_to_add:
+            edge.update_virtuals(virtuals_to_add)
 
 
 class SpecfileReaderBase:

lib/spack/spack/test/bindist.py

@@ -569,7 +569,6 @@ def test_FetchCacheError_only_accepts_lists_of_errors():
 def test_FetchCacheError_pretty_printing_multiple():
     e = bindist.FetchCacheError([RuntimeError("Oops!"), TypeError("Trouble!")])
     str_e = str(e)
-    print("'" + str_e + "'")
     assert "Multiple errors" in str_e
     assert "Error 1: RuntimeError: Oops!" in str_e
     assert "Error 2: TypeError: Trouble!" in str_e

lib/spack/spack/test/build_environment.py

@@ -388,7 +388,7 @@ def test_wrapper_variables(
     root = spack.concretize.concretize_one("dt-diamond")
 
     for s in root.traverse():
-        s.prefix = "/{0}-prefix/".format(s.name)
+        s.set_prefix(f"/{s.name}-prefix/")
 
     dep_pkg = root["dt-diamond-left"].package
     dep_lib_paths = ["/test/path/to/ex1.so", "/test/path/to/subdir/ex2.so"]
@@ -396,7 +396,7 @@ def test_wrapper_variables(
     dep_libs = LibraryList(dep_lib_paths)
 
     dep2_pkg = root["dt-diamond-right"].package
-    dep2_pkg.spec.prefix = str(installation_dir_with_headers)
+    dep2_pkg.spec.set_prefix(str(installation_dir_with_headers))
 
     setattr(dep_pkg, "libs", dep_libs)
     try:
@@ -542,7 +542,7 @@ def test_build_jobs_sequential_is_sequential():
         spack.config.determine_number_of_jobs(
             parallel=False,
             max_cpus=8,
-            config=spack.config.Configuration(
+            config=spack.config.create_from(
                 spack.config.InternalConfigScope("command_line", {"config": {"build_jobs": 8}}),
                 spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 8}}),
             ),
@@ -556,7 +556,7 @@ def test_build_jobs_command_line_overrides():
         spack.config.determine_number_of_jobs(
             parallel=True,
             max_cpus=1,
-            config=spack.config.Configuration(
+            config=spack.config.create_from(
                 spack.config.InternalConfigScope("command_line", {"config": {"build_jobs": 10}}),
                 spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 1}}),
             ),
@@ -567,7 +567,7 @@ def test_build_jobs_command_line_overrides():
         spack.config.determine_number_of_jobs(
             parallel=True,
             max_cpus=100,
-            config=spack.config.Configuration(
+            config=spack.config.create_from(
                 spack.config.InternalConfigScope("command_line", {"config": {"build_jobs": 10}}),
                 spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 100}}),
             ),
@@ -581,7 +581,7 @@ def test_build_jobs_defaults():
         spack.config.determine_number_of_jobs(
             parallel=True,
             max_cpus=10,
-            config=spack.config.Configuration(
+            config=spack.config.create_from(
                 spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 1}})
             ),
         )
@@ -591,7 +591,7 @@ def test_build_jobs_defaults():
         spack.config.determine_number_of_jobs(
             parallel=True,
             max_cpus=10,
-            config=spack.config.Configuration(
+            config=spack.config.create_from(
                 spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 100}})
             ),
         )

lib/spack/spack/test/build_systems.py

@@ -403,8 +403,8 @@ def test_autoreconf_search_path_args_multiple(default_mock_concretization, tmpdi
     aclocal_fst = str(tmpdir.mkdir("fst").mkdir("share").mkdir("aclocal"))
     aclocal_snd = str(tmpdir.mkdir("snd").mkdir("share").mkdir("aclocal"))
     build_dep_one, build_dep_two = spec.dependencies(deptype="build")
-    build_dep_one.prefix = str(tmpdir.join("fst"))
-    build_dep_two.prefix = str(tmpdir.join("snd"))
+    build_dep_one.set_prefix(str(tmpdir.join("fst")))
+    build_dep_two.set_prefix(str(tmpdir.join("snd")))
     assert spack.build_systems.autotools._autoreconf_search_path_args(spec) == [
         "-I",
         aclocal_fst,
@@ -422,8 +422,8 @@ def test_autoreconf_search_path_args_skip_automake(default_mock_concretization,
     aclocal_snd = str(tmpdir.mkdir("snd").mkdir("share").mkdir("aclocal"))
     build_dep_one, build_dep_two = spec.dependencies(deptype="build")
     build_dep_one.name = "automake"
-    build_dep_one.prefix = str(tmpdir.join("fst"))
-    build_dep_two.prefix = str(tmpdir.join("snd"))
+    build_dep_one.set_prefix(str(tmpdir.join("fst")))
+    build_dep_two.set_prefix(str(tmpdir.join("snd")))
     assert spack.build_systems.autotools._autoreconf_search_path_args(spec) == ["-I", aclocal_snd]
 
 
@@ -434,7 +434,7 @@ def test_autoreconf_search_path_args_external_order(default_mock_concretization,
     aclocal_snd = str(tmpdir.mkdir("snd").mkdir("share").mkdir("aclocal"))
     build_dep_one, build_dep_two = spec.dependencies(deptype="build")
     build_dep_one.external_path = str(tmpdir.join("fst"))
-    build_dep_two.prefix = str(tmpdir.join("snd"))
+    build_dep_two.set_prefix(str(tmpdir.join("snd")))
     assert spack.build_systems.autotools._autoreconf_search_path_args(spec) == [
         "-I",
         aclocal_snd,
@@ -447,8 +447,8 @@ def test_autoreconf_search_path_skip_nonexisting(default_mock_concretization, tm
     """Skip -I flags for non-existing directories"""
     spec = default_mock_concretization("dttop")
     build_dep_one, build_dep_two = spec.dependencies(deptype="build")
-    build_dep_one.prefix = str(tmpdir.join("fst"))
-    build_dep_two.prefix = str(tmpdir.join("snd"))
+    build_dep_one.set_prefix(str(tmpdir.join("fst")))
+    build_dep_two.set_prefix(str(tmpdir.join("snd")))
     assert spack.build_systems.autotools._autoreconf_search_path_args(spec) == []
 
 

lib/spack/spack/test/cc.py

@@ -210,7 +210,6 @@ def check_args_contents(cc, args, must_contain, must_not_contain):
     """
     with set_env(SPACK_TEST_COMMAND="dump-args"):
         cc_modified_args = cc(*args, output=str).strip().split("\n")
-        print(cc_modified_args)
         for a in must_contain:
             assert a in cc_modified_args
         for a in must_not_contain:

lib/spack/spack/test/ci.py

@@ -347,7 +347,6 @@ def test_get_spec_filter_list(mutable_mock_env_path, mutable_mock_repo):
     for key, val in expectations.items():
         affected_specs = ci.get_spec_filter_list(e1, touched, dependent_traverse_depth=key)
         affected_pkg_names = set([s.name for s in affected_specs])
-        print(f"{key}: {affected_pkg_names}")
         assert affected_pkg_names == val
 
 

lib/spack/spack/test/cmd/build_env.py

@@ -12,7 +12,7 @@
 build_env = SpackCommand("build-env")
 
 
-@pytest.mark.parametrize("pkg", [("zlib",), ("zlib", "--")])
+@pytest.mark.parametrize("pkg", [("pkg-c",), ("pkg-c", "--")])
 @pytest.mark.usefixtures("config", "mock_packages", "working_env")
 def test_it_just_runs(pkg):
     build_env(*pkg)
@@ -38,7 +38,7 @@ def test_build_env_requires_a_spec(args):
 @pytest.mark.usefixtures("config", "mock_packages", "working_env")
 def test_dump(shell_as, shell, tmpdir):
     with tmpdir.as_cwd():
-        build_env("--dump", _out_file, "zlib")
+        build_env("--dump", _out_file, "pkg-c")
         with open(_out_file, encoding="utf-8") as f:
             if shell == "pwsh":
                 assert any(line.startswith("$Env:PATH") for line in f.readlines())
@@ -51,7 +51,7 @@ def test_dump(shell_as, shell, tmpdir):
 @pytest.mark.usefixtures("config", "mock_packages", "working_env")
 def test_pickle(tmpdir):
     with tmpdir.as_cwd():
-        build_env("--pickle", _out_file, "zlib")
+        build_env("--pickle", _out_file, "pkg-c")
         environment = pickle.load(open(_out_file, "rb"))
         assert isinstance(environment, dict)
         assert "PATH" in environment

lib/spack/spack/test/cmd/buildcache.py

@@ -148,7 +148,7 @@ def test_update_key_index(
     s = spack.concretize.concretize_one("libdwarf")
 
     # Install a package
-    install(s.name)
+    install("--fake", s.name)
 
     # Put installed package in the buildcache, which, because we're signing
     # it, should result in the public key getting pushed to the buildcache
@@ -178,7 +178,7 @@ def test_buildcache_autopush(tmp_path, install_mockery, mock_fetch):
     s = spack.concretize.concretize_one("libdwarf")
 
     # Install and generate build cache index
-    PackageInstaller([s.package], explicit=True).install()
+    PackageInstaller([s.package], fake=True, explicit=True).install()
 
     metadata_file = spack.binary_distribution.tarball_name(s, ".spec.json")
 
@@ -214,13 +214,11 @@ def verify_mirror_contents():
             if in_env_pkg in p:
                 found_pkg = True
 
-        if not found_pkg:
-            print("Expected to find {0} in {1}".format(in_env_pkg, dest_mirror_dir))
-            assert False
+        assert found_pkg, f"Expected to find {in_env_pkg} in {dest_mirror_dir}"
 
     # Install a package and put it in the buildcache
     s = spack.concretize.concretize_one(out_env_pkg)
-    install(s.name)
+    install("--fake", s.name)
     buildcache("push", "-u", "-f", src_mirror_url, s.name)
 
     env("create", "test")

lib/spack/spack/test/cmd/checksum.py

@@ -23,7 +23,15 @@
 
 
 @pytest.fixture
-def can_fetch_versions(monkeypatch):
+def no_add(monkeypatch):
+    def add_versions_to_pkg(pkg, version_lines, open_in_editor):
+        raise AssertionError("Should not be called")
+
+    monkeypatch.setattr(spack.cmd.checksum, "add_versions_to_pkg", add_versions_to_pkg)
+
+
+@pytest.fixture
+def can_fetch_versions(monkeypatch, no_add):
     """Fake successful version detection."""
 
     def fetch_remote_versions(pkg, concurrency):
@@ -38,10 +46,6 @@ def get_checksums_for_versions(url_by_version, package_name, **kwargs):
     def url_exists(url, curl=None):
         return True
 
-    def add_versions_to_pkg(pkg, version_lines, open_in_editor):
-        raise AssertionError("Should not be called")
-
-    monkeypatch.setattr(spack.cmd.checksum, "add_versions_to_pkg", add_versions_to_pkg)
     monkeypatch.setattr(
         spack.package_base.PackageBase, "fetch_remote_versions", fetch_remote_versions
     )
@@ -50,7 +54,7 @@ def add_versions_to_pkg(pkg, version_lines, open_in_editor):
 
 
 @pytest.fixture
-def cannot_fetch_versions(monkeypatch):
+def cannot_fetch_versions(monkeypatch, no_add):
     """Fake unsuccessful version detection."""
 
     def fetch_remote_versions(pkg, concurrency):
@@ -283,11 +287,7 @@ def test_checksum_versions(mock_packages, can_fetch_versions, monkeypatch):
     assert "version(" in output
 
 
-def test_checksum_missing_version(mock_packages, cannot_fetch_versions, monkeypatch):
-    def add_versions_to_pkg(pkg, version_lines, open_in_editor):
-        raise AssertionError("Should not be called")
-
-    monkeypatch.setattr(spack.cmd.checksum, "add_versions_to_pkg", add_versions_to_pkg)
+def test_checksum_missing_version(mock_packages, cannot_fetch_versions):
     output = spack_checksum("preferred-test", "99.99.99", fail_on_error=False)
     assert "Could not find any remote versions" in output
     output = spack_checksum("--add-to-package", "preferred-test", "99.99.99", fail_on_error=False)
@@ -295,9 +295,6 @@ def add_versions_to_pkg(pkg, version_lines, open_in_editor):
 
 
 def test_checksum_deprecated_version(mock_packages, can_fetch_versions):
-    def add_versions_to_pkg(pkg, version_lines, open_in_editor):
-        raise AssertionError("Should not be called")
-
     output = spack_checksum("deprecated-versions", "1.1.0", fail_on_error=False)
     assert "Version 1.1.0 is deprecated" in output
     output = spack_checksum(

lib/spack/spack/test/cmd/ci.py

@@ -28,6 +28,7 @@
 from spack.ci.generator_registry import generator
 from spack.cmd.ci import FAILED_CREATE_BUILDCACHE_CODE
 from spack.database import INDEX_JSON_FILE
+from spack.error import SpackError
 from spack.schema.buildcache_spec import schema as specfile_schema
 from spack.schema.database_index import schema as db_idx_schema
 from spack.spec import Spec
@@ -170,7 +171,9 @@ def test_ci_generate_with_env(ci_generate_test, tmp_path, mock_binary_index):
     url: https://my.fake.cdash
     project: Not used
     site: Nothing
-"""
+""",
+        "--artifacts-root",
+        str(tmp_path / "my_artifacts_root"),
     )
     yaml_contents = syaml.load(outputfile.read_text())
 
@@ -192,7 +195,7 @@ def test_ci_generate_with_env(ci_generate_test, tmp_path, mock_binary_index):
 
     assert "variables" in yaml_contents
     assert "SPACK_ARTIFACTS_ROOT" in yaml_contents["variables"]
-    assert yaml_contents["variables"]["SPACK_ARTIFACTS_ROOT"] == "jobs_scratch_dir"
+    assert yaml_contents["variables"]["SPACK_ARTIFACTS_ROOT"] == "my_artifacts_root"
 
 
 def test_ci_generate_with_env_missing_section(ci_generate_test, tmp_path, mock_binary_index):
@@ -1062,7 +1065,7 @@ def test_ci_rebuild_index(
             with open(tmp_path / "spec.json", "w", encoding="utf-8") as f:
                 f.write(concrete_spec.to_json(hash=ht.dag_hash))
 
-            install_cmd("--add", "-f", str(tmp_path / "spec.json"))
+            install_cmd("--fake", "--add", "-f", str(tmp_path / "spec.json"))
             buildcache_cmd("push", "-u", "-f", mirror_url, "callpath")
             ci_cmd("rebuild-index")
 
@@ -1322,44 +1325,50 @@ def test_ci_reproduce(
         env.concretize()
         env.write()
 
-        repro_dir.mkdir()
+    def fake_download_and_extract_artifacts(url, work_dir, merge_commit_test=True):
+        with working_dir(tmp_path), ev.Environment(".") as env:
+            if not os.path.exists(repro_dir):
+                repro_dir.mkdir()
 
-        job_spec = env.concrete_roots()[0]
-        with open(repro_dir / "archivefiles.json", "w", encoding="utf-8") as f:
-            f.write(job_spec.to_json(hash=ht.dag_hash))
+            job_spec = env.concrete_roots()[0]
+            with open(repro_dir / "archivefiles.json", "w", encoding="utf-8") as f:
+                f.write(job_spec.to_json(hash=ht.dag_hash))
+                artifacts_root = repro_dir / "jobs_scratch_dir"
+                pipeline_path = artifacts_root / "pipeline.yml"
 
-        artifacts_root = repro_dir / "scratch_dir"
-        pipeline_path = artifacts_root / "pipeline.yml"
-
-        ci_cmd(
-            "generate",
-            "--output-file",
-            str(pipeline_path),
-            "--artifacts-root",
-            str(artifacts_root),
-        )
-
-        job_name = gitlab_generator.get_job_name(job_spec)
-
-        with open(repro_dir / "repro.json", "w", encoding="utf-8") as f:
-            f.write(
-                json.dumps(
-                    {
-                        "job_name": job_name,
-                        "job_spec_json": "archivefiles.json",
-                        "ci_project_dir": str(repro_dir),
-                    }
+                ci_cmd(
+                    "generate",
+                    "--output-file",
+                    str(pipeline_path),
+                    "--artifacts-root",
+                    str(artifacts_root),
                 )
-            )
 
-        with open(repro_dir / "install.sh", "w", encoding="utf-8") as f:
-            f.write("#!/bin/sh\n\n#fake install\nspack install blah\n")
+                job_name = gitlab_generator.get_job_name(job_spec)
 
-        with open(repro_dir / "spack_info.txt", "w", encoding="utf-8") as f:
-            f.write(f"\nMerge {last_two_git_commits[1]} into {last_two_git_commits[0]}\n\n")
+                with open(repro_dir / "repro.json", "w", encoding="utf-8") as f:
+                    f.write(
+                        json.dumps(
+                            {
+                                "job_name": job_name,
+                                "job_spec_json": "archivefiles.json",
+                                "ci_project_dir": str(repro_dir),
+                            }
+                        )
+                    )
 
-    def fake_download_and_extract_artifacts(url, work_dir):
-        pass
+                with open(repro_dir / "install.sh", "w", encoding="utf-8") as f:
+                    f.write("#!/bin/sh\n\n#fake install\nspack install blah\n")
+
+                with open(repro_dir / "spack_info.txt", "w", encoding="utf-8") as f:
+                    if merge_commit_test:
+                        f.write(
+                            f"\nMerge {last_two_git_commits[1]} into {last_two_git_commits[0]}\n\n"
+                        )
+                    else:
+                        f.write(f"\ncommit {last_two_git_commits[1]}\n\n")
+
+            return "jobs_scratch_dir"
 
     monkeypatch.setattr(ci, "download_and_extract_artifacts", fake_download_and_extract_artifacts)
     rep_out = ci_cmd(
@@ -1375,6 +1384,64 @@ def fake_download_and_extract_artifacts(url, work_dir):
     # Make sure we tell the user where it is when not in interactive mode
     assert f"$ {repro_dir}/start.sh" in rep_out
 
+    # Ensure the correct commits are used
+    assert f"checkout_commit: {last_two_git_commits[0]}" in rep_out
+    assert f"merge_commit: {last_two_git_commits[1]}" in rep_out
+
+    # Test re-running in dirty working dir
+    with pytest.raises(SpackError, match=f"{repro_dir}"):
+        rep_out = ci_cmd(
+            "reproduce-build",
+            "https://example.com/api/v1/projects/1/jobs/2/artifacts",
+            "--working-dir",
+            str(repro_dir),
+            output=str,
+        )
+
+    # Cleanup between  tests
+    shutil.rmtree(repro_dir)
+
+    # Test --use-local-head
+    rep_out = ci_cmd(
+        "reproduce-build",
+        "https://example.com/api/v1/projects/1/jobs/2/artifacts",
+        "--use-local-head",
+        "--working-dir",
+        str(repro_dir),
+        output=str,
+    )
+
+    # Make sure we are checkout out the HEAD commit without a merge commit
+    assert "checkout_commit: HEAD" in rep_out
+    assert "merge_commit: None" in rep_out
+
+    # Test the case where the spack_info.txt is not a merge commit
+    monkeypatch.setattr(
+        ci,
+        "download_and_extract_artifacts",
+        lambda url, wd: fake_download_and_extract_artifacts(url, wd, False),
+    )
+
+    # Cleanup between  tests
+    shutil.rmtree(repro_dir)
+
+    rep_out = ci_cmd(
+        "reproduce-build",
+        "https://example.com/api/v1/projects/1/jobs/2/artifacts",
+        "--working-dir",
+        str(repro_dir),
+        output=str,
+    )
+    # Make sure the script was generated
+    assert (repro_dir / "start.sh").exists()
+
+    # Make sure we tell the user where it is when not in interactive mode
+    assert f"$ {repro_dir}/start.sh" in rep_out
+
+    # Ensure the correct commit is used (different than HEAD)
+    assert f"checkout_commit: {last_two_git_commits[1]}" in rep_out
+    assert "merge_commit: None" in rep_out
+
 
 @pytest.mark.parametrize(
     "url_in,url_out",

lib/spack/spack/test/cmd/config.py

@@ -335,7 +335,7 @@ def test_config_add_override_leaf_from_file(mutable_empty_config, tmpdir):
 
 
 def test_config_add_update_dict_from_file(mutable_empty_config, tmpdir):
-    config("add", "packages:all:compiler:[gcc]")
+    config("add", "packages:all:require:['%gcc']")
 
     # contents to add to file
     contents = """spack:
@@ -357,7 +357,7 @@ def test_config_add_update_dict_from_file(mutable_empty_config, tmpdir):
     expected = """packages:
   all:
     target: [x86_64]
-    compiler: [gcc]
+    require: ['%gcc']
 """
 
     assert expected == output
@@ -606,7 +606,6 @@ def test_config_prefer_upstream(
     packages = syaml.load(open(cfg_file, encoding="utf-8"))["packages"]
 
     # Make sure only the non-default variants are set.
-    assert packages["all"] == {"compiler": ["gcc@=10.2.1"]}
     assert packages["boost"] == {"variants": "+debug +graph", "version": ["1.63.0"]}
     assert packages["dependency-install"] == {"version": ["2.0"]}
     # Ensure that neither variant gets listed for hdf5, since they conflict

lib/spack/spack/test/cmd/deprecate.py

@@ -17,16 +17,16 @@
 
 
 def test_deprecate(mock_packages, mock_archive, mock_fetch, install_mockery):
-    install("libelf@0.8.13")
-    install("libelf@0.8.10")
+    install("--fake", "libelf@0.8.13")
+    install("--fake", "libelf@0.8.10")
 
-    all_installed = spack.store.STORE.db.query()
+    all_installed = spack.store.STORE.db.query("libelf")
     assert len(all_installed) == 2
 
     deprecate("-y", "libelf@0.8.10", "libelf@0.8.13")
 
-    non_deprecated = spack.store.STORE.db.query()
-    all_available = spack.store.STORE.db.query(installed=InstallRecordStatus.ANY)
+    non_deprecated = spack.store.STORE.db.query("libelf")
+    all_available = spack.store.STORE.db.query("libelf", installed=InstallRecordStatus.ANY)
     assert all_available == all_installed
     assert non_deprecated == spack.store.STORE.db.query("libelf@0.8.13")
 
@@ -39,24 +39,24 @@ def test_deprecate_fails_no_such_package(mock_packages, mock_archive, mock_fetch
     output = deprecate("-y", "libelf@0.8.10", "libelf@0.8.13", fail_on_error=False)
     assert "Spec 'libelf@0.8.10' matches no installed packages" in output
 
-    install("libelf@0.8.10")
+    install("--fake", "libelf@0.8.10")
 
     output = deprecate("-y", "libelf@0.8.10", "libelf@0.8.13", fail_on_error=False)
     assert "Spec 'libelf@0.8.13' matches no installed packages" in output
 
 
-def test_deprecate_install(mock_packages, mock_archive, mock_fetch, install_mockery):
-    """Tests that the ```-i`` option allows us to deprecate in favor of a spec
-    that is not yet installed."""
-    install("libelf@0.8.10")
-
-    to_deprecate = spack.store.STORE.db.query()
+def test_deprecate_install(mock_packages, mock_archive, mock_fetch, install_mockery, monkeypatch):
+    """Tests that the -i option allows us to deprecate in favor of a spec
+    that is not yet installed.
+    """
+    install("--fake", "libelf@0.8.10")
+    to_deprecate = spack.store.STORE.db.query("libelf")
     assert len(to_deprecate) == 1
 
     deprecate("-y", "-i", "libelf@0.8.10", "libelf@0.8.13")
 
-    non_deprecated = spack.store.STORE.db.query()
-    deprecated = spack.store.STORE.db.query(installed=InstallRecordStatus.DEPRECATED)
+    non_deprecated = spack.store.STORE.db.query("libelf")
+    deprecated = spack.store.STORE.db.query("libelf", installed=InstallRecordStatus.DEPRECATED)
     assert deprecated == to_deprecate
     assert len(non_deprecated) == 1
     assert non_deprecated[0].satisfies("libelf@0.8.13")
@@ -64,8 +64,8 @@ def test_deprecate_install(mock_packages, mock_archive, mock_fetch, install_mock
 
 def test_deprecate_deps(mock_packages, mock_archive, mock_fetch, install_mockery):
     """Test that the deprecate command deprecates all dependencies properly."""
-    install("libdwarf@20130729 ^libelf@0.8.13")
-    install("libdwarf@20130207 ^libelf@0.8.10")
+    install("--fake", "libdwarf@20130729 ^libelf@0.8.13")
+    install("--fake", "libdwarf@20130207 ^libelf@0.8.10")
 
     new_spec = spack.concretize.concretize_one("libdwarf@20130729^libelf@0.8.13")
     old_spec = spack.concretize.concretize_one("libdwarf@20130207^libelf@0.8.10")
@@ -81,14 +81,14 @@ def test_deprecate_deps(mock_packages, mock_archive, mock_fetch, install_mockery
     assert all_available == all_installed
     assert sorted(all_available) == sorted(deprecated + non_deprecated)
 
-    assert sorted(non_deprecated) == sorted(list(new_spec.traverse()))
-    assert sorted(deprecated) == sorted(list(old_spec.traverse()))
+    assert sorted(non_deprecated) == sorted(new_spec.traverse())
+    assert sorted(deprecated) == sorted([old_spec, old_spec["libelf"]])
 
 
 def test_uninstall_deprecated(mock_packages, mock_archive, mock_fetch, install_mockery):
     """Tests that we can still uninstall deprecated packages."""
-    install("libelf@0.8.13")
-    install("libelf@0.8.10")
+    install("--fake", "libelf@0.8.13")
+    install("--fake", "libelf@0.8.10")
 
     deprecate("-y", "libelf@0.8.10", "libelf@0.8.13")
 
@@ -104,9 +104,9 @@ def test_uninstall_deprecated(mock_packages, mock_archive, mock_fetch, install_m
 
 def test_deprecate_already_deprecated(mock_packages, mock_archive, mock_fetch, install_mockery):
     """Tests that we can re-deprecate a spec to change its deprecator."""
-    install("libelf@0.8.13")
-    install("libelf@0.8.12")
-    install("libelf@0.8.10")
+    install("--fake", "libelf@0.8.13")
+    install("--fake", "libelf@0.8.12")
+    install("--fake", "libelf@0.8.10")
 
     deprecated_spec = spack.concretize.concretize_one("libelf@0.8.10")
 
@@ -117,8 +117,8 @@ def test_deprecate_already_deprecated(mock_packages, mock_archive, mock_fetch, i
 
     deprecate("-y", "libelf@0.8.10", "libelf@0.8.13")
 
-    non_deprecated = spack.store.STORE.db.query()
-    all_available = spack.store.STORE.db.query(installed=InstallRecordStatus.ANY)
+    non_deprecated = spack.store.STORE.db.query("libelf")
+    all_available = spack.store.STORE.db.query("libelf", installed=InstallRecordStatus.ANY)
     assert len(non_deprecated) == 2
     assert len(all_available) == 3
 
@@ -129,9 +129,9 @@ def test_deprecate_already_deprecated(mock_packages, mock_archive, mock_fetch, i
 def test_deprecate_deprecator(mock_packages, mock_archive, mock_fetch, install_mockery):
     """Tests that when a deprecator spec is deprecated, its deprecatee specs
     are updated to point to the new deprecator."""
-    install("libelf@0.8.13")
-    install("libelf@0.8.12")
-    install("libelf@0.8.10")
+    install("--fake", "libelf@0.8.13")
+    install("--fake", "libelf@0.8.12")
+    install("--fake", "libelf@0.8.10")
 
     first_deprecated_spec = spack.concretize.concretize_one("libelf@0.8.10")
     second_deprecated_spec = spack.concretize.concretize_one("libelf@0.8.12")
@@ -144,8 +144,8 @@ def test_deprecate_deprecator(mock_packages, mock_archive, mock_fetch, install_m
 
     deprecate("-y", "libelf@0.8.12", "libelf@0.8.13")
 
-    non_deprecated = spack.store.STORE.db.query()
-    all_available = spack.store.STORE.db.query(installed=InstallRecordStatus.ANY)
+    non_deprecated = spack.store.STORE.db.query("libelf")
+    all_available = spack.store.STORE.db.query("libelf", installed=InstallRecordStatus.ANY)
     assert len(non_deprecated) == 1
     assert len(all_available) == 3
 
@@ -158,8 +158,8 @@ def test_deprecate_deprecator(mock_packages, mock_archive, mock_fetch, install_m
 def test_concretize_deprecated(mock_packages, mock_archive, mock_fetch, install_mockery):
     """Tests that the concretizer throws an error if we concretize to a
     deprecated spec"""
-    install("libelf@0.8.13")
-    install("libelf@0.8.10")
+    install("--fake", "libelf@0.8.13")
+    install("--fake", "libelf@0.8.10")
 
     deprecate("-y", "libelf@0.8.10", "libelf@0.8.13")
 

lib/spack/spack/test/cmd/dev_build.py

@@ -127,16 +127,15 @@ def test_dev_build_before_until(tmpdir, install_mockery):
         assert not_installed in out
 
 
-def print_spack_cc(*args):
-    # Eat arguments and print environment variable to test
-    print(os.environ.get("CC", ""))
+def _print_spack_short_spec(*args):
+    print(f"SPACK_SHORT_SPEC={os.environ['SPACK_SHORT_SPEC']}")
 
 
 def test_dev_build_drop_in(tmpdir, mock_packages, monkeypatch, install_mockery, working_env):
-    monkeypatch.setattr(os, "execvp", print_spack_cc)
+    monkeypatch.setattr(os, "execvp", _print_spack_short_spec)
     with tmpdir.as_cwd():
         output = dev_build("-b", "edit", "--drop-in", "sh", "dev-build-test-install@0.0.0")
-        assert os.path.join("lib", "spack", "env") in output
+        assert "SPACK_SHORT_SPEC=dev-build-test-install@0.0.0" in output
 
 
 def test_dev_build_fails_already_installed(tmpdir, install_mockery):

lib/spack/spack/test/cmd/diff.py

@@ -194,7 +194,7 @@ def test_diff_cmd(install_mockery, mock_fetch, mock_archive, mock_packages):
 
 def test_load_first(install_mockery, mock_fetch, mock_archive, mock_packages):
     """Test with and without the --first option"""
-    install_cmd("mpileaks")
+    install_cmd("--fake", "mpileaks")
 
     # Only one version of mpileaks will work
     diff_cmd("mpileaks", "mpileaks")

lib/spack/spack/test/cmd/env.py

@@ -1750,7 +1750,10 @@ def check_stage(spec):
         spec = spack.concretize.concretize_one(spec)
         for dep in spec.traverse():
             stage_name = f"{stage_prefix}{dep.name}-{dep.version}-{dep.dag_hash()}"
-            assert os.path.isdir(os.path.join(root, stage_name))
+            if dep.external:
+                assert not os.path.exists(os.path.join(root, stage_name))
+            else:
+                assert os.path.isdir(os.path.join(root, stage_name))
 
     check_stage("mpileaks")
     check_stage("zmpi")
@@ -3075,11 +3078,10 @@ def test_stack_view_activate_from_default(
         assert "FOOBAR=mpileaks" in shell
 
 
-def test_envvar_set_in_activate(tmpdir, mock_fetch, mock_packages, mock_archive, install_mockery):
-    filename = str(tmpdir.join("spack.yaml"))
-    with open(filename, "w", encoding="utf-8") as f:
-        f.write(
-            """\
+def test_envvar_set_in_activate(tmp_path, mock_packages, install_mockery):
+    spack_yaml = tmp_path / "spack.yaml"
+    spack_yaml.write_text(
+        """
 spack:
   specs:
     - cmake%gcc
@@ -3087,21 +3089,21 @@ def test_envvar_set_in_activate(tmpdir, mock_fetch, mock_packages, mock_archive,
     set:
       ENVAR_SET_IN_ENV_LOAD: "True"
 """
-        )
-    with tmpdir.as_cwd():
-        env("create", "test", "./spack.yaml")
-        with ev.read("test"):
-            install()
+    )
 
-        test_env = ev.read("test")
-        output = env("activate", "--sh", "test")
+    env("create", "test", str(spack_yaml))
+    with ev.read("test"):
+        install("--fake")
 
-        assert "ENVAR_SET_IN_ENV_LOAD=True" in output
+    test_env = ev.read("test")
+    output = env("activate", "--sh", "test")
 
-        with test_env:
-            with spack.util.environment.set_env(ENVAR_SET_IN_ENV_LOAD="True"):
-                output = env("deactivate", "--sh")
-                assert "unset ENVAR_SET_IN_ENV_LOAD" in output
+    assert "ENVAR_SET_IN_ENV_LOAD=True" in output
+
+    with test_env:
+        with spack.util.environment.set_env(ENVAR_SET_IN_ENV_LOAD="True"):
+            output = env("deactivate", "--sh")
+            assert "unset ENVAR_SET_IN_ENV_LOAD" in output
 
 
 def test_stack_view_no_activate_without_default(

lib/spack/spack/test/cmd/find.py

@@ -233,21 +233,27 @@ def test_display_json_deps(database, capsys):
 @pytest.mark.db
 def test_find_format(database, config):
     output = find("--format", "{name}-{^mpi.name}", "mpileaks")
-    assert set(output.strip().split("\n")) == set(
-        ["mpileaks-zmpi", "mpileaks-mpich", "mpileaks-mpich2"]
-    )
+    assert set(output.strip().split("\n")) == {
+        "mpileaks-zmpi",
+        "mpileaks-mpich",
+        "mpileaks-mpich2",
+    }
 
     output = find("--format", "{name}-{version}-{compiler.name}-{^mpi.name}", "mpileaks")
     assert "installed package" not in output
-    assert set(output.strip().split("\n")) == set(
-        ["mpileaks-2.3-gcc-zmpi", "mpileaks-2.3-gcc-mpich", "mpileaks-2.3-gcc-mpich2"]
-    )
+    assert set(output.strip().split("\n")) == {
+        "mpileaks-2.3-gcc-zmpi",
+        "mpileaks-2.3-gcc-mpich",
+        "mpileaks-2.3-gcc-mpich2",
+    }
 
     output = find("--format", "{name}-{^mpi.name}-{hash:7}", "mpileaks")
     elements = output.strip().split("\n")
-    assert set(e[:-7] for e in elements) == set(
-        ["mpileaks-zmpi-", "mpileaks-mpich-", "mpileaks-mpich2-"]
-    )
+    assert set(e[:-7] for e in elements) == {
+        "mpileaks-zmpi-",
+        "mpileaks-mpich-",
+        "mpileaks-mpich2-",
+    }
 
     # hashes are in base32
     for e in elements:
@@ -348,7 +354,7 @@ def test_find_prefix_in_env(
     """Test `find` formats requiring concrete specs work in environments."""
     env("create", "test")
     with ev.read("test"):
-        install("--add", "mpileaks")
+        install("--fake", "--add", "mpileaks")
         find("-p")
         find("-l")
         find("-L")

lib/spack/spack/test/cmd/install.py

@@ -898,7 +898,6 @@ def test_cdash_configure_warning(tmpdir, mock_fetch, install_mockery, capfd):
         specfile = "./spec.json"
         with open(specfile, "w", encoding="utf-8") as f:
             f.write(spec.to_json())
-        print(spec.to_json())
         install("--log-file=cdash_reports", "--log-format=cdash", specfile)
         # Verify Configure.xml exists with expected contents.
         report_dir = tmpdir.join("cdash_reports")

lib/spack/spack/test/cmd/pkg.py

@@ -42,7 +42,7 @@ def mock_pkg_git_repo(git, tmp_path_factory):
     repo_dir = root_dir / "builtin.mock"
     shutil.copytree(spack.paths.mock_packages_path, str(repo_dir))
 
-    repo_cache = spack.util.file_cache.FileCache(str(root_dir / "cache"))
+    repo_cache = spack.util.file_cache.FileCache(root_dir / "cache")
     mock_repo = spack.repo.RepoPath(str(repo_dir), cache=repo_cache)
     mock_repo_packages = mock_repo.repos[0].packages_path
 

lib/spack/spack/test/cmd/repo.py

@@ -5,9 +5,13 @@
 
 import pytest
 
+import spack.config
+import spack.environment as ev
 import spack.main
+from spack.main import SpackCommand
 
 repo = spack.main.SpackCommand("repo")
+env = SpackCommand("env")
 
 
 def test_help_option():
@@ -33,3 +37,33 @@ def test_create_add_list_remove(mutable_config, tmpdir):
     repo("remove", "--scope=site", str(tmpdir))
     output = repo("list", "--scope=site", output=str)
     assert "mockrepo" not in output
+
+
+def test_env_repo_path_vars_substitution(
+    tmpdir, install_mockery, mutable_mock_env_path, monkeypatch
+):
+    """Test Spack correctly substitues repo paths with environment variables when creating an
+    environment from a manifest file."""
+
+    monkeypatch.setenv("CUSTOM_REPO_PATH", ".")
+
+    # setup environment from spack.yaml
+    envdir = tmpdir.mkdir("env")
+    with envdir.as_cwd():
+        with open("spack.yaml", "w", encoding="utf-8") as f:
+            f.write(
+                """\
+spack:
+  specs: []
+
+  repos:
+    - $CUSTOM_REPO_PATH
+"""
+            )
+        # creating env from manifest file
+        env("create", "test", "./spack.yaml")
+        # check that repo path was correctly substituted with the environment variable
+        current_dir = os.getcwd()
+        with ev.read("test") as newenv:
+            repos_specs = spack.config.get("repos", default={}, scope=newenv.scope_name)
+            assert current_dir in repos_specs

lib/spack/spack/test/cmd/unit_test.py

@@ -50,7 +50,7 @@ def test_list_long(capsys):
 def test_list_long_with_pytest_arg(capsys):
     with capsys.disabled():
         output = spack_test("--list-long", cmd_test_py)
-    print(output)
+
     assert "unit_test.py::\n" in output
     assert "test_list" in output
     assert "test_list_with_pytest_arg" in output

lib/spack/spack/test/cmd/verify.py

@@ -4,19 +4,31 @@
 
 """Tests for the `spack verify` command"""
 import os
+import platform
+
+import pytest
 
 import llnl.util.filesystem as fs
 
+import spack.cmd.verify
 import spack.concretize
+import spack.installer
 import spack.store
+import spack.util.executable
 import spack.util.spack_json as sjson
 import spack.verify
-from spack.main import SpackCommand
+from spack.main import SpackCommand, SpackCommandError
 
 verify = SpackCommand("verify")
 install = SpackCommand("install")
 
 
+def skip_unless_linux(f):
+    return pytest.mark.skipif(
+        str(platform.system()) != "Linux", reason="only tested on linux for now"
+    )(f)
+
+
 def test_single_file_verify_cmd(tmpdir):
     # Test the verify command interface to verifying a single file.
     filedir = os.path.join(str(tmpdir), "a", "b", "c", "d")
@@ -36,15 +48,14 @@ def test_single_file_verify_cmd(tmpdir):
     with open(manifest_file, "w", encoding="utf-8") as f:
         sjson.dump({filepath: data}, f)
 
-    results = verify("-f", filepath, fail_on_error=False)
-    print(results)
+    results = verify("manifest", "-f", filepath, fail_on_error=False)
     assert not results
 
     os.utime(filepath, (0, 0))
     with open(filepath, "w", encoding="utf-8") as f:
         f.write("I changed.")
 
-    results = verify("-f", filepath, fail_on_error=False)
+    results = verify("manifest", "-f", filepath, fail_on_error=False)
 
     expected = ["hash"]
     mtime = os.stat(filepath).st_mtime
@@ -55,7 +66,7 @@ def test_single_file_verify_cmd(tmpdir):
     assert filepath in results
     assert all(x in results for x in expected)
 
-    results = verify("-fj", filepath, fail_on_error=False)
+    results = verify("manifest", "-fj", filepath, fail_on_error=False)
     res = sjson.load(results)
     assert len(res) == 1
     errors = res.pop(filepath)
@@ -69,18 +80,68 @@ def test_single_spec_verify_cmd(tmpdir, mock_packages, mock_archive, mock_fetch,
     prefix = s.prefix
     hash = s.dag_hash()
 
-    results = verify("/%s" % hash, fail_on_error=False)
+    results = verify("manifest", "/%s" % hash, fail_on_error=False)
     assert not results
 
     new_file = os.path.join(prefix, "new_file_for_verify_test")
     with open(new_file, "w", encoding="utf-8") as f:
         f.write("New file")
 
-    results = verify("/%s" % hash, fail_on_error=False)
+    results = verify("manifest", "/%s" % hash, fail_on_error=False)
     assert new_file in results
     assert "added" in results
 
-    results = verify("-j", "/%s" % hash, fail_on_error=False)
+    results = verify("manifest", "-j", "/%s" % hash, fail_on_error=False)
     res = sjson.load(results)
     assert len(res) == 1
     assert res[new_file] == ["added"]
+
+
+@pytest.mark.requires_executables("gcc")
+@skip_unless_linux
+def test_libraries(tmp_path, install_mockery, mock_fetch):
+    gcc = spack.util.executable.which("gcc", required=True)
+    s = spack.concretize.concretize_one("libelf")
+    spack.installer.PackageInstaller([s.package]).install()
+    os.mkdir(s.prefix.bin)
+
+    # There are no ELF files so the verification should pass
+    verify("libraries", f"/{s.dag_hash()}")
+
+    # Now put main_with_rpath linking to libf.so inside the prefix and verify again. This should
+    # work because libf.so can be located in the rpath.
+    (tmp_path / "f.c").write_text("void f(void){return;}")
+    (tmp_path / "main.c").write_text("void f(void); int main(void){f();return 0;}")
+
+    gcc("-shared", "-fPIC", "-o", str(tmp_path / "libf.so"), str(tmp_path / "f.c"))
+    gcc(
+        "-o",
+        str(s.prefix.bin.main_with_rpath),
+        str(tmp_path / "main.c"),
+        "-L",
+        str(tmp_path),
+        f"-Wl,-rpath,{tmp_path}",
+        "-lf",
+    )
+    verify("libraries", f"/{s.dag_hash()}")
+
+    # Now put main_without_rpath linking to libf.so inside the prefix and verify again. This should
+    # fail because libf.so cannot be located in the rpath.
+    gcc(
+        "-o",
+        str(s.prefix.bin.main_without_rpath),
+        str(tmp_path / "main.c"),
+        "-L",
+        str(tmp_path),
+        "-lf",
+    )
+
+    with pytest.raises(SpackCommandError):
+        verify("libraries", f"/{s.dag_hash()}")
+
+    # Check the error message
+    msg = spack.cmd.verify._verify_libraries(s, [])
+    assert msg is not None and "libf.so => not found" in msg
+
+    # And check that we can make it pass by ignoring it.
+    assert spack.cmd.verify._verify_libraries(s, ["libf.so"]) is None

lib/spack/spack/test/concretization/core.py

@@ -2018,7 +2018,6 @@ def test_git_ref_version_is_equivalent_to_specified_version(self, git_ref):
         s = Spec("develop-branch-version@git.%s=develop" % git_ref)
         c = spack.concretize.concretize_one(s)
         assert git_ref in str(c)
-        print(str(c))
         assert s.satisfies("@develop")
         assert s.satisfies("@0.1:")
 
@@ -2888,6 +2887,23 @@ def test_specifying_different_versions_build_deps(self):
         assert any(x.satisfies(hdf5_str) for x in result.specs)
         assert any(x.satisfies(pinned_str) for x in result.specs)
 
+    @pytest.mark.regression("44289")
+    def test_all_extensions_depend_on_same_extendee(self):
+        """Tests that we don't reuse dependencies that bring in a different extendee"""
+        setuptools = spack.concretize.concretize_one("py-setuptools ^python@3.10")
+
+        solver = spack.solver.asp.Solver()
+        setup = spack.solver.asp.SpackSolverSetup()
+        result, _, _ = solver.driver.solve(
+            setup, [Spec("py-floating ^python@3.11")], reuse=list(setuptools.traverse())
+        )
+        assert len(result.specs) == 1
+
+        floating = result.specs[0]
+        assert all(setuptools.dag_hash() != x.dag_hash() for x in floating.traverse())
+        pythons = [x for x in floating.traverse() if x.name == "python"]
+        assert len(pythons) == 1 and pythons[0].satisfies("@3.11")
+
 
 @pytest.mark.parametrize(
     "v_str,v_opts,checksummed",
@@ -3238,3 +3254,54 @@ def test_spec_unification(unify, mutable_config, mock_packages):
     maybe_fails = pytest.raises if unify is True else llnl.util.lang.nullcontext
     with maybe_fails(spack.solver.asp.UnsatisfiableSpecError):
         _ = spack.cmd.parse_specs([a_restricted, b], concretize=True)
+
+
+def test_concretization_cache_roundtrip(use_concretization_cache, monkeypatch, mutable_config):
+    """Tests whether we can write the results of a clingo solve to the cache
+    and load the same spec request from the cache to produce identical specs"""
+    # Force determinism:
+    # Solver setup is normally non-deterministic due to non-determinism in
+    # asp solver setup logic generation. The only other inputs to the cache keys are
+    # the .lp files, which are invariant over the course of this test.
+    # This method forces the same setup to be produced for the same specs
+    # which gives us a guarantee of cache hits, as it removes the only
+    # element of non deterministic solver setup for the same spec
+    # Basically just a quick and dirty memoization
+    solver_setup = spack.solver.asp.SpackSolverSetup.setup
+
+    def _setup(self, specs, *, reuse=None, allow_deprecated=False):
+        if not getattr(_setup, "cache_setup", None):
+            cache_setup = solver_setup(self, specs, reuse=reuse, allow_deprecated=allow_deprecated)
+            setattr(_setup, "cache_setup", cache_setup)
+        return getattr(_setup, "cache_setup")
+
+    # monkeypatch our forced determinism setup method into solver setup
+    monkeypatch.setattr(spack.solver.asp.SpackSolverSetup, "setup", _setup)
+
+    assert spack.config.get("config:concretization_cache:enable")
+
+    # run one standard concretization to populate the cache and the setup method
+    # memoization
+    h = spack.concretize.concretize_one("hdf5")
+
+    # due to our forced determinism above, we should not be observing
+    # cache misses, assert that we're not storing any new cache entries
+    def _ensure_no_store(self, problem: str, result, statistics, test=False):
+        # always throw, we never want to reach this code path
+        assert False, "Concretization cache hit expected"
+
+    # Assert that we're actually hitting the cache
+    cache_fetch = spack.solver.asp.ConcretizationCache.fetch
+
+    def _ensure_cache_hits(self, problem: str):
+        result, statistics = cache_fetch(self, problem)
+        assert result, "Expected successful concretization cache hit"
+        assert statistics, "Expected statistics to be non null on cache hit"
+        return result, statistics
+
+    monkeypatch.setattr(spack.solver.asp.ConcretizationCache, "store", _ensure_no_store)
+    monkeypatch.setattr(spack.solver.asp.ConcretizationCache, "fetch", _ensure_cache_hits)
+    # ensure subsequent concretizations of the same spec produce the same spec
+    # object
+    for _ in range(5):
+        assert h == spack.concretize.concretize_one("hdf5")

lib/spack/spack/test/config.py

@@ -33,6 +33,8 @@
 import spack.util.path as spack_path
 import spack.util.spack_yaml as syaml
 
+from ..enums import ConfigScopePriority
+
 # sample config data
 config_low = {
     "config": {
@@ -710,10 +712,7 @@ def assert_marked(obj):
 
 
 def test_internal_config_from_data():
-    config = spack.config.Configuration()
-
-    # add an internal config initialized from an inline dict
-    config.push_scope(
+    config = spack.config.create_from(
         spack.config.InternalConfigScope(
             "_builtin", {"config": {"verify_ssl": False, "build_jobs": 6}}
         )
@@ -1445,7 +1444,7 @@ def test_config_path_dsl(path, it_should_work, expected_parsed):
 
 
 @pytest.mark.regression("48254")
-def test_env_activation_preserves_config_scopes(mutable_mock_env_path):
+def test_env_activation_preserves_command_line_scope(mutable_mock_env_path):
     """Check that the "command_line" scope remains the highest priority scope, when we activate,
     or deactivate, environments.
     """
@@ -1469,3 +1468,51 @@ def test_env_activation_preserves_config_scopes(mutable_mock_env_path):
         assert spack.config.CONFIG.highest() == expected_cl_scope
 
     assert spack.config.CONFIG.highest() == expected_cl_scope
+
+
+@pytest.mark.regression("48414")
+@pytest.mark.regression("49188")
+def test_env_activation_preserves_config_scopes(mutable_mock_env_path):
+    """Check that the priority of scopes is respected when merging configuration files."""
+    custom_scope = spack.config.InternalConfigScope("custom_scope")
+    spack.config.CONFIG.push_scope(custom_scope, priority=ConfigScopePriority.CUSTOM)
+    expected_scopes_without_env = ["custom_scope", "command_line"]
+    expected_scopes_with_first_env = ["custom_scope", "env:test", "command_line"]
+    expected_scopes_with_second_env = ["custom_scope", "env:test-2", "command_line"]
+
+    def highest_priority_scopes(config, *, nscopes):
+        return list(config.scopes)[-nscopes:]
+
+    assert highest_priority_scopes(spack.config.CONFIG, nscopes=2) == expected_scopes_without_env
+    # Creating an environment pushes a new scope
+    ev.create("test")
+    with ev.read("test"):
+        assert (
+            highest_priority_scopes(spack.config.CONFIG, nscopes=3)
+            == expected_scopes_with_first_env
+        )
+
+        # No active environment pops the scope
+        with ev.no_active_environment():
+            assert (
+                highest_priority_scopes(spack.config.CONFIG, nscopes=2)
+                == expected_scopes_without_env
+            )
+        assert (
+            highest_priority_scopes(spack.config.CONFIG, nscopes=3)
+            == expected_scopes_with_first_env
+        )
+
+        # Switch the environment to another one
+        ev.create("test-2")
+        with ev.read("test-2"):
+            assert (
+                highest_priority_scopes(spack.config.CONFIG, nscopes=3)
+                == expected_scopes_with_second_env
+            )
+        assert (
+            highest_priority_scopes(spack.config.CONFIG, nscopes=3)
+            == expected_scopes_with_first_env
+        )
+
+    assert highest_priority_scopes(spack.config.CONFIG, nscopes=2) == expected_scopes_without_env

lib/spack/spack/test/conftest.py

@@ -66,6 +66,8 @@
 from spack.main import SpackCommand
 from spack.util.pattern import Bunch
 
+from ..enums import ConfigScopePriority
+
 mirror_cmd = SpackCommand("mirror")
 
 
@@ -339,6 +341,16 @@ def pytest_collection_modifyitems(config, items):
             item.add_marker(skip_as_slow)
 
 
+@pytest.fixture(scope="function")
+def use_concretization_cache(mutable_config, tmpdir):
+    """Enables the use of the concretization cache"""
+    spack.config.set("config:concretization_cache:enable", True)
+    # ensure we have an isolated concretization cache
+    new_conc_cache_loc = str(tmpdir.mkdir("concretization"))
+    spack.config.set("config:concretization_cache:path", new_conc_cache_loc)
+    yield
+
+
 #
 # These fixtures are applied to all tests
 #
@@ -723,11 +735,23 @@ def configuration_dir(tmpdir_factory, linux_os):
 def _create_mock_configuration_scopes(configuration_dir):
     """Create the configuration scopes used in `config` and `mutable_config`."""
     return [
-        spack.config.InternalConfigScope("_builtin", spack.config.CONFIG_DEFAULTS),
-        spack.config.DirectoryConfigScope("site", str(configuration_dir.join("site"))),
-        spack.config.DirectoryConfigScope("system", str(configuration_dir.join("system"))),
-        spack.config.DirectoryConfigScope("user", str(configuration_dir.join("user"))),
-        spack.config.InternalConfigScope("command_line"),
+        (
+            ConfigScopePriority.BUILTIN,
+            spack.config.InternalConfigScope("_builtin", spack.config.CONFIG_DEFAULTS),
+        ),
+        (
+            ConfigScopePriority.CONFIG_FILES,
+            spack.config.DirectoryConfigScope("site", str(configuration_dir.join("site"))),
+        ),
+        (
+            ConfigScopePriority.CONFIG_FILES,
+            spack.config.DirectoryConfigScope("system", str(configuration_dir.join("system"))),
+        ),
+        (
+            ConfigScopePriority.CONFIG_FILES,
+            spack.config.DirectoryConfigScope("user", str(configuration_dir.join("user"))),
+        ),
+        (ConfigScopePriority.COMMAND_LINE, spack.config.InternalConfigScope("command_line")),
     ]
 
 
@@ -794,13 +818,11 @@ def mock_wsdk_externals(monkeypatch_session):
 def concretize_scope(mutable_config, tmpdir):
     """Adds a scope for concretization preferences"""
     tmpdir.ensure_dir("concretize")
-    mutable_config.push_scope(
+    with spack.config.override(
         spack.config.DirectoryConfigScope("concretize", str(tmpdir.join("concretize")))
-    )
+    ):
+        yield str(tmpdir.join("concretize"))
 
-    yield str(tmpdir.join("concretize"))
-
-    mutable_config.pop_scope()
     spack.repo.PATH._provider_index = None
 
 
@@ -2126,8 +2148,7 @@ def _c_compiler_always_exists():
 @pytest.fixture(scope="session")
 def mock_test_cache(tmp_path_factory):
     cache_dir = tmp_path_factory.mktemp("cache")
-    print(cache_dir)
-    return spack.util.file_cache.FileCache(str(cache_dir))
+    return spack.util.file_cache.FileCache(cache_dir)
 
 
 class MockHTTPResponse(io.IOBase):

lib/spack/spack/test/data/config/config.yaml

@@ -14,3 +14,5 @@ config:
   checksum: true
   dirty: false
   locks: {1}
+  concretization_cache:
+    enable: false

lib/spack/spack/test/directory_layout.py

@@ -161,7 +161,7 @@ def test_handle_unknown_package(temporary_store, config, mock_packages, tmp_path
     """
     layout = temporary_store.layout
 
-    repo_cache = spack.util.file_cache.FileCache(str(tmp_path / "cache"))
+    repo_cache = spack.util.file_cache.FileCache(tmp_path / "cache")
     mock_db = spack.repo.RepoPath(spack.paths.mock_packages_path, cache=repo_cache)
 
     not_in_mock = set.difference(

lib/spack/spack/test/env.py

@@ -519,7 +519,9 @@ def test_error_message_when_using_too_new_lockfile(tmp_path):
         ("when_possible", True),
     ],
 )
-def test_environment_concretizer_scheme_used(tmp_path, unify_in_lower_scope, unify_in_spack_yaml):
+def test_environment_concretizer_scheme_used(
+    tmp_path, mutable_config, unify_in_lower_scope, unify_in_spack_yaml
+):
     """Tests that "unify" settings in spack.yaml always take precedence over settings in lower
     configuration scopes.
     """
@@ -533,10 +535,11 @@ def test_environment_concretizer_scheme_used(tmp_path, unify_in_lower_scope, uni
     unify: {str(unify_in_spack_yaml).lower()}
 """
     )
-
-    with spack.config.override("concretizer:unify", unify_in_lower_scope):
-        with ev.Environment(manifest.parent) as e:
-            assert e.unify == unify_in_spack_yaml
+    mutable_config.set("concretizer:unify", unify_in_lower_scope)
+    assert mutable_config.get("concretizer:unify") == unify_in_lower_scope
+    with ev.Environment(manifest.parent) as e:
+        assert mutable_config.get("concretizer:unify") == unify_in_spack_yaml
+        assert e.unify == unify_in_spack_yaml
 
 
 @pytest.mark.parametrize("unify_in_config", [True, False, "when_possible"])

lib/spack/spack/test/installer.py

@@ -977,7 +977,6 @@ class MyBuildException(Exception):
 
 
 def _install_fail_my_build_exception(installer, task, install_status, **kwargs):
-    print(task, task.pkg.name)
     if task.pkg.name == "pkg-a":
         raise MyBuildException("mock internal package build error for pkg-a")
     else:

lib/spack/spack/test/llnl/util/lang.py

@@ -364,3 +364,44 @@ def test_fnmatch_multiple():
     assert not regex.match("libbar.so.1")
     assert not regex.match("libfoo.solibbar.so")
     assert not regex.match("libbaz.so")
+
+
+class TestPriorityOrderedMapping:
+    @pytest.mark.parametrize(
+        "elements,expected",
+        [
+            # Push out-of-order with explicit, and different, priorities
+            ([("b", 2), ("a", 1), ("d", 4), ("c", 3)], ["a", "b", "c", "d"]),
+            # Push in-order with priority=None
+            ([("a", None), ("b", None), ("c", None), ("d", None)], ["a", "b", "c", "d"]),
+            # Mix explicit and implicit priorities
+            ([("b", 2), ("c", None), ("a", 1), ("d", None)], ["a", "b", "c", "d"]),
+            ([("b", 10), ("c", None), ("a", -20), ("d", None)], ["a", "b", "c", "d"]),
+            ([("b", 10), ("c", None), ("a", 20), ("d", None)], ["b", "c", "a", "d"]),
+            # Adding the same key twice with different priorities
+            ([("b", 10), ("c", None), ("a", 20), ("d", None), ("a", -20)], ["a", "b", "c", "d"]),
+            # Adding the same key twice, no priorities
+            ([("b", None), ("a", None), ("b", None)], ["a", "b"]),
+        ],
+    )
+    def test_iteration_order(self, elements, expected):
+        """Tests that the iteration order respects priorities, no matter the insertion order."""
+        m = llnl.util.lang.PriorityOrderedMapping()
+        for key, priority in elements:
+            m.add(key, value=None, priority=priority)
+        assert list(m) == expected
+
+    def test_reverse_iteration(self):
+        """Tests that we can conveniently use reverse iteration"""
+        m = llnl.util.lang.PriorityOrderedMapping()
+        for key, value in [("a", 1), ("b", 2), ("c", 3)]:
+            m.add(key, value=value)
+
+        assert list(m) == ["a", "b", "c"]
+        assert list(reversed(m)) == ["c", "b", "a"]
+
+        assert list(m.keys()) == ["a", "b", "c"]
+        assert list(m.reversed_keys()) == ["c", "b", "a"]
+
+        assert list(m.values()) == [1, 2, 3]
+        assert list(m.reversed_values()) == [3, 2, 1]

lib/spack/spack/test/modules/common.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
+import pickle
 import stat
 
 import pytest
@@ -223,3 +224,10 @@ def test_check_module_set_name(mutable_config):
 
     with pytest.raises(spack.error.ConfigError, match=msg):
         spack.cmd.modules.check_module_set_name("third")
+
+
+@pytest.mark.parametrize("module_type", ["tcl", "lmod"])
+def test_module_writers_are_pickleable(default_mock_concretization, module_type):
+    s = default_mock_concretization("mpileaks")
+    writer = spack.modules.module_types[module_type](s, "default")
+    assert pickle.loads(pickle.dumps(writer)).spec == s

lib/spack/spack/test/repo.py

@@ -34,7 +34,7 @@ def extra_repo(tmp_path_factory, request):
   subdirectory: '{request.param}'
 """
         )
-    repo_cache = spack.util.file_cache.FileCache(str(cache_dir))
+    repo_cache = spack.util.file_cache.FileCache(cache_dir)
     return spack.repo.Repo(str(repo_dir), cache=repo_cache), request.param
 
 
@@ -194,7 +194,7 @@ def _repo_paths(repos):
 
     repo_paths, namespaces = _repo_paths(repos)
 
-    repo_cache = spack.util.file_cache.FileCache(str(tmp_path / "cache"))
+    repo_cache = spack.util.file_cache.FileCache(tmp_path / "cache")
     repo_path = spack.repo.RepoPath(*repo_paths, cache=repo_cache)
     assert len(repo_path.repos) == len(namespaces)
     assert [x.namespace for x in repo_path.repos] == namespaces
@@ -319,3 +319,48 @@ def test_get_repo(self, mock_test_cache):
         # foo is not there, raise
         with pytest.raises(spack.repo.UnknownNamespaceError):
             repo.get_repo("foo")
+
+
+def test_parse_package_api_version():
+    """Test that we raise an error if a repository has a version that is not supported."""
+    # valid version
+    assert spack.repo._parse_package_api_version(
+        {"api": "v1.2"}, min_api=(1, 0), max_api=(2, 3)
+    ) == (1, 2)
+    # too new and too old
+    with pytest.raises(
+        spack.repo.BadRepoError,
+        match=r"Package API v2.4 is not supported .* \(must be between v1.0 and v2.3\)",
+    ):
+        spack.repo._parse_package_api_version({"api": "v2.4"}, min_api=(1, 0), max_api=(2, 3))
+    with pytest.raises(
+        spack.repo.BadRepoError,
+        match=r"Package API v0.9 is not supported .* \(must be between v1.0 and v2.3\)",
+    ):
+        spack.repo._parse_package_api_version({"api": "v0.9"}, min_api=(1, 0), max_api=(2, 3))
+    # default to v1.0 if not specified
+    assert spack.repo._parse_package_api_version({}, min_api=(1, 0), max_api=(2, 3)) == (1, 0)
+    # if v1.0 support is dropped we should also raise
+    with pytest.raises(
+        spack.repo.BadRepoError,
+        match=r"Package API v1.0 is not supported .* \(must be between v2.0 and v2.3\)",
+    ):
+        spack.repo._parse_package_api_version({}, min_api=(2, 0), max_api=(2, 3))
+    # finally test invalid input
+    with pytest.raises(spack.repo.BadRepoError, match="Invalid Package API version"):
+        spack.repo._parse_package_api_version({"api": "v2"}, min_api=(1, 0), max_api=(3, 3))
+    with pytest.raises(spack.repo.BadRepoError, match="Invalid Package API version"):
+        spack.repo._parse_package_api_version({"api": 2.0}, min_api=(1, 0), max_api=(3, 3))
+
+
+def test_repo_package_api_version(tmp_path: pathlib.Path):
+    """Test that we can specify the API version of a repository."""
+    (tmp_path / "example" / "packages").mkdir(parents=True)
+    (tmp_path / "example" / "repo.yaml").write_text(
+        """\
+repo:
+    namespace: example
+"""
+    )
+    cache = spack.util.file_cache.FileCache(tmp_path / "cache")
+    assert spack.repo.Repo(str(tmp_path / "example"), cache=cache).package_api == (1, 0)

lib/spack/spack/test/rewiring.py

@@ -27,9 +27,7 @@ def check_spliced_spec_prefixes(spliced_spec):
         text_file_path = os.path.join(node.prefix, node.name)
         with open(text_file_path, "r", encoding="utf-8") as f:
             text = f.read()
-            print(text)
             for modded_spec in node.traverse(root=True, deptype=dt.ALL & ~dt.BUILD):
-                print(modded_spec)
                 assert modded_spec.prefix in text
 
 

lib/spack/spack/test/schema.py

@@ -17,7 +17,7 @@
 def validate_spec_schema():
     return {
         "type": "object",
-        "validate_spec": True,
+        "additionalKeysAreSpecs": True,
         "patternProperties": {r"\w[\w-]*": {"type": "string"}},
     }
 
@@ -34,7 +34,7 @@ def module_suffixes_schema():
                         "type": "object",
                         "properties": {
                             "suffixes": {
-                                "validate_spec": True,
+                                "additionalKeysAreSpecs": True,
                                 "patternProperties": {r"\w[\w-]*": {"type": "string"}},
                             }
                         },

lib/spack/spack/test/spec_yaml.py

@@ -427,6 +427,9 @@ def test_load_json_specfiles(specfile, expected_hash, reader_cls):
     openmpi_edges = s2.edges_to_dependencies(name="openmpi")
     assert len(openmpi_edges) == 1
 
+    # Check that virtuals have been reconstructed
+    assert "mpi" in openmpi_edges[0].virtuals
+
     # The virtuals attribute must be a tuple, when read from a
     # JSON or YAML file, not a list
     for edge in s2.traverse_edges():

lib/spack/spack/test/util/environment.py

@@ -149,11 +149,8 @@ def test_reverse_environment_modifications(working_env):
     os.environ.clear()
     os.environ.update(start_env)
 
-    print(os.environ)
     to_reverse.apply_modifications()
-    print(os.environ)
     reversal.apply_modifications()
-    print(os.environ)
 
     start_env.pop("UNSET")
     assert os.environ == start_env

lib/spack/spack/test/util/unparse/unparse.py

@@ -257,7 +257,6 @@ def test_core_lib_files():
                 names.append(os.path.join(test_dir, n))
 
     for filename in names:
-        print("Testing %s" % filename)
         source = read_pyfile(filename)
         check_ast_roundtrip(source)
 

lib/spack/spack/test/variant.py

@@ -685,22 +685,6 @@ def test_str(self) -> None:
         c["shared"] = BoolValuedVariant("shared", True)
         assert str(c) == "+shared feebar=foo foo=bar,baz foobar=fee"
 
-    def test_concrete(self, mock_packages, config) -> None:
-        spec = Spec("pkg-a")
-        assert not VariantMap(spec).concrete
-
-        # concrete if associated spec is concrete
-        spec = spack.concretize.concretize_one(spec)
-        assert VariantMap(spec).concrete
-
-        # concrete if all variants are present (even if spec not concrete)
-        spec._mark_concrete(False)
-        assert spec.variants.concrete
-
-        # remove a variant to test the condition
-        del spec.variants["foo"]
-        assert not spec.variants.concrete
-
 
 def test_disjoint_set_initialization_errors():
     # Constructing from non-disjoint sets should raise an exception

lib/spack/spack/test/verification.py

@@ -133,7 +133,7 @@ def test_check_prefix_manifest(tmpdir):
 
     spec = spack.spec.Spec("libelf")
     spec._mark_concrete()
-    spec.prefix = prefix
+    spec.set_prefix(prefix)
 
     results = spack.verify.check_spec_manifest(spec)
     assert results.has_errors()

lib/spack/spack/test/views.py

@@ -35,8 +35,8 @@ def test_view_with_spec_not_contributing_files(mock_packages, tmpdir):
 
     a = Spec("pkg-a")
     b = Spec("pkg-b")
-    a.prefix = os.path.join(tmpdir, "a")
-    b.prefix = os.path.join(tmpdir, "b")
+    a.set_prefix(os.path.join(tmpdir, "a"))
+    b.set_prefix(os.path.join(tmpdir, "b"))
     a._mark_concrete()
     b._mark_concrete()
 

lib/spack/spack/user_environment.py

@@ -68,7 +68,9 @@ def project_env_mods(
     *specs: spack.spec.Spec, view, env: environment.EnvironmentModifications
 ) -> None:
     """Given a list of environment modifications, project paths changes to the view."""
-    prefix_to_prefix = {s.prefix: view.get_projection_for_spec(s) for s in specs if not s.external}
+    prefix_to_prefix = {
+        str(s.prefix): view.get_projection_for_spec(s) for s in specs if not s.external
+    }
     # Avoid empty regex if all external
     if not prefix_to_prefix:
         return

lib/spack/spack/util/file_cache.py

@@ -5,16 +5,17 @@
 import errno
 import math
 import os
+import pathlib
 import shutil
-from typing import IO, Optional, Tuple
+from typing import IO, Dict, Optional, Tuple, Union
 
-from llnl.util.filesystem import mkdirp, rename
+from llnl.util.filesystem import rename
 
 from spack.error import SpackError
 from spack.util.lock import Lock, ReadTransaction, WriteTransaction
 
 
-def _maybe_open(path: str) -> Optional[IO[str]]:
+def _maybe_open(path: Union[str, pathlib.Path]) -> Optional[IO[str]]:
     try:
         return open(path, "r", encoding="utf-8")
     except OSError as e:
@@ -24,7 +25,7 @@ def _maybe_open(path: str) -> Optional[IO[str]]:
 
 
 class ReadContextManager:
-    def __init__(self, path: str) -> None:
+    def __init__(self, path: Union[str, pathlib.Path]) -> None:
         self.path = path
 
     def __enter__(self) -> Optional[IO[str]]:
@@ -70,7 +71,7 @@ class FileCache:
 
     """
 
-    def __init__(self, root, timeout=120):
+    def __init__(self, root: Union[str, pathlib.Path], timeout=120):
         """Create a file cache object.
 
         This will create the cache directory if it does not exist yet.
@@ -82,58 +83,60 @@ def __init__(self, root, timeout=120):
                 for cache files, this specifies how long Spack should wait
                 before assuming that there is a deadlock.
         """
-        self.root = root.rstrip(os.path.sep)
-        if not os.path.exists(self.root):
-            mkdirp(self.root)
+        if isinstance(root, str):
+            root = pathlib.Path(root)
+        self.root = root
+        self.root.mkdir(parents=True, exist_ok=True)
 
-        self._locks = {}
+        self._locks: Dict[Union[pathlib.Path, str], Lock] = {}
         self.lock_timeout = timeout
 
     def destroy(self):
         """Remove all files under the cache root."""
-        for f in os.listdir(self.root):
-            path = os.path.join(self.root, f)
-            if os.path.isdir(path):
-                shutil.rmtree(path, True)
+        for f in self.root.iterdir():
+            if f.is_dir():
+                shutil.rmtree(f, True)
             else:
-                os.remove(path)
+                f.unlink()
 
-    def cache_path(self, key):
+    def cache_path(self, key: Union[str, pathlib.Path]):
         """Path to the file in the cache for a particular key."""
-        return os.path.join(self.root, key)
+        return self.root / key
 
-    def _lock_path(self, key):
+    def _lock_path(self, key: Union[str, pathlib.Path]):
         """Path to the file in the cache for a particular key."""
         keyfile = os.path.basename(key)
         keydir = os.path.dirname(key)
 
-        return os.path.join(self.root, keydir, "." + keyfile + ".lock")
+        return self.root / keydir / ("." + keyfile + ".lock")
 
-    def _get_lock(self, key):
+    def _get_lock(self, key: Union[str, pathlib.Path]):
         """Create a lock for a key, if necessary, and return a lock object."""
         if key not in self._locks:
-            self._locks[key] = Lock(self._lock_path(key), default_timeout=self.lock_timeout)
+            self._locks[key] = Lock(str(self._lock_path(key)), default_timeout=self.lock_timeout)
         return self._locks[key]
 
-    def init_entry(self, key):
+    def init_entry(self, key: Union[str, pathlib.Path]):
         """Ensure we can access a cache file. Create a lock for it if needed.
 
         Return whether the cache file exists yet or not.
         """
         cache_path = self.cache_path(key)
-
+        # Avoid using pathlib here to allow the logic below to
+        # function as is
+        # TODO: Maybe refactor the following logic for pathlib
         exists = os.path.exists(cache_path)
         if exists:
-            if not os.path.isfile(cache_path):
+            if not cache_path.is_file():
                 raise CacheError("Cache file is not a file: %s" % cache_path)
 
             if not os.access(cache_path, os.R_OK):
                 raise CacheError("Cannot access cache file: %s" % cache_path)
         else:
             # if the file is hierarchical, make parent directories
-            parent = os.path.dirname(cache_path)
-            if parent.rstrip(os.path.sep) != self.root:
-                mkdirp(parent)
+            parent = cache_path.parent
+            if parent != self.root:
+                parent.mkdir(parents=True, exist_ok=True)
 
             if not os.access(parent, os.R_OK | os.W_OK):
                 raise CacheError("Cannot access cache directory: %s" % parent)
@@ -142,7 +145,7 @@ def init_entry(self, key):
             self._get_lock(key)
         return exists
 
-    def read_transaction(self, key):
+    def read_transaction(self, key: Union[str, pathlib.Path]):
         """Get a read transaction on a file cache item.
 
         Returns a ReadTransaction context manager and opens the cache file for
@@ -153,9 +156,11 @@ def read_transaction(self, key):
 
         """
         path = self.cache_path(key)
-        return ReadTransaction(self._get_lock(key), acquire=lambda: ReadContextManager(path))
+        return ReadTransaction(
+            self._get_lock(key), acquire=lambda: ReadContextManager(path)  # type: ignore
+        )
 
-    def write_transaction(self, key):
+    def write_transaction(self, key: Union[str, pathlib.Path]):
         """Get a write transaction on a file cache item.
 
         Returns a WriteTransaction context manager that opens a temporary file
@@ -167,9 +172,11 @@ def write_transaction(self, key):
         if os.path.exists(path) and not os.access(path, os.W_OK):
             raise CacheError(f"Insufficient permissions to write to file cache at {path}")
 
-        return WriteTransaction(self._get_lock(key), acquire=lambda: WriteContextManager(path))
+        return WriteTransaction(
+            self._get_lock(key), acquire=lambda: WriteContextManager(path)  # type: ignore
+        )
 
-    def mtime(self, key) -> float:
+    def mtime(self, key: Union[str, pathlib.Path]) -> float:
         """Return modification time of cache file, or -inf if it does not exist.
 
         Time is in units returned by os.stat in the mtime field, which is
@@ -179,14 +186,14 @@ def mtime(self, key) -> float:
         if not self.init_entry(key):
             return -math.inf
         else:
-            return os.stat(self.cache_path(key)).st_mtime
+            return self.cache_path(key).stat().st_mtime
 
-    def remove(self, key):
+    def remove(self, key: Union[str, pathlib.Path]):
         file = self.cache_path(key)
         lock = self._get_lock(key)
         try:
             lock.acquire_write()
-            os.unlink(file)
+            file.unlink()
         except OSError as e:
             # File not found is OK, so remove is idempotent.
             if e.errno != errno.ENOENT:

lib/spack/spack/verify_libraries.py

@@ -0,0 +1,212 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import fnmatch
+import os
+import re
+from typing import IO, Dict, List
+
+from llnl.util.filesystem import BaseDirectoryVisitor
+from llnl.util.lang import stable_partition
+
+import spack.util.elf as elf
+
+#: Patterns for names of libraries that are allowed to be unresolved when *just* looking at RPATHs
+#: added by Spack. These are libraries outside of Spack's control, and assumed to be located in
+#: default search paths of the dynamic linker.
+ALLOW_UNRESOLVED = [
+    # kernel
+    "linux-vdso.so.*",
+    "libselinux.so.*",
+    # musl libc
+    "ld-musl-*.so.*",
+    # glibc
+    "ld-linux*.so.*",
+    "ld64.so.*",
+    "libanl.so.*",
+    "libc.so.*",
+    "libdl.so.*",
+    "libm.so.*",
+    "libmemusage.so.*",
+    "libmvec.so.*",
+    "libnsl.so.*",
+    "libnss_compat.so.*",
+    "libnss_db.so.*",
+    "libnss_dns.so.*",
+    "libnss_files.so.*",
+    "libnss_hesiod.so.*",
+    "libpcprofile.so.*",
+    "libpthread.so.*",
+    "libresolv.so.*",
+    "librt.so.*",
+    "libSegFault.so.*",
+    "libthread_db.so.*",
+    "libutil.so.*",
+    # gcc -- this is required even with gcc-runtime, because e.g. libstdc++ depends on libgcc_s,
+    # but the binaries we copy from the compiler don't have an $ORIGIN rpath.
+    "libasan.so.*",
+    "libatomic.so.*",
+    "libcc1.so.*",
+    "libgcc_s.so.*",
+    "libgfortran.so.*",
+    "libgomp.so.*",
+    "libitm.so.*",
+    "liblsan.so.*",
+    "libquadmath.so.*",
+    "libssp.so.*",
+    "libstdc++.so.*",
+    "libtsan.so.*",
+    "libubsan.so.*",
+    # systemd
+    "libudev.so.*",
+    # cuda driver
+    "libcuda.so.*",
+    # intel-oneapi-runtime
+    "libur_loader.so.*",
+]
+
+
+def is_compatible(parent: elf.ElfFile, child: elf.ElfFile) -> bool:
+    return (
+        child.elf_hdr.e_type == elf.ELF_CONSTANTS.ET_DYN
+        and parent.is_little_endian == child.is_little_endian
+        and parent.is_64_bit == child.is_64_bit
+        and parent.elf_hdr.e_machine == child.elf_hdr.e_machine
+    )
+
+
+def candidate_matches(current_elf: elf.ElfFile, candidate_path: bytes) -> bool:
+    try:
+        with open(candidate_path, "rb") as g:
+            return is_compatible(current_elf, elf.parse_elf(g))
+    except (OSError, elf.ElfParsingError):
+        return False
+
+
+class Problem:
+    def __init__(
+        self, resolved: Dict[bytes, bytes], unresolved: List[bytes], relative_rpaths: List[bytes]
+    ) -> None:
+        self.resolved = resolved
+        self.unresolved = unresolved
+        self.relative_rpaths = relative_rpaths
+
+
+class ResolveSharedElfLibDepsVisitor(BaseDirectoryVisitor):
+    def __init__(self, allow_unresolved_patterns: List[str]) -> None:
+        self.problems: Dict[str, Problem] = {}
+        self._allow_unresolved_regex = re.compile(
+            "|".join(fnmatch.translate(x) for x in allow_unresolved_patterns)
+        )
+
+    def allow_unresolved(self, needed: bytes) -> bool:
+        try:
+            name = needed.decode("utf-8")
+        except UnicodeDecodeError:
+            return False
+        return bool(self._allow_unresolved_regex.match(name))
+
+    def visit_file(self, root: str, rel_path: str, depth: int) -> None:
+        # We work with byte strings for paths.
+        path = os.path.join(root, rel_path).encode("utf-8")
+
+        # For $ORIGIN interpolation: should not have trailing dir seperator.
+        origin = os.path.dirname(path)
+
+        # Retrieve the needed libs + rpaths.
+        try:
+            with open(path, "rb") as f:
+                parsed_elf = elf.parse_elf(f, interpreter=False, dynamic_section=True)
+        except (OSError, elf.ElfParsingError):
+            # Not dealing with an invalid ELF file.
+            return
+
+        # If there's no needed libs all is good
+        if not parsed_elf.has_needed:
+            return
+
+        # Get the needed libs and rpaths (notice: byte strings)
+        # Don't force an encoding cause paths are just a bag of bytes.
+        needed_libs = parsed_elf.dt_needed_strs
+
+        rpaths = parsed_elf.dt_rpath_str.split(b":") if parsed_elf.has_rpath else []
+
+        # We only interpolate $ORIGIN, not $LIB and $PLATFORM, they're not really
+        # supported in general. Also remove empty paths.
+        rpaths = [x.replace(b"$ORIGIN", origin) for x in rpaths if x]
+
+        # Do not allow relative rpaths (they are relative to the current working directory)
+        rpaths, relative_rpaths = stable_partition(rpaths, os.path.isabs)
+
+        # If there's a / in the needed lib, it's opened directly, otherwise it needs
+        # a search.
+        direct_libs, search_libs = stable_partition(needed_libs, lambda x: b"/" in x)
+
+        # Do not allow relative paths in direct libs (they are relative to the current working
+        # directory)
+        direct_libs, unresolved = stable_partition(direct_libs, os.path.isabs)
+
+        resolved: Dict[bytes, bytes] = {}
+
+        for lib in search_libs:
+            if self.allow_unresolved(lib):
+                continue
+            for rpath in rpaths:
+                candidate = os.path.join(rpath, lib)
+                if candidate_matches(parsed_elf, candidate):
+                    resolved[lib] = candidate
+                    break
+            else:
+                unresolved.append(lib)
+
+        # Check if directly opened libs are compatible
+        for lib in direct_libs:
+            if candidate_matches(parsed_elf, lib):
+                resolved[lib] = lib
+            else:
+                unresolved.append(lib)
+
+        if unresolved or relative_rpaths:
+            self.problems[rel_path] = Problem(resolved, unresolved, relative_rpaths)
+
+    def visit_symlinked_file(self, root: str, rel_path: str, depth: int) -> None:
+        pass
+
+    def before_visit_dir(self, root: str, rel_path: str, depth: int) -> bool:
+        # There can be binaries in .spack/test which shouldn't be checked.
+        if rel_path == ".spack":
+            return False
+        return True
+
+    def before_visit_symlinked_dir(self, root: str, rel_path: str, depth: int) -> bool:
+        return False
+
+    def write(self, output: IO[str], *, indent=0, brief: bool = False) -> None:
+        indent_str = " " * indent
+        for path, problem in self.problems.items():
+            output.write(indent_str)
+            output.write(path)
+            output.write("\n")
+            if not brief:
+                for needed, full_path in problem.resolved.items():
+                    output.write(indent_str)
+                    output.write("        ")
+                    if needed == full_path:
+                        output.write(_decode_or_raw(needed))
+                    else:
+                        output.write(f"{_decode_or_raw(needed)} => {_decode_or_raw(full_path)}")
+                    output.write("\n")
+            for not_found in problem.unresolved:
+                output.write(indent_str)
+                output.write(f"        {_decode_or_raw(not_found)} => not found\n")
+            for relative_rpath in problem.relative_rpaths:
+                output.write(indent_str)
+                output.write(f"        {_decode_or_raw(relative_rpath)} => relative rpath\n")
+
+
+def _decode_or_raw(byte_str: bytes) -> str:
+    try:
+        return byte_str.decode("utf-8")
+    except UnicodeDecodeError:
+        return f"{byte_str!r}"

lib/spack/spack/version/common.py

@@ -10,7 +10,7 @@
 COMMIT_VERSION = re.compile(r"^[a-f0-9]{40}$")
 
 # Infinity-like versions. The order in the list implies the comparison rules
-infinity_versions = ["stable", "trunk", "head", "master", "main", "develop"]
+infinity_versions = ["stable", "nightly", "trunk", "head", "master", "main", "develop"]
 
 iv_min_len = min(len(s) for s in infinity_versions)
 

share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml

@@ -426,7 +426,7 @@ developer-tools-aarch64-linux-gnu-build:
     SPACK_CI_STACK_NAME: developer-tools-darwin
 
 developer-tools-darwin-generate:
-  tags: [ "macos-sonoma", "apple-clang-16", "aarch64-macos" ]
+  tags: [ "macos-sequoia", "apple-clang-16", "aarch64-macos" ]
   extends: [ ".developer-tools-darwin", ".generate-base"]
 
 developer-tools-darwin-build:
@@ -686,7 +686,7 @@ ml-linux-aarch64-cuda-build:
     SPACK_CI_STACK_NAME: ml-darwin-aarch64-mps
 
 ml-darwin-aarch64-mps-generate:
-  tags: [ "macos-sonoma", "apple-clang-16", "aarch64-macos" ]
+  tags: [ "macos-sequoia", "apple-clang-16", "aarch64-macos" ]
   extends: [ ".ml-darwin-aarch64-mps", ".generate-base"]
 
 ml-darwin-aarch64-mps-build:
@@ -910,7 +910,7 @@ bootstrap-x86_64-linux-gnu-build:
     SPACK_CI_STACK_NAME: bootstrap-aarch64-darwin
 
 bootstrap-aarch64-darwin-generate:
-  tags: [ "macos-sonoma", "apple-clang-16", "aarch64-macos" ]
+  tags: [ "macos-sequoia", "apple-clang-16", "aarch64-macos" ]
   extends: [.bootstrap-aarch64-darwin, .generate-base]
 
 bootstrap-aarch64-darwin-build:

share/spack/gitlab/cloud_pipelines/configs/ci.yaml

@@ -20,8 +20,9 @@ ci:
         - k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k
         - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k
       script::
-      - - spack config blame mirrors
-        - spack --color=always --backtrace ci rebuild -j ${SPACK_BUILD_JOBS} --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+      - - if [ -n "$SPACK_EXTRA_MIRROR" ]; then spack mirror add local "$SPACK_EXTRA_MIRROR"; fi
+        - spack config blame mirrors
+      - - spack --color=always --backtrace ci rebuild -j ${SPACK_BUILD_JOBS} --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
       after_script:
       - - cat /proc/loadavg || true
         - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true

share/spack/gitlab/cloud_pipelines/configs/darwin/aarch64/config.yaml

@@ -1,3 +1,5 @@
 config:
+  build_stage:
+  - $spack/tmp/stage
   install_tree:
     root: $spack/opt/spack

share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml

@@ -1,6 +1,20 @@
 ci:
+  broken-tests-packages:
+  - mpich
+  - openmpi
+  - py-mpi4py
   pipeline-gen:
+  - build-job-remove:
+      tags: [spack]
   - build-job:
-      tags: [ "macos-sonoma", "apple-clang-16", "aarch64-macos" ]
+      tags: [ "macos-sequoia", "apple-clang-16", "aarch64-macos" ]
+
+      # after_script intended to ensure all stage files are properly cleaned up,
+      # including those that may have been created as read-only by `go mod`
+      # as part of installation of a golang package
+      # see: https://github.com/spack/spack/issues/49147
+      after_script-:
+      - - if [[ -d tmp ]] ; then chmod -R u+w tmp ; else echo tmp not found ; fi
+        - ./bin/spack clean -a
   - build-job-remove:
       image:: macos-run-on-metal

share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml

@@ -52,10 +52,12 @@ spack:
   packages:
     acfl:
       require:
-      - '%gcc target=aarch64'
+      - "%gcc"
+      - "target=aarch64"
     gromacs:
       require:
-      - gromacs@2024.3 %gcc ^armpl-gcc ^openmpi
+      - gromacs@2024.3 ^armpl-gcc ^openmpi
+      - "%gcc"
     libfabric:
       buildable: true
       externals:
@@ -67,13 +69,14 @@ spack:
       variants: ~lldb
     mpas-model:
       require:
-      - precision=single %gcc ^parallelio+pnetcdf
+      - precision=single ^parallelio+pnetcdf
+      - "%gcc"
     mpich:
       require:
       - mpich pmi=pmi2 device=ch4 netmod=ofi +slurm
     nvhpc:
       require:
-      - nvhpc %gcc target=aarch64
+      - "target=aarch64"
     openfoam:
       require:
       - openfoam ^scotch@6.0.9
@@ -85,7 +88,7 @@ spack:
     #   require:
     #     - one_of: ["palace cxxflags=\"-include cstdint\" ^fmt@9.1.0"]
     pmix:
-      require: 'pmix@3:'
+      require: "pmix@3:"
     quantum-espresso:
       require:
       - quantum-espresso@6.6 %gcc ^armpl-gcc

share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml

@@ -82,6 +82,12 @@ spack:
       require:
       - lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl
       - one_of: [+intel target=x86_64_v4, target=x86_64_v3]
+    bison:
+      require:
+      - "%gcc"
+    boost:
+      require:
+      - "%gcc"
     libfabric:
       buildable: true
       externals:

share/spack/gitlab/cloud_pipelines/stacks/bootstrap-aarch64-darwin/spack.yaml

@@ -25,8 +25,6 @@ spack:
 
   ci:
     pipeline-gen:
-    - build-job-remove:
-        tags: [spack, public]
     - build-job:
         variables:
           CI_GPG_KEY_ROOT: /etc/protected-runner

share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml

@@ -2,25 +2,19 @@ spack:
   view: false
   packages:
     all:
-      require: target=x86_64_v3
-  definitions:
-    - default_specs:
-      - 'uncrustify build_system=autotools'
-      - 'uncrustify build_system=cmake'
-      - lz4  # MakefilePackage
-      - mpich~fortran  # AutotoolsPackage
-      - py-setuptools  # PythonPackage
-      - openjpeg  # CMakePackage
-      - r-rcpp  # RPackage
-      - ruby-rake  # RubyPackage
-      - perl-data-dumper # PerlPackage
-    - arch:
-      - '%gcc'
+      require:
+      - target=x86_64_v3
 
   specs:
-    - matrix:
-        - - $default_specs
-        - - $arch
+  - 'uncrustify build_system=autotools'
+  - 'uncrustify build_system=cmake'
+  - lz4  # MakefilePackage
+  - mpich~fortran  # AutotoolsPackage
+  - py-setuptools  # PythonPackage
+  - openjpeg  # CMakePackage
+  - r-rcpp  # RPackage
+  - ruby-rake  # RubyPackage
+  - perl-data-dumper # PerlPackage
 
   cdash:
     build-group: Build Systems

share/spack/gitlab/cloud_pipelines/stacks/developer-tools-aarch64-linux-gnu/spack.yaml

@@ -2,79 +2,75 @@ spack:
   view: false
   packages:
     all:
-      require: target=aarch64
+      require:
+      - target=aarch64
+      prefer:
+      - '%gcc'
+
   concretizer:
     unify: true
     reuse: false
-  definitions:
-  - default_specs:
-      # editors
-    - neovim~no_luajit
-    - py-pynvim
-    - emacs+json+native+treesitter   # note, pulls in gcc
-      # - tree-sitter is a dep, should also have cli but no package
-    - nano   # just in case
-      # tags and scope search helpers
-    - universal-ctags   # only maintained ctags, works better with c++
-    - direnv
-      # runtimes and compilers
-    - python
-    - llvm+link_llvm_dylib~lld~lldb~polly+python build_type=MinSizeRel   # for clangd, clang-format
-    - node-js   # for editor plugins etc., pyright language server
-    - npm
-    - cmake
-    - libtool
-    - go   # to build fzf, gh, hub
-    - rust+dev   # fd, ripgrep, hyperfine, exa, rust-analyzer
-    - binutils+ld+gold+plugins   # support linking with built gcc
-      # styling and lints
-    - astyle
-    - cppcheck
-    - uncrustify
-    - py-fprettify
-    - py-fortran-language-server
-    - py-python-lsp-server
-      # cli dev tools
-    - ripgrep
-    - gh
-    - fd
-    - bfs
-    - fzf
-    - tree
-    - jq
-    - py-yq
-    - hub
-    - ncdu
-    - eza
-    - lsd
-    - hyperfine
-    - htop
-    - tmux
-    - ccache
-      # ensure we can use a jobserver build and do this fast
-    - gmake
-    - ninja   # should be @kitware, can't be because of meson requirement
-    - openssl certs=system     # must be this, system external does not work
-    - libtree
-    - patchelf
-    - sed
-    - which
-    - elfutils
-    - fontconfig
-    - font-util
-    - gdb
-    - flex
-    - graphviz
-    - doxygen
-    - meson
-
-  - arch:
-    - '%gcc target=aarch64'
 
   specs:
-  - matrix:
-    - - $default_specs
-    - - $arch
+    # editors
+  - neovim~no_luajit
+  - py-pynvim
+  - emacs+json+native+treesitter   # note, pulls in gcc
+    # - tree-sitter is a dep, should also have cli but no package
+  - nano   # just in case
+    # tags and scope search helpers
+  - universal-ctags   # only maintained ctags, works better with c++
+  - direnv
+    # runtimes and compilers
+  - python
+  - llvm+link_llvm_dylib~lld~lldb~polly+python build_type=MinSizeRel   # for clangd, clang-format
+  - node-js   # for editor plugins etc., pyright language server
+  - npm
+  - cmake
+  - libtool
+  - go   # to build fzf, gh, hub
+  - rust+dev   # fd, ripgrep, hyperfine, exa, rust-analyzer
+  - binutils+ld+gold+plugins   # support linking with built gcc
+    # styling and lints
+  - astyle
+  - cppcheck
+  - uncrustify
+  - py-fprettify
+  - py-fortran-language-server
+  - py-python-lsp-server
+    # cli dev tools
+  - ripgrep
+  - gh
+  - fd
+  - bfs
+  - fzf
+  - tree
+  - jq
+  - py-yq
+  - hub
+  - ncdu
+  - eza
+  - lsd
+  - hyperfine
+  - htop
+  - tmux
+  - ccache
+    # ensure we can use a jobserver build and do this fast
+  - gmake
+  - ninja   # should be @kitware, can't be because of meson requirement
+  - openssl certs=system     # must be this, system external does not work
+  - libtree
+  - patchelf
+  - sed
+  - which
+  - elfutils
+  - fontconfig
+  - font-util
+  - gdb
+  - flex
+  - graphviz
+  - doxygen
+  - meson
 
   ci:
     pipeline-gen:

share/spack/gitlab/cloud_pipelines/stacks/developer-tools-darwin/spack.yaml

@@ -9,7 +9,7 @@ spack:
     reuse: false
   specs:
       # editors
-    - neovim~no_luajit
+    #- neovim~no_luajit # build fails: https://github.com/spack/spack/pull/48453#issuecomment-2624788262
     - py-pynvim
     - emacs+json~native+treesitter   # TODO native not supported until gcc builds on darwin
       # - tree-sitter is a dep, should also have cli but no package
@@ -64,8 +64,6 @@ spack:
 
   ci:
     pipeline-gen:
-    - build-job-remove:
-        tags: [ spack, public ]
     - build-job:
         variables:
           CI_GPG_KEY_ROOT: /etc/protected-runner

share/spack/gitlab/cloud_pipelines/stacks/developer-tools-x86_64_v3-linux-gnu/spack.yaml

@@ -1,86 +1,79 @@
 spack:
   view: false
+
   packages:
     all:
       require:
       - target=x86_64_v3
       - ~cuda
       - ~rocm
-
+      prefer:
+      - "%gcc"
   concretizer:
     unify: true
     reuse: false
     static_analysis: true
 
-  definitions:
-  - default_specs:
-      # editors
-    - neovim~no_luajit
-    - py-pynvim
-    - emacs+json+native+treesitter   # note, pulls in gcc
-      # - tree-sitter is a dep, should also have cli but no package
-    - nano   # just in case
-      # tags and scope search helpers
-    - universal-ctags   # only maintained ctags, works better with c++
-    - direnv
-      # runtimes and compilers
-    - python
-    - llvm+link_llvm_dylib~lld~lldb~polly+python build_type=MinSizeRel   # for clangd, clang-format
-    - node-js   # for editor plugins etc., pyright language server
-    - npm
-    - cmake
-    - libtool
-    - go   # to build fzf, gh, hub
-    - rust+dev   # fd, ripgrep, hyperfine, exa, rust-analyzer
-    - binutils+ld+gold+plugins   # support linking with built gcc
-      # styling and lints
-    - astyle
-    - cppcheck
-    - uncrustify
-    - py-fprettify
-    - py-fortran-language-server
-    - py-python-lsp-server
-      # cli dev tools
-    - ripgrep
-    - gh
-    - fd
-    - bfs
-    - fzf
-    - tree
-    - jq
-    - py-yq
-    - hub
-    - ncdu
-    - eza
-    - lsd
-    - hyperfine
-    - htop
-    - tmux
-    - ccache
-      # ensure we can use a jobserver build and do this fast
-    - gmake
-    - ninja   # should be @kitware, can't be because of meson requirement
-    - openssl certs=system     # must be this, system external does not work
-    - libtree
-    - patchelf
-    - sed
-    - which
-    - elfutils
-    - fontconfig
-    - font-util
-    - gdb
-    - flex
-    - graphviz
-    - doxygen
-    - meson
-
-  - arch:
-    - '%gcc target=x86_64_v3'
-
   specs:
-  - matrix:
-    - - $default_specs
-    - - $arch
+    # editors
+  - neovim~no_luajit
+  - py-pynvim
+  - emacs+json+native+treesitter   # note, pulls in gcc
+    # - tree-sitter is a dep, should also have cli but no package
+  - nano   # just in case
+    # tags and scope search helpers
+  - universal-ctags   # only maintained ctags, works better with c++
+  - direnv
+    # runtimes and compilers
+  - python
+  - llvm+link_llvm_dylib~lld~lldb~polly+python build_type=MinSizeRel   # for clangd, clang-format
+  - node-js   # for editor plugins etc., pyright language server
+  - npm
+  - cmake
+  - libtool
+  - go   # to build fzf, gh, hub
+  - rust+dev   # fd, ripgrep, hyperfine, exa, rust-analyzer
+  - binutils+ld+gold+plugins   # support linking with built gcc
+    # styling and lints
+  - astyle
+  - cppcheck
+  - uncrustify
+  - py-fprettify
+  - py-fortran-language-server
+  - py-python-lsp-server
+    # cli dev tools
+  - ripgrep
+  - gh
+  - fd
+  - bfs
+  - fzf
+  - tree
+  - jq
+  - py-yq
+  - hub
+  - ncdu
+  - eza
+  - lsd
+  - hyperfine
+  - htop
+  - tmux
+  - ccache
+    # ensure we can use a jobserver build and do this fast
+  - gmake
+  - ninja   # should be @kitware, can't be because of meson requirement
+  - openssl certs=system     # must be this, system external does not work
+  - libtree
+  - patchelf
+  - sed
+  - which
+  - elfutils
+  - fontconfig
+  - font-util
+  - gdb
+  - flex
+  - graphviz
+  - doxygen
+  - meson
 
   ci:
     pipeline-gen:

share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml

@@ -5,6 +5,7 @@ spack:
   view: false
 
   concretizer:
+    static_analysis: true
     reuse: false
     unify: false
 
@@ -14,8 +15,9 @@ spack:
 
   packages:
     all:
-      require: "%cce@18.0.0 target=x86_64_v3"
-      compiler: [cce]
+      require:
+      - target=x86_64_v3
+      - "%cce"
       providers:
         blas: [cray-libsci]
         lapack: [cray-libsci]
@@ -23,6 +25,21 @@ spack:
         tbb: [intel-tbb]
         scalapack: [netlib-scalapack]
       variants: +mpi
+
+    # Virtuals
+    blas:
+      require:
+      - cray-libsci
+    lapack:
+      require:
+      - cray-libsci
+    mpi:
+      require:
+      - cray-mpich
+    scalapack:
+      require:
+      - netlib-scalapack
+
     ncurses:
       require: +termlib ldflags=-Wl,--undefined-version
     tbb:
@@ -33,21 +50,28 @@ spack:
       variants: +python +filesystem +iostreams +system
     elfutils:
       variants: ~nls
-      require: "%gcc"
+      require:
+      - target=x86_64_v3
+      - "%gcc"
     gcc-runtime:
-      require: "%gcc"
+      require:
+      - target=x86_64_v3
+      - "%gcc"
     hdf5:
       variants: +fortran +hl +shared
     libfabric:
       variants: fabrics=sockets,tcp,udp,rxm
     mgard:
       require:
+        - target=x86_64_v3
         - "@2023-01-10:"
     mpich:
       variants: ~wrapperrpath
     paraview:
       # Don't build GUI support or GLX rendering for HPC/container deployments
-      require: "~qt ^[virtuals=gl] osmesa"
+      require:
+      - "~qt ^[virtuals=gl] osmesa"
+      - target=x86_64_v3
     trilinos:
       require:
       - one_of: [+amesos +amesos2 +anasazi +aztec +boost +epetra +epetraext +ifpack
@@ -58,6 +82,7 @@ spack:
       - one_of: [~ml ~muelu ~zoltan2 ~teko, +ml +muelu +zoltan2 +teko]
       - one_of: [+superlu-dist, ~superlu-dist]
       - one_of: [+shylu, ~shylu]
+      - target=x86_64_v3
 
   specs:
   # CPU
@@ -140,8 +165,8 @@ spack:
   # - gptune              # py-scipy: meson.build:82:0: ERROR: Unknown compiler(s): [['/home/gitlab-runner-3/builds/dWfnZWPh/0/spack/spack/lib/spack/env/cce/ftn']]
   # - hpctoolkit          # dyninst requires %gcc
   # - hpx max_cpu_count=512 networking=mpi  # libxcrypt-4.4.35
-  # - lammps              # lammps-20240829.1: Reversed (or previously applied) patch detected!  Assume -R? [n] 
-  # - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +mgard # mgard: 
+  # - lammps              # lammps-20240829.1: Reversed (or previously applied) patch detected!  Assume -R? [n]
+  # - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +mgard # mgard:
   # - mgard +serial +openmp +timing +unstructured ~cuda # mgard
   # - nrm                 # py-scipy: meson.build:82:0: ERROR: Unknown compiler(s): [['/home/gitlab-runner-3/builds/dWfnZWPh/0/spack/spack/lib/spack/env/cce/ftn']]
   # - nvhpc               # requires %gcc

share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse-v2/spack.yaml

@@ -7,7 +7,9 @@ spack:
 
   packages:
     all:
-      require: '%gcc target=neoverse_v2'
+      require:
+      - "%gcc"
+      - target=neoverse_v2
       providers:
         blas: [openblas]
         mpi: [mpich]

share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml

@@ -4,12 +4,13 @@ spack:
   concretizer:
     reuse: false
     unify: false
+    static_analysis: false
 
   packages:
     all:
       require:
-      - "target=x86_64_v3"
-      - "%oneapi"
+      - target=x86_64_v3
+      - '%oneapi'
       providers:
         blas: [openblas]
         tbb: [intel-tbb]
@@ -38,32 +39,43 @@ spack:
     xz:
       variants: +pic
     mpi:
-      require: 'mpich@4: target=x86_64_v3'
-    mpich:
-      require: '~wrapperrpath ~hwloc target=x86_64_v3'
+      require: intel-oneapi-mpi
+    intel-oneapi-mpi:
+      buildable: false
+      externals:
+      - spec: intel-oneapi-mpi@2021.13.1
+        prefix: /opt/intel/oneapi
     unzip:
-      require: '%gcc target=x86_64_v3'
+      require:
+      - '%gcc target=x86_64_v3'
     binutils:
-      require: '%gcc target=x86_64_v3'
+      require:
+      - '%gcc target=x86_64_v3'
       variants: +ld +gold +headers +libiberty ~nls
     llvm:
-      require: '%gcc target=x86_64_v3'
+      require:
+      - '%gcc target=x86_64_v3'
     ruby:
-      require: '%gcc target=x86_64_v3'
+      require:
+      - '%gcc target=x86_64_v3'
     rust:
-      require: '%gcc target=x86_64_v3'
+      require:
+      - '%gcc target=x86_64_v3'
     krb5:
-      require: '%gcc target=x86_64_v3'
-    papi:
-      require: '%gcc target=x86_64_v3'
+      require:
+      - '%gcc target=x86_64_v3'
     openssh:
-      require: '%gcc target=x86_64_v3'
+      require:
+      - '%gcc target=x86_64_v3'
     dyninst:
-      require: "%gcc target=x86_64_v3"
+      require:
+      - '%gcc target=x86_64_v3'
     bison:
-      require: '%gcc target=x86_64_v3'
+      require:
+      - '%gcc target=x86_64_v3'
     paraview:
-      require: "+examples %oneapi target=x86_64_v3"
+      require:
+      - +examples target=x86_64_v3
 
   specs:
   # CPU
@@ -128,7 +140,7 @@ spack:
   - nccmp
   - nco
   - netlib-scalapack
-  - nrm
+  - nrm ^py-scipy cflags="-Wno-error=incompatible-function-pointer-types" # py-scipy@1.8.1 fails without cflags here
   - nwchem
   - omega-h
   - openfoam

share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml

@@ -7,7 +7,8 @@ spack:
 
   packages:
     all:
-      require: '%gcc target=x86_64_v3'
+      require:
+      - 'target=x86_64_v3'
       providers:
         blas: [openblas]
       variants: +mpi
@@ -21,7 +22,9 @@ spack:
       variants: threads=openmp
     paraview:
       # Don't build GUI support or GLX rendering for HPC/container deployments
-      require: "@5.11 +examples ~qt ^[virtuals=gl] osmesa %gcc target=x86_64_v3"
+      require:
+      - "@5.11 +examples ~qt ^[virtuals=gl] osmesa"
+      - 'target=x86_64_v3'
 
     # ROCm
     comgr:

share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml

@@ -9,7 +9,8 @@ spack:
   packages:
     all:
       require:
-      - '%gcc target=x86_64_v3'
+      - "%gcc"
+      - target=x86_64_v3
       variants: +mpi
     mpi:
       require:

share/spack/gitlab/cloud_pipelines/stacks/hep/spack.yaml

@@ -8,7 +8,9 @@ spack:
 
   packages:
     all:
-      require: '%gcc target=x86_64_v3'
+      require:
+      - "%gcc"
+      - target=x86_64_v3
       providers:
         blas: [openblas]
         mpi: [mpich]

share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml

@@ -9,13 +9,13 @@ spack:
       - ~cuda
       - ~rocm
     mpi:
-      require: openmpi
+      require: mpich
     openblas:
       require: ~fortran
 
   specs:
   # Horovod
-  - py-horovod
+  # - py-horovod # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
 
   # Hugging Face
   - py-transformers
@@ -45,18 +45,18 @@ spack:
   - py-segmentation-models-pytorch
   - py-timm
   - py-torch
-  - py-torch-cluster
+  # - py-torch-cluster      # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
   - py-torch-geometric
-  - py-torch-nvidia-apex
-  - py-torch-scatter
-  - py-torch-sparse
-  - py-torch-spline-conv
+  # - py-torch-nvidia-apex  # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
+  # - py-torch-scatter      # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
+  # - py-torch-sparse       # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
+  # - py-torch-spline-conv  # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
   - py-torchaudio
   - py-torchdata
   - py-torchfile
   - py-torchgeo
   - py-torchmetrics
-  - py-torchtext
+  # - py-torchtext          # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
   - py-torchvision
   - py-vector-quantize-pytorch
 
@@ -84,8 +84,6 @@ spack:
 
   ci:
     pipeline-gen:
-    - build-job-remove:
-        tags: [ spack, public ]
     - build-job:
         variables:
           CI_GPG_KEY_ROOT: /etc/protected-runner