Revert "Solver: Cache Concretization Results (#48198)"

This reverts commit d234df62d7.

.github/workflows/requirements/style/requirements.txt

@@ -1,7 +1,7 @@
 black==25.1.0
 clingo==5.7.1
 flake8==7.1.2
-isort==6.0.0
-mypy==1.11.2
-types-six==1.17.0.20241205
+isort==6.0.1
+mypy==1.15.0
+types-six==1.17.0.20250304
 vermin==1.6.0

.gitignore

@@ -201,7 +201,6 @@ tramp
 
 # Org-mode
 .org-id-locations
-*_archive
 
 # flymake-mode
 *_flymake.*

etc/spack/defaults/concretizer.yaml

@@ -54,9 +54,15 @@ concretizer:
       # Regular packages
       cmake: 2
       gmake: 2
+      python: 2
+      python-venv: 2
       py-cython: 2
       py-flit-core: 2
+      py-pip: 2
       py-setuptools: 2
+      py-wheel: 2
+      xcb-proto: 2
+      # Compilers
       gcc: 2
       llvm: 2
   # Option to specify compatibility between operating systems for reuse of compilers and packages

lib/spack/docs/basic_usage.rst

@@ -1761,19 +1761,24 @@ Verifying installations
 The ``spack verify`` command can be used to verify the validity of
 Spack-installed packages any time after installation.
 
+
+^^^^^^^^^^^^^^^^^^^^^^^^^
+``spack verify manifest``
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
 At installation time, Spack creates a manifest of every file in the
 installation prefix. For links, Spack tracks the mode, ownership, and
 destination. For directories, Spack tracks the mode, and
 ownership. For files, Spack tracks the mode, ownership, modification
-time, hash, and size. The Spack verify command will check, for every
-file in each package, whether any of those attributes have changed. It
-will also check for newly added files or deleted files from the
-installation prefix. Spack can either check all installed packages
+time, hash, and size. The ``spack verify manifest`` command will check,
+for every file in each package, whether any of those attributes have
+changed. It will also check for newly added files or deleted files from
+the installation prefix. Spack can either check all installed packages
 using the `-a,--all` or accept specs listed on the command line to
 verify.
 
-The ``spack verify`` command can also verify for individual files that
-they haven't been altered since installation time. If the given file
+The ``spack verify manifest`` command can also verify for individual files
+that they haven't been altered since installation time. If the given file
 is not in a Spack installation prefix, Spack will report that it is
 not owned by any package. To check individual files instead of specs,
 use the ``-f,--files`` option.
@@ -1788,6 +1793,22 @@ check only local packages (as opposed to those used transparently from
 ``upstream`` spack instances) and the ``-j,--json`` option to output
 machine-readable json data for any errors.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+``spack verify libraries``
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``spack verify libraries`` command can be used to verify that packages
+do not have accidental system dependencies. This command scans the install
+prefixes of packages for executables and shared libraries, and resolves
+their needed libraries in their RPATHs. When needed libraries cannot be
+located, an error is reported. This typically indicates that a package
+was linked against a system library, instead of a library provided by
+a Spack package.
+
+This verification can also be enabled as a post-install hook by setting
+``config:shared_linking:missing_library_policy`` to ``error`` or ``warn``
+in :ref:`config.yaml <config-yaml>`.
+
 -----------------------
 Filesystem requirements
 -----------------------

lib/spack/docs/conf.py

@@ -223,6 +223,10 @@ def setup(sphinx):
     ("py:class", "spack.compiler.CompilerCache"),
     # TypeVar that is not handled correctly
     ("py:class", "llnl.util.lang.T"),
+    ("py:class", "llnl.util.lang.KT"),
+    ("py:class", "llnl.util.lang.VT"),
+    ("py:obj", "llnl.util.lang.KT"),
+    ("py:obj", "llnl.util.lang.VT"),
 ]
 
 # The reST default role (used for this markup: `text`) to use for all documents.

lib/spack/docs/configuration.rst

@@ -14,6 +14,7 @@ case you want to skip directly to specific docs:
 * :ref:`compilers.yaml <compiler-config>`
 * :ref:`concretizer.yaml <concretizer-options>`
 * :ref:`config.yaml <config-yaml>`
+* :ref:`include.yaml <include-yaml>`
 * :ref:`mirrors.yaml <mirrors>`
 * :ref:`modules.yaml <modules>`
 * :ref:`packages.yaml <packages-config>`

lib/spack/docs/environments.rst

@@ -670,24 +670,45 @@ This configuration sets the default compiler for all packages to
 Included configurations
 ^^^^^^^^^^^^^^^^^^^^^^^
 
-Spack environments allow an ``include`` heading in their yaml
-schema. This heading pulls in external configuration files and applies
-them to the environment.
+Spack environments allow an ``include`` heading in their yaml schema.
+This heading pulls in external configuration files and applies them to
+the environment.
 
 .. code-block:: yaml
 
    spack:
      include:
-     - relative/path/to/config.yaml
+     - environment/relative/path/to/config.yaml
      - https://github.com/path/to/raw/config/compilers.yaml
      - /absolute/path/to/packages.yaml
+     - path: /path/to/$os/$target/environment
+       optional: true
+     - path: /path/to/os-specific/config-dir
+       when: os == "ventura"
+
+Included configuration files are required *unless* they are explicitly optional
+or the entry's condition evaluates to ``false``. Optional includes are specified
+with the ``optional`` clause and conditional with the ``when`` clause. (See
+:ref:`include-yaml` for more information on optional and conditional entries.)
+
+Files are listed using paths to individual files or directories containing them.
+Path entries may be absolute or relative to the environment or specified as 
+URLs. URLs to individual files need link to the **raw** form of the file's 
+contents (e.g., `GitHub
+<https://docs.github.com/en/repositories/working-with-files/using-files/viewing-and-understanding-files#viewing-or-copying-the-raw-file-content>`_ 
+or `GitLab
+<https://docs.gitlab.com/ee/api/repository_files.html#get-raw-file-from-repository>`_).
+Only the ``file``, ``ftp``, ``http`` and ``https`` protocols (or schemes) are
+supported. Spack-specific, environment and user path variables can be used.
+(See :ref:`config-file-variables` for more information.)
+
+.. warning::
+
+   Recursive includes are not currently processed in a breadth-first manner
+   so the value of a configuration option that is altered by multiple included
+   files may not be what you expect. This will be addressed in a future
+   update.
 
-Environments can include files or URLs. File paths can be relative or
-absolute. URLs include the path to the text for individual files or
-can be the path to a directory containing configuration files.
-Spack supports ``file``, ``http``, ``https`` and ``ftp`` protocols (or
-schemes). Spack-specific, environment and user path variables may be
-used in these paths. See :ref:`config-file-variables` for more information.
 
 ^^^^^^^^^^^^^^^^^^^^^^^^
 Configuration precedence

lib/spack/docs/include_yaml.rst

@@ -0,0 +1,51 @@
+.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+
+   SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+.. _include-yaml:
+
+===============================
+Include Settings (include.yaml)
+===============================
+
+Spack allows you to include configuration files through ``include.yaml``.
+Using the ``include:`` heading results in pulling in external configuration
+information to be used by any Spack command.
+
+Included configuration files are required *unless* they are explicitly optional
+or the entry's condition evaluates to ``false``. Optional includes are specified
+with the ``optional`` clause and conditional with the ``when`` clause. For
+example,
+
+.. code-block:: yaml
+
+   include:
+   - /path/to/a/required/config.yaml
+   - path: /path/to/$os/$target/config
+     optional: true
+   - path: /path/to/os-specific/config-dir
+     when: os == "ventura"
+
+shows all three. The first entry, ``/path/to/a/required/config.yaml``,
+indicates that included ``config.yaml`` file is required (so must exist).
+Use of ``optional: true`` for ``/path/to/$os/$target/config`` means
+the path is only included if it exists. The condition ``os == "ventura"``
+in the ``when`` clause for ``/path/to/os-specific/config-dir`` means the
+path is only included when the operating system (``os``) is ``ventura``.
+
+The same conditions and variables in `Spec List References 
+<https://spack.readthedocs.io/en/latest/environments.html#spec-list-references>`_
+can be used for conditional activation in the ``when`` clauses.
+
+Included files can be specified by path or by their parent directory.
+Paths may be absolute, relative (to the configuration file including the path), 
+or specified as URLs. Only the ``file``, ``ftp``, ``http`` and ``https`` protocols (or
+schemes) are supported. Spack-specific, environment and user path variables
+can be used. (See :ref:`config-file-variables` for more information.)
+
+.. warning::
+
+   Recursive includes are not currently processed in a breadth-first manner
+   so the value of a configuration option that is altered by multiple included
+   files may not be what you expect. This will be addressed in a future
+   update.

lib/spack/docs/index.rst

@@ -71,6 +71,7 @@ or refer to the full manual below.
 
    configuration
    config_yaml
+   include_yaml
    packages_yaml
    build_settings
    environments

lib/spack/docs/requirements.txt

@@ -1,13 +1,13 @@
-sphinx==8.2.0
+sphinx==8.2.3
 sphinxcontrib-programoutput==0.18
 sphinx_design==0.6.1
 sphinx-rtd-theme==3.0.2
-python-levenshtein==0.26.1
+python-levenshtein==0.27.1
 docutils==0.21.2
 pygments==2.19.1
 urllib3==2.3.0
-pytest==8.3.4
-isort==6.0.0
+pytest==8.3.5
+isort==6.0.1
 black==25.1.0
 flake8==7.1.2
 mypy==1.11.1

lib/spack/llnl/util/filesystem.py

@@ -2454,26 +2454,69 @@ class WindowsSimulatedRPath:
     and vis versa.
     """
 
-    def __init__(self, package, link_install_prefix=True):
+    def __init__(
+        self,
+        package,
+        base_modification_prefix: Optional[Union[str, pathlib.Path]] = None,
+        link_install_prefix: bool = True,
+    ):
         """
         Args:
             package (spack.package_base.PackageBase): Package requiring links
+            base_modification_prefix (str|pathlib.Path): Path representation indicating
+                the root directory in which to establish the simulated rpath, ie where the
+                symlinks that comprise the "rpath" behavior will be installed.
+
+                Note: This is a mutually exclusive option with `link_install_prefix` using
+                both is an error.
+
+                Default: None
             link_install_prefix (bool): Link against package's own install or stage root.
                 Packages that run their own executables during build and require rpaths to
-                the build directory during build time require this option. Default: install
+                the build directory during build time require this option.
+
+                Default: install
                 root
+
+                Note: This is a mutually exclusive option with `base_modification_prefix`, using
+                both is an error.
         """
         self.pkg = package
-        self._addl_rpaths = set()
+        self._addl_rpaths: set[str] = set()
+        if link_install_prefix and base_modification_prefix:
+            raise RuntimeError(
+                "Invalid combination of arguments given to WindowsSimulated RPath.\n"
+                "Select either `link_install_prefix` to create an install prefix rpath"
+                " or specify a `base_modification_prefix` for any other link type. "
+                "Specifying both arguments is invalid."
+            )
+        if not (link_install_prefix or base_modification_prefix):
+            raise RuntimeError(
+                "Insufficient arguments given to WindowsSimulatedRpath.\n"
+                "WindowsSimulatedRPath requires one of link_install_prefix"
+                " or base_modification_prefix to be specified."
+                " Neither was provided."
+            )
+
         self.link_install_prefix = link_install_prefix
-        self._additional_library_dependents = set()
+        if base_modification_prefix:
+            self.base_modification_prefix = pathlib.Path(base_modification_prefix)
+        else:
+            self.base_modification_prefix = pathlib.Path(self.pkg.prefix)
+        self._additional_library_dependents: set[pathlib.Path] = set()
+        if not self.link_install_prefix:
+            tty.debug(f"Generating rpath for non install context: {base_modification_prefix}")
 
     @property
     def library_dependents(self):
         """
         Set of directories where package binaries/libraries are located.
         """
-        return set([pathlib.Path(self.pkg.prefix.bin)]) | self._additional_library_dependents
+        base_pths = set()
+        if self.link_install_prefix:
+            base_pths.add(pathlib.Path(self.pkg.prefix.bin))
+        base_pths |= self._additional_library_dependents
+        return base_pths
 
     def add_library_dependent(self, *dest):
         """
@@ -2489,6 +2532,12 @@ def add_library_dependent(self, *dest):
                 new_pth = pathlib.Path(pth).parent
             else:
                 new_pth = pathlib.Path(pth)
+            path_is_in_prefix = new_pth.is_relative_to(self.base_modification_prefix)
+            if not path_is_in_prefix:
+                raise RuntimeError(
+                    f"Attempting to generate rpath symlink out of rpath context:\
+{str(self.base_modification_prefix)}"
+                )
             self._additional_library_dependents.add(new_pth)
 
     @property
@@ -2577,6 +2626,33 @@ def establish_link(self):
                 self._link(library, lib_dir)
 
 
+def make_package_test_rpath(pkg, test_dir: Union[str, pathlib.Path]):
+    """Establishes a temp Windows simulated rpath for the pkg in the testing directory
+    so an executable can test the libraries/executables with proper access
+    to dependent dlls
+
+    Note: this is a no-op on all other platforms besides Windows
+
+    Args:
+        pkg (spack.package_base.PackageBase): the package for which the rpath should be computed
+        test_dir: the testing directory in which we should construct an rpath
+    """
+    # link_install_prefix as false ensures we're not linking into the install prefix
+    mini_rpath = WindowsSimulatedRPath(pkg, link_install_prefix=False)
+    # add the testing directory as a location to install rpath symlinks
+    mini_rpath.add_library_dependent(test_dir)
+
+    # check for whether build_directory is available, if not
+    # assume the stage root is the build dir
+    build_dir_attr = getattr(pkg, "build_directory", None)
+    build_directory = build_dir_attr if build_dir_attr else pkg.stage.path
+    # add the build dir & build dir bin
+    mini_rpath.add_rpath(os.path.join(build_directory, "bin"))
+    mini_rpath.add_rpath(os.path.join(build_directory))
+    # construct rpath
+    mini_rpath.establish_link()
+
+
 @system_path_filter
 @memoized
 def can_access_dir(path):

lib/spack/llnl/util/lang.py

@@ -11,10 +11,11 @@
 import re
 import sys
 import traceback
+import types
 import typing
 import warnings
 from datetime import datetime, timedelta
-from typing import Callable, Dict, Iterable, List, Tuple, TypeVar
+from typing import Callable, Dict, Iterable, List, Mapping, Optional, Tuple, TypeVar
 
 # Ignore emacs backups when listing modules
 ignore_modules = r"^\.#|~$"
@@ -707,14 +708,24 @@ def __init__(self, wrapped_object):
 
 
 class Singleton:
-    """Simple wrapper for lazily initialized singleton objects."""
+    """Wrapper for lazily initialized singleton objects."""
 
-    def __init__(self, factory):
+    def __init__(self, factory: Callable[[], object]):
         """Create a new singleton to be inited with the factory function.
 
+        Most factories will simply create the object to be initialized and
+        return it.
+
+        In some cases, e.g. when bootstrapping some global state, the singleton
+        may need to be initialized incrementally. If the factory returns a generator
+        instead of a regular object, the singleton will assign each result yielded by
+        the generator to the singleton instance. This allows methods called by
+        the factory in later stages to refer back to the singleton.
+
         Args:
-            factory (function): function taking no arguments that
-                creates the singleton instance.
+            factory (function): function taking no arguments that creates the
+                singleton instance.
+
         """
         self.factory = factory
         self._instance = None
@@ -722,7 +733,16 @@ def __init__(self, factory):
     @property
     def instance(self):
         if self._instance is None:
-            self._instance = self.factory()
+            instance = self.factory()
+
+            if isinstance(instance, types.GeneratorType):
+                # if it's a generator, assign every value
+                for value in instance:
+                    self._instance = value
+            else:
+                # if not, just assign the result like a normal singleton
+                self._instance = instance
+
         return self._instance
 
     def __getattr__(self, name):
@@ -1080,3 +1100,88 @@ def __set__(self, instance, value):
 
     def factory(self, instance, owner):
         raise NotImplementedError("must be implemented by derived classes")
+
+
+KT = TypeVar("KT")
+VT = TypeVar("VT")
+
+
+class PriorityOrderedMapping(Mapping[KT, VT]):
+    """Mapping that iterates over key according to an integer priority. If the priority is
+    the same for two keys, insertion order is what matters.
+
+    The priority is set when the key/value pair is added. If not set, the highest current priority
+    is used.
+    """
+
+    _data: Dict[KT, VT]
+    _priorities: List[Tuple[int, KT]]
+
+    def __init__(self) -> None:
+        self._data = {}
+        # Tuple of (priority, key)
+        self._priorities = []
+
+    def __getitem__(self, key: KT) -> VT:
+        return self._data[key]
+
+    def __len__(self) -> int:
+        return len(self._data)
+
+    def __iter__(self):
+        yield from (key for _, key in self._priorities)
+
+    def __reversed__(self):
+        yield from (key for _, key in reversed(self._priorities))
+
+    def reversed_keys(self):
+        """Iterates over keys from the highest priority, to the lowest."""
+        return reversed(self)
+
+    def reversed_values(self):
+        """Iterates over values from the highest priority, to the lowest."""
+        yield from (self._data[key] for _, key in reversed(self._priorities))
+
+    def _highest_priority(self) -> int:
+        if not self._priorities:
+            return 0
+        result, _ = self._priorities[-1]
+        return result
+
+    def add(self, key: KT, *, value: VT, priority: Optional[int] = None) -> None:
+        """Adds a key/value pair to the mapping, with a specific priority.
+
+        If the priority is None, then it is assumed to be the highest priority value currently
+        in the container.
+
+        Raises:
+              ValueError: when the same priority is already in the mapping
+        """
+        if priority is None:
+            priority = self._highest_priority()
+
+        if key in self._data:
+            self.remove(key)
+
+        self._priorities.append((priority, key))
+        # We rely on sort being stable
+        self._priorities.sort(key=lambda x: x[0])
+        self._data[key] = value
+        assert len(self._data) == len(self._priorities)
+
+    def remove(self, key: KT) -> VT:
+        """Removes a key from the mapping.
+
+        Returns:
+            The value associated with the key being removed
+
+        Raises:
+            KeyError: if the key is not in the mapping
+        """
+        if key not in self._data:
+            raise KeyError(f"cannot find {key}")
+
+        popped_item = self._data.pop(key)
+        self._priorities = [(p, k) for p, k in self._priorities if k != key]
+        assert len(self._data) == len(self._priorities)
+        return popped_item

lib/spack/spack/__init__.py

@@ -13,6 +13,18 @@
 __version__ = "1.0.0.dev0"
 spack_version = __version__
 
+#: The current Package API version implemented by this version of Spack. The Package API defines
+#: the Python interface for packages as well as the layout of package repositories. The minor
+#: version is incremented when the package API is extended in a backwards-compatible way. The major
+#: version is incremented upon breaking changes. This version is changed independently from the
+#: Spack version.
+package_api_version = (1, 0)
+
+#: The minimum Package API version that this version of Spack is compatible with. This should
+#: always be a tuple of the form ``(major, 0)``, since compatibility with vX.Y implies
+#: compatibility with vX.0.
+min_package_api_version = (1, 0)
+
 
 def __try_int(v):
     try:
@@ -79,4 +91,6 @@ def get_short_version() -> str:
     "get_version",
     "get_spack_commit",
     "get_short_version",
+    "package_api_version",
+    "min_package_api_version",
 ]

lib/spack/spack/build_environment.py

@@ -881,21 +881,6 @@ def get_rpath_deps(pkg: spack.package_base.PackageBase) -> List[spack.spec.Spec]
     return _get_rpath_deps_from_spec(pkg.spec, pkg.transitive_rpaths)
 
 
-def load_external_modules(pkg):
-    """Traverse a package's spec DAG and load any external modules.
-
-    Traverse a package's dependencies and load any external modules
-    associated with them.
-
-    Args:
-        pkg (spack.package_base.PackageBase): package to load deps for
-    """
-    for dep in list(pkg.spec.traverse()):
-        external_modules = dep.external_modules or []
-        for external_module in external_modules:
-            load_module(external_module)
-
-
 def setup_package(pkg, dirty, context: Context = Context.BUILD):
     """Execute all environment setup routines."""
     if context not in (Context.BUILD, Context.TEST):
@@ -946,7 +931,7 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD):
         for mod in pkg.compiler.modules:
             load_module(mod)
 
-    load_external_modules(pkg)
+    load_external_modules(setup_context)
 
     # Make sure nothing's strange about the Spack environment.
     validate(env_mods, tty.warn)
@@ -1235,6 +1220,21 @@ def _make_runnable(self, dep: spack.spec.Spec, env: EnvironmentModifications):
                 env.prepend_path("PATH", bin_dir)
 
 
+def load_external_modules(context: SetupContext) -> None:
+    """Traverse a package's spec DAG and load any external modules.
+
+    Traverse a package's dependencies and load any external modules
+    associated with them.
+
+    Args:
+        context: A populated SetupContext object
+    """
+    for spec, _ in context.external:
+        external_modules = spec.external_modules or []
+        for external_module in external_modules:
+            load_module(external_module)
+
+
 def _setup_pkg_and_run(
     serialized_pkg: "spack.subprocess_context.PackageInstallContext",
     function: Callable,

lib/spack/spack/ci/__init__.py

@@ -6,6 +6,7 @@
 import codecs
 import json
 import os
+import pathlib
 import re
 import shutil
 import stat
@@ -23,7 +24,6 @@
 
 import spack
 import spack.binary_distribution as bindist
-import spack.builder
 import spack.concretize
 import spack.config as cfg
 import spack.environment as ev
@@ -33,6 +33,7 @@
 import spack.paths
 import spack.repo
 import spack.spec
+import spack.store
 import spack.util.git
 import spack.util.gpg as gpg_util
 import spack.util.spack_yaml as syaml
@@ -224,7 +225,7 @@ def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision:
 
 def _format_pruning_message(spec: spack.spec.Spec, prune: bool, reasons: List[str]) -> str:
     reason_msg = ", ".join(reasons)
-    spec_fmt = "{name}{@version}{%compiler}{/hash:7}"
+    spec_fmt = "{name}{@version}{/hash:7}{%compiler}"
 
     if not prune:
         status = colorize("@*g{[x]}  ")
@@ -581,22 +582,25 @@ def copy_stage_logs_to_artifacts(job_spec: spack.spec.Spec, job_log_dir: str) ->
     tty.debug(f"job spec: {job_spec}")
 
     try:
-        pkg_cls = spack.repo.PATH.get_pkg_class(job_spec.name)
-        job_pkg = pkg_cls(job_spec)
-        tty.debug(f"job package: {job_pkg}")
-    except AssertionError:
-        msg = f"Cannot copy stage logs: job spec ({job_spec}) must be concrete"
-        tty.error(msg)
+        package_metadata_root = pathlib.Path(spack.store.STORE.layout.metadata_path(job_spec))
+    except spack.error.SpackError as e:
+        tty.error(f"Cannot copy logs: {str(e)}")
         return
 
-    stage_dir = job_pkg.stage.path
-    tty.debug(f"stage dir: {stage_dir}")
-    for file in [
-        job_pkg.log_path,
-        job_pkg.env_mods_path,
-        *spack.builder.create(job_pkg).archive_files,
-    ]:
-        copy_files_to_artifacts(file, job_log_dir)
+    # Get the package's archived files
+    archive_files = []
+    archive_root = package_metadata_root / "archived-files"
+    if archive_root.is_dir():
+        archive_files = [f for f in archive_root.rglob("*") if f.is_file()]
+    else:
+        msg = "Cannot copy package archived files: archived-files must be a directory"
+        tty.warn(msg)
+
+    build_log_zipped = package_metadata_root / "spack-build-out.txt.gz"
+    build_env_mods = package_metadata_root / "spack-build-env.txt"
+
+    for f in [build_log_zipped, build_env_mods, *archive_files]:
+        copy_files_to_artifacts(str(f), job_log_dir)
 
 
 def copy_test_logs_to_artifacts(test_stage, job_test_dir):

lib/spack/spack/cmd/__init__.py

@@ -330,7 +330,7 @@ def ensure_single_spec_or_die(spec, matching_specs):
     if len(matching_specs) <= 1:
         return
 
-    format_string = "{name}{@version}{%compiler.name}{@compiler.version}{ arch=architecture}"
+    format_string = "{name}{@version}{ arch=architecture} {%compiler.name}{@compiler.version}"
     args = ["%s matches multiple packages." % spec, "Matching packages:"]
     args += [
         colorize("  @K{%s} " % s.dag_hash(7)) + s.cformat(format_string) for s in matching_specs
@@ -471,12 +471,11 @@ def get_arg(name, default=None):
         nfmt = "{fullname}" if namespaces else "{name}"
         ffmt = ""
         if full_compiler or flags:
-            ffmt += "{%compiler.name}"
+            ffmt += "{compiler_flags} {%compiler.name}"
             if full_compiler:
                 ffmt += "{@compiler.version}"
-            ffmt += " {compiler_flags}"
         vfmt = "{variants}" if variants else ""
-        format_string = nfmt + "{@version}" + ffmt + vfmt
+        format_string = nfmt + "{@version}" + vfmt + ffmt
 
     def fmt(s, depth=0):
         """Formatter function for all output specs"""

lib/spack/spack/cmd/ci.py

@@ -427,7 +427,7 @@ def ci_rebuild(args):
 
     # Arguments when installing the root from sources
     deps_install_args = install_args + ["--only=dependencies"]
-    root_install_args = install_args + ["--keep-stage", "--only=package"]
+    root_install_args = install_args + ["--only=package"]
 
     if cdash_handler:
         # Add additional arguments to `spack install` for CDash reporting.
@@ -464,8 +464,7 @@ def ci_rebuild(args):
                 job_spec.to_dict(hash=ht.dag_hash),
             )
 
-    # We generated the "spack install ..." command to "--keep-stage", copy
-    # any logs from the staging directory to artifacts now
+    # Copy logs and archived files from the install metadata (.spack) directory to artifacts now
     spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir)
 
     # If the installation succeeded and we're running stand-alone tests for

lib/spack/spack/cmd/common/arguments.py

@@ -528,7 +528,6 @@ def __call__(self, parser, namespace, values, option_string):
         # the const from the constructor or a value from the CLI.
         # Note that this is only called if the argument is actually
         # specified on the command line.
-        spack.config.CONFIG.ensure_scope_ordering()
         spack.config.set(self.config_path, self.const, scope="command_line")
 
 

lib/spack/spack/cmd/config.py

@@ -350,9 +350,12 @@ def _config_change(config_path, match_spec_str=None):
                 if spack.config.get(key_path, scope=scope):
                     ideal_scope_to_modify = scope
                     break
+            # If we find our key in a specific scope, that's the one we want
+            # to modify. Otherwise we use the default write scope.
+            write_scope = ideal_scope_to_modify or spack.config.default_modify_scope()
 
             update_path = f"{key_path}:[{str(spec)}]"
-            spack.config.add(update_path, scope=ideal_scope_to_modify)
+            spack.config.add(update_path, scope=write_scope)
     else:
         raise ValueError("'config change' can currently only change 'require' sections")
 

lib/spack/spack/cmd/dependencies.py

@@ -55,7 +55,7 @@ def dependencies(parser, args):
         env = ev.active_environment()
         spec = spack.cmd.disambiguate_spec(specs[0], env)
 
-        format_string = "{name}{@version}{%compiler}{/hash:7}"
+        format_string = "{name}{@version}{/hash:7}{%compiler}"
         if sys.stdout.isatty():
             tty.msg("Dependencies of %s" % spec.format(format_string, color=True))
         deps = spack.store.STORE.db.installed_relatives(

lib/spack/spack/cmd/dependents.py

@@ -93,7 +93,7 @@ def dependents(parser, args):
         env = ev.active_environment()
         spec = spack.cmd.disambiguate_spec(specs[0], env)
 
-        format_string = "{name}{@version}{%compiler}{/hash:7}"
+        format_string = "{name}{@version}{/hash:7}{%compiler}"
         if sys.stdout.isatty():
             tty.msg("Dependents of %s" % spec.cformat(format_string))
         deps = spack.store.STORE.db.installed_relatives(spec, "parents", args.transitive)

lib/spack/spack/cmd/help.py

@@ -73,7 +73,7 @@
       boxlib @B{dim=2}                  boxlib built for 2 dimensions
       libdwarf @g{%intel} ^libelf@g{%gcc}
           libdwarf, built with intel compiler, linked to libelf built with gcc
-      mvapich2 @g{%gcc} @B{fabrics=psm,mrail,sock}
+      mvapich2 @B{fabrics=psm,mrail,sock} @g{%gcc}
           mvapich2, built with gcc compiler, with support for multiple fabrics
 """
 

lib/spack/spack/cmd/modules/__init__.py

@@ -383,8 +383,10 @@ def modules_cmd(parser, args, module_type, callbacks=callbacks):
         query = " ".join(str(s) for s in args.constraint_specs)
         msg = f"the constraint '{query}' matches multiple packages:\n"
         for s in specs:
-            spec_fmt = "{hash:7} {name}{@version}{%compiler}"
-            spec_fmt += "{compiler_flags}{variants}{arch=architecture}"
+            spec_fmt = (
+                "{hash:7} {name}{@version}{compiler_flags}{variants}"
+                "{arch=architecture} {%compiler}"
+            )
             msg += "\t" + s.cformat(spec_fmt) + "\n"
         tty.die(msg, "In this context exactly *one* match is needed.")
 

lib/spack/spack/cmd/style.py

@@ -6,8 +6,9 @@
 import os
 import re
 import sys
+import warnings
 from itertools import islice, zip_longest
-from typing import Dict, List, Optional
+from typing import Callable, Dict, List, Optional
 
 import llnl.util.tty as tty
 import llnl.util.tty.color as color
@@ -16,6 +17,9 @@
 import spack.paths
 import spack.repo
 import spack.util.git
+import spack.util.spack_yaml
+from spack.spec_parser import SPEC_TOKENIZER, SpecTokens
+from spack.tokenize import Token
 from spack.util.executable import Executable, which
 
 description = "runs source code style checks on spack"
@@ -198,6 +202,13 @@ def setup_parser(subparser):
         action="append",
         help="specify tools to skip (choose from %s)" % ", ".join(tool_names),
     )
+    subparser.add_argument(
+        "--spec-strings",
+        action="store_true",
+        help="upgrade spec strings in Python, JSON and YAML files for compatibility with Spack "
+        "v1.0 and v0.x. Example: spack style --spec-strings $(git ls-files). Note: this flag "
+        "will be removed in Spack v1.0.",
+    )
 
     subparser.add_argument("files", nargs=argparse.REMAINDER, help="specific files to check")
 
@@ -507,7 +518,196 @@ def _bootstrap_dev_dependencies():
         spack.bootstrap.ensure_environment_dependencies()
 
 
+IS_PROBABLY_COMPILER = re.compile(r"%[a-zA-Z_][a-zA-Z0-9\-]")
+
+
+def _spec_str_reorder_compiler(idx: int, blocks: List[List[Token]]) -> None:
+    # only move the compiler to the back if it exists and is not already at the end
+    if not 0 <= idx < len(blocks) - 1:
+        return
+    # if there's only whitespace after the compiler, don't move it
+    if all(token.kind == SpecTokens.WS for block in blocks[idx + 1 :] for token in block):
+        return
+    # rotate left and always add at least one WS token between compiler and previous token
+    compiler_block = blocks.pop(idx)
+    if compiler_block[0].kind != SpecTokens.WS:
+        compiler_block.insert(0, Token(SpecTokens.WS, " "))
+    # delete the WS tokens from the new first block if it was at the very start, to prevent leading
+    # WS tokens.
+    while idx == 0 and blocks[0][0].kind == SpecTokens.WS:
+        blocks[0].pop(0)
+    blocks.append(compiler_block)
+
+
+def _spec_str_format(spec_str: str) -> Optional[str]:
+    """Given any string, try to parse as spec string, and rotate the compiler token to the end
+    of each spec instance. Returns the formatted string if it was changed, otherwise None."""
+    # We parse blocks of tokens that include leading whitespace, and move the compiler block to
+    # the end when we hit a dependency ^... or the end of a string.
+    # [@3.1][ +foo][ +bar][ %gcc@3.1][ +baz]
+    # [@3.1][ +foo][ +bar][ +baz][ %gcc@3.1]
+
+    current_block: List[Token] = []
+    blocks: List[List[Token]] = []
+    compiler_block_idx = -1
+    in_edge_attr = False
+
+    for token in SPEC_TOKENIZER.tokenize(spec_str):
+        if token.kind == SpecTokens.UNEXPECTED:
+            # parsing error, we cannot fix this string.
+            return None
+        elif token.kind in (SpecTokens.COMPILER, SpecTokens.COMPILER_AND_VERSION):
+            # multiple compilers are not supported in Spack v0.x, so early return
+            if compiler_block_idx != -1:
+                return None
+            current_block.append(token)
+            blocks.append(current_block)
+            current_block = []
+            compiler_block_idx = len(blocks) - 1
+        elif token.kind in (
+            SpecTokens.START_EDGE_PROPERTIES,
+            SpecTokens.DEPENDENCY,
+            SpecTokens.UNQUALIFIED_PACKAGE_NAME,
+            SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME,
+        ):
+            _spec_str_reorder_compiler(compiler_block_idx, blocks)
+            compiler_block_idx = -1
+            if token.kind == SpecTokens.START_EDGE_PROPERTIES:
+                in_edge_attr = True
+            current_block.append(token)
+            blocks.append(current_block)
+            current_block = []
+        elif token.kind == SpecTokens.END_EDGE_PROPERTIES:
+            in_edge_attr = False
+            current_block.append(token)
+            blocks.append(current_block)
+            current_block = []
+        elif in_edge_attr:
+            current_block.append(token)
+        elif token.kind in (
+            SpecTokens.VERSION_HASH_PAIR,
+            SpecTokens.GIT_VERSION,
+            SpecTokens.VERSION,
+            SpecTokens.PROPAGATED_BOOL_VARIANT,
+            SpecTokens.BOOL_VARIANT,
+            SpecTokens.PROPAGATED_KEY_VALUE_PAIR,
+            SpecTokens.KEY_VALUE_PAIR,
+            SpecTokens.DAG_HASH,
+        ):
+            current_block.append(token)
+            blocks.append(current_block)
+            current_block = []
+        elif token.kind == SpecTokens.WS:
+            current_block.append(token)
+        else:
+            raise ValueError(f"unexpected token {token}")
+
+    if current_block:
+        blocks.append(current_block)
+    _spec_str_reorder_compiler(compiler_block_idx, blocks)
+
+    new_spec_str = "".join(token.value for block in blocks for token in block)
+    return new_spec_str if spec_str != new_spec_str else None
+
+
+SpecStrHandler = Callable[[str, int, int, str, str], None]
+
+
+def _spec_str_default_handler(path: str, line: int, col: int, old: str, new: str):
+    """A SpecStrHandler that prints formatted spec strings and their locations."""
+    print(f"{path}:{line}:{col}: `{old}` -> `{new}`")
+
+
+def _spec_str_fix_handler(path: str, line: int, col: int, old: str, new: str):
+    """A SpecStrHandler that updates formatted spec strings in files."""
+    with open(path, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+    new_line = lines[line - 1].replace(old, new)
+    if new_line == lines[line - 1]:
+        tty.warn(f"{path}:{line}:{col}: could not apply fix: `{old}` -> `{new}`")
+        return
+    lines[line - 1] = new_line
+    print(f"{path}:{line}:{col}: fixed `{old}` -> `{new}`")
+    with open(path, "w", encoding="utf-8") as f:
+        f.writelines(lines)
+
+
+def _spec_str_ast(path: str, tree: ast.AST, handler: SpecStrHandler) -> None:
+    """Walk the AST of a Python file and apply handler to formatted spec strings."""
+    has_constant = sys.version_info >= (3, 8)
+    for node in ast.walk(tree):
+        if has_constant and isinstance(node, ast.Constant) and isinstance(node.value, str):
+            current_str = node.value
+        elif not has_constant and isinstance(node, ast.Str):
+            current_str = node.s
+        else:
+            continue
+        if not IS_PROBABLY_COMPILER.search(current_str):
+            continue
+        new = _spec_str_format(current_str)
+        if new is not None:
+            handler(path, node.lineno, node.col_offset, current_str, new)
+
+
+def _spec_str_json_and_yaml(path: str, data: dict, handler: SpecStrHandler) -> None:
+    """Walk a YAML or JSON data structure and apply handler to formatted spec strings."""
+    queue = [data]
+    seen = set()
+
+    while queue:
+        current = queue.pop(0)
+        if id(current) in seen:
+            continue
+        seen.add(id(current))
+        if isinstance(current, dict):
+            queue.extend(current.values())
+            queue.extend(current.keys())
+        elif isinstance(current, list):
+            queue.extend(current)
+        elif isinstance(current, str) and IS_PROBABLY_COMPILER.search(current):
+            new = _spec_str_format(current)
+            if new is not None:
+                mark = getattr(current, "_start_mark", None)
+                if mark:
+                    line, col = mark.line + 1, mark.column + 1
+                else:
+                    line, col = 0, 0
+                handler(path, line, col, current, new)
+
+
+def _check_spec_strings(
+    paths: List[str], handler: SpecStrHandler = _spec_str_default_handler
+) -> None:
+    """Open Python, JSON and YAML files, and format their string literals that look like spec
+    strings. A handler is called for each formatting, which can be used to print or apply fixes."""
+    for path in paths:
+        is_json_or_yaml = path.endswith(".json") or path.endswith(".yaml") or path.endswith(".yml")
+        is_python = path.endswith(".py")
+        if not is_json_or_yaml and not is_python:
+            continue
+
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                # skip files that are likely too large to be user code or config
+                if os.fstat(f.fileno()).st_size > 1024 * 1024:
+                    warnings.warn(f"skipping {path}: too large.")
+                    continue
+                if is_json_or_yaml:
+                    _spec_str_json_and_yaml(path, spack.util.spack_yaml.load_config(f), handler)
+                elif is_python:
+                    _spec_str_ast(path, ast.parse(f.read()), handler)
+        except (OSError, spack.util.spack_yaml.SpackYAMLError, SyntaxError, ValueError):
+            warnings.warn(f"skipping {path}")
+            continue
+
+
 def style(parser, args):
+    if args.spec_strings:
+        if not args.files:
+            tty.die("No files provided to check spec strings.")
+        handler = _spec_str_fix_handler if args.fix else _spec_str_default_handler
+        return _check_spec_strings(args.files, handler)
+
     # save initial working directory for relativizing paths later
     args.initial_working_dir = os.getcwd()
 

lib/spack/spack/cmd/verify.py

@@ -2,35 +2,48 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import argparse
+import io
+from typing import List, Optional
 
 import llnl.util.tty as tty
+from llnl.string import plural
+from llnl.util.filesystem import visit_directory_tree
 
 import spack.cmd
 import spack.environment as ev
+import spack.spec
 import spack.store
 import spack.verify
+import spack.verify_libraries
+from spack.cmd.common import arguments
 
-description = "check that all spack packages are on disk as installed"
+description = "verify spack installations on disk"
 section = "admin"
 level = "long"
 
+MANIFEST_SUBPARSER: Optional[argparse.ArgumentParser] = None
 
-def setup_parser(subparser):
-    setup_parser.parser = subparser
 
-    subparser.add_argument(
+def setup_parser(subparser: argparse.ArgumentParser):
+    global MANIFEST_SUBPARSER
+    sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="verify_command")
+
+    MANIFEST_SUBPARSER = sp.add_parser(
+        "manifest", help=verify_manifest.__doc__, description=verify_manifest.__doc__
+    )
+    MANIFEST_SUBPARSER.add_argument(
         "-l", "--local", action="store_true", help="verify only locally installed packages"
     )
-    subparser.add_argument(
+    MANIFEST_SUBPARSER.add_argument(
         "-j", "--json", action="store_true", help="ouptut json-formatted errors"
     )
-    subparser.add_argument("-a", "--all", action="store_true", help="verify all packages")
-    subparser.add_argument(
+    MANIFEST_SUBPARSER.add_argument("-a", "--all", action="store_true", help="verify all packages")
+    MANIFEST_SUBPARSER.add_argument(
         "specs_or_files", nargs=argparse.REMAINDER, help="specs or files to verify"
     )
 
-    type = subparser.add_mutually_exclusive_group()
-    type.add_argument(
+    manifest_sp_type = MANIFEST_SUBPARSER.add_mutually_exclusive_group()
+    manifest_sp_type.add_argument(
         "-s",
         "--specs",
         action="store_const",
@@ -39,7 +52,7 @@ def setup_parser(subparser):
         default="specs",
         help="treat entries as specs (default)",
     )
-    type.add_argument(
+    manifest_sp_type.add_argument(
         "-f",
         "--files",
         action="store_const",
@@ -49,14 +62,67 @@ def setup_parser(subparser):
         help="treat entries as absolute filenames\n\ncannot be used with '-a'",
     )
 
+    libraries_subparser = sp.add_parser(
+        "libraries", help=verify_libraries.__doc__, description=verify_libraries.__doc__
+    )
+
+    arguments.add_common_arguments(libraries_subparser, ["constraint"])
+
 
 def verify(parser, args):
+    cmd = args.verify_command
+    if cmd == "libraries":
+        return verify_libraries(args)
+    elif cmd == "manifest":
+        return verify_manifest(args)
+    parser.error("invalid verify subcommand")
+
+
+def verify_libraries(args):
+    """verify that shared libraries of install packages can be located in rpaths (Linux only)"""
+    specs_from_db = [s for s in args.specs(installed=True) if not s.external]
+
+    tty.info(f"Checking {len(specs_from_db)} packages for shared library resolution")
+
+    errors = 0
+    for spec in specs_from_db:
+        try:
+            pkg = spec.package
+        except Exception:
+            tty.warn(f"Skipping {spec.cformat('{name}{@version}{/hash}')} due to missing package")
+        error_msg = _verify_libraries(spec, pkg.unresolved_libraries)
+        if error_msg is not None:
+            errors += 1
+            tty.error(error_msg)
+
+    if errors:
+        tty.error(f"Cannot resolve shared libraries in {plural(errors, 'package')}")
+        return 1
+
+
+def _verify_libraries(spec: spack.spec.Spec, unresolved_libraries: List[str]) -> Optional[str]:
+    """Go over the prefix of the installed spec and verify its shared libraries can be resolved."""
+    visitor = spack.verify_libraries.ResolveSharedElfLibDepsVisitor(
+        [*spack.verify_libraries.ALLOW_UNRESOLVED, *unresolved_libraries]
+    )
+    visit_directory_tree(spec.prefix, visitor)
+
+    if not visitor.problems:
+        return None
+
+    output = io.StringIO()
+    visitor.write(output, indent=4, brief=True)
+    message = output.getvalue().rstrip()
+    return f"{spec.cformat('{name}{@version}{/hash}')}: {spec.prefix}:\n{message}"
+
+
+def verify_manifest(args):
+    """verify that install directories have not been modified since installation"""
     local = args.local
 
     if args.type == "files":
         if args.all:
-            setup_parser.parser.print_help()
-            return 1
+            MANIFEST_SUBPARSER.error("cannot use --all with --files")
 
         for file in args.specs_or_files:
             results = spack.verify.check_file_manifest(file)
@@ -87,8 +153,7 @@ def verify(parser, args):
         env = ev.active_environment()
         specs = list(map(lambda x: spack.cmd.disambiguate_spec(x, env, local=local), spec_args))
     else:
-        setup_parser.parser.print_help()
-        return 1
+        MANIFEST_SUBPARSER.error("use --all or specify specs to verify")
 
     for spec in specs:
         tty.debug("Verifying package %s")

lib/spack/spack/config.py

@@ -32,9 +32,10 @@
 import copy
 import functools
 import os
+import os.path
 import re
 import sys
-from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, Generator, List, NamedTuple, Optional, Tuple, Union
 
 import jsonschema
 
@@ -42,7 +43,6 @@
 
 import spack.error
 import spack.paths
-import spack.platforms
 import spack.schema
 import spack.schema.bootstrap
 import spack.schema.cdash
@@ -54,17 +54,20 @@
 import spack.schema.develop
 import spack.schema.env
 import spack.schema.env_vars
+import spack.schema.include
+import spack.schema.merged
 import spack.schema.mirrors
 import spack.schema.modules
 import spack.schema.packages
 import spack.schema.repos
 import spack.schema.upstreams
 import spack.schema.view
-
-# Hacked yaml for configuration files preserves line numbers.
+import spack.util.remote_file_cache as rfc_util
 import spack.util.spack_yaml as syaml
-import spack.util.web as web_util
 from spack.util.cpus import cpus_available
+from spack.util.spack_yaml import get_mark_from_yaml_data
+
+from .enums import ConfigScopePriority
 
 #: Dict from section names -> schema for that section
 SECTION_SCHEMAS: Dict[str, Any] = {
@@ -72,6 +75,7 @@
     "concretizer": spack.schema.concretizer.schema,
     "definitions": spack.schema.definitions.schema,
     "env_vars": spack.schema.env_vars.schema,
+    "include": spack.schema.include.schema,
     "view": spack.schema.view.schema,
     "develop": spack.schema.develop.schema,
     "mirrors": spack.schema.mirrors.schema,
@@ -119,6 +123,17 @@
 #: Type used for raw YAML configuration
 YamlConfigDict = Dict[str, Any]
 
+#: prefix for name of included configuration scopes
+INCLUDE_SCOPE_PREFIX = "include"
+
+#: safeguard for recursive includes -- maximum include depth
+MAX_RECURSIVE_INCLUDES = 100
+
+
+def _include_cache_location():
+    """Location to cache included configuration files."""
+    return os.path.join(spack.paths.user_cache_path, "includes")
+
 
 class ConfigScope:
     def __init__(self, name: str) -> None:
@@ -126,6 +141,25 @@ def __init__(self, name: str) -> None:
         self.writable = False
         self.sections = syaml.syaml_dict()
 
+        #: names of any included scopes
+        self._included_scopes: Optional[List["ConfigScope"]] = None
+
+    @property
+    def included_scopes(self) -> List["ConfigScope"]:
+        """Memoized list of included scopes, in the order they appear in this scope."""
+        if self._included_scopes is None:
+            self._included_scopes = []
+
+            includes = self.get_section("include")
+            if includes:
+                include_paths = [included_path(data) for data in includes["include"]]
+                for path in include_paths:
+                    included_scope = include_path_scope(path)
+                    if included_scope:
+                        self._included_scopes.append(included_scope)
+
+        return self._included_scopes
+
     def get_section_filename(self, section: str) -> str:
         raise NotImplementedError
 
@@ -408,26 +442,18 @@ def _method(self, *args, **kwargs):
     return _method
 
 
-class Configuration:
-    """A full Spack configuration, from a hierarchy of config files.
+ScopeWithOptionalPriority = Union[ConfigScope, Tuple[int, ConfigScope]]
+ScopeWithPriority = Tuple[int, ConfigScope]
 
-    This class makes it easy to add a new scope on top of an existing one.
-    """
+
+class Configuration:
+    """A hierarchical configuration, merging a number of scopes at different priorities."""
 
     # convert to typing.OrderedDict when we drop 3.6, or OrderedDict when we reach 3.9
-    scopes: Dict[str, ConfigScope]
+    scopes: lang.PriorityOrderedMapping[str, ConfigScope]
 
-    def __init__(self, *scopes: ConfigScope) -> None:
-        """Initialize a configuration with an initial list of scopes.
-
-        Args:
-            scopes: list of scopes to add to this
-                Configuration, ordered from lowest to highest precedence
-
-        """
-        self.scopes = collections.OrderedDict()
-        for scope in scopes:
-            self.push_scope(scope)
+    def __init__(self) -> None:
+        self.scopes = lang.PriorityOrderedMapping()
         self.format_updates: Dict[str, List[ConfigScope]] = collections.defaultdict(list)
 
     def ensure_unwrapped(self) -> "Configuration":
@@ -435,36 +461,59 @@ def ensure_unwrapped(self) -> "Configuration":
         return self
 
     def highest(self) -> ConfigScope:
-        """Scope with highest precedence"""
-        return next(reversed(self.scopes.values()))  # type: ignore
+        """Scope with the highest precedence"""
+        return next(self.scopes.reversed_values())  # type: ignore
 
     @_config_mutator
-    def ensure_scope_ordering(self):
-        """Ensure that scope order matches documented precedent"""
-        # FIXME: We also need to consider that custom configurations and other orderings
-        # may not be preserved correctly
-        if "command_line" in self.scopes:
-            # TODO (when dropping python 3.6): self.scopes.move_to_end
-            self.scopes["command_line"] = self.remove_scope("command_line")
+    def push_scope(
+        self, scope: ConfigScope, priority: Optional[int] = None, _depth: int = 0
+    ) -> None:
+        """Adds a scope to the Configuration, at a given priority.
 
-    @_config_mutator
-    def push_scope(self, scope: ConfigScope) -> None:
-        """Add a higher precedence scope to the Configuration."""
-        tty.debug(f"[CONFIGURATION: PUSH SCOPE]: {str(scope)}", level=2)
-        self.scopes[scope.name] = scope
+        If a priority is not given, it is assumed to be the current highest priority.
 
-    @_config_mutator
-    def pop_scope(self) -> ConfigScope:
-        """Remove the highest precedence scope and return it."""
-        name, scope = self.scopes.popitem(last=True)  # type: ignore[call-arg]
-        tty.debug(f"[CONFIGURATION: POP SCOPE]: {str(scope)}", level=2)
-        return scope
+        Args:
+            scope: scope to be added
+            priority: priority of the scope
+        """
+        # TODO: As a follow on to #48784, change this to create a graph of the
+        # TODO: includes AND ensure properly sorted such that the order included
+        # TODO: at the highest level is reflected in the value of an option that
+        # TODO: is set in multiple included files.
+        # before pushing the scope itself, push any included scopes recursively, at same priority
+        for included_scope in reversed(scope.included_scopes):
+            if _depth + 1 > MAX_RECURSIVE_INCLUDES:  # make sure we're not recursing endlessly
+                mark = ""
+                if hasattr(included_scope, "path") and syaml.marked(included_scope.path):
+                    mark = included_scope.path._start_mark  # type: ignore
+                raise RecursiveIncludeError(
+                    f"Maximum include recursion exceeded in {included_scope.name}", str(mark)
+                )
+
+            # record this inclusion so that remove_scope() can use it
+            self.push_scope(included_scope, priority=priority, _depth=_depth + 1)
+
+        tty.debug(f"[CONFIGURATION: PUSH SCOPE]: {str(scope)}, priority={priority}", level=2)
+        self.scopes.add(scope.name, value=scope, priority=priority)
 
     @_config_mutator
     def remove_scope(self, scope_name: str) -> Optional[ConfigScope]:
-        """Remove scope by name; has no effect when ``scope_name`` does not exist"""
-        scope = self.scopes.pop(scope_name, None)
-        tty.debug(f"[CONFIGURATION: POP SCOPE]: {str(scope)}", level=2)
+        """Removes a scope by name, and returns it. If the scope does not exist, returns None."""
+
+        try:
+            scope = self.scopes.remove(scope_name)
+            tty.debug(f"[CONFIGURATION: REMOVE SCOPE]: {str(scope)}", level=2)
+        except KeyError as e:
+            tty.debug(f"[CONFIGURATION: REMOVE SCOPE]: {e}", level=2)
+            return None
+
+        # transitively remove included scopes
+        for included_scope in scope.included_scopes:
+            assert (
+                included_scope.name in self.scopes
+            ), f"Included scope '{included_scope.name}' was never added to configuration!"
+            self.remove_scope(included_scope.name)
+
         return scope
 
     @property
@@ -473,15 +522,13 @@ def writable_scopes(self) -> Generator[ConfigScope, None, None]:
         return (s for s in self.scopes.values() if s.writable)
 
     def highest_precedence_scope(self) -> ConfigScope:
-        """Writable scope with highest precedence."""
-        return next(s for s in reversed(self.scopes.values()) if s.writable)  # type: ignore
+        """Writable scope with the highest precedence."""
+        return next(s for s in self.scopes.reversed_values() if s.writable)
 
     def highest_precedence_non_platform_scope(self) -> ConfigScope:
-        """Writable non-platform scope with highest precedence"""
+        """Writable non-platform scope with the highest precedence"""
         return next(
-            s
-            for s in reversed(self.scopes.values())  # type: ignore
-            if s.writable and not s.is_platform_dependent
+            s for s in self.scopes.reversed_values() if s.writable and not s.is_platform_dependent
         )
 
     def matching_scopes(self, reg_expr) -> List[ConfigScope]:
@@ -748,7 +795,7 @@ def override(
     """
     if isinstance(path_or_scope, ConfigScope):
         overrides = path_or_scope
-        CONFIG.push_scope(path_or_scope)
+        CONFIG.push_scope(path_or_scope, priority=None)
     else:
         base_name = _OVERRIDES_BASE_NAME
         # Ensure the new override gets a unique scope name
@@ -762,7 +809,7 @@ def override(
                 break
 
         overrides = InternalConfigScope(scope_name)
-        CONFIG.push_scope(overrides)
+        CONFIG.push_scope(overrides, priority=None)
         CONFIG.set(path_or_scope, value, scope=scope_name)
 
     try:
@@ -772,13 +819,86 @@ def override(
         assert scope is overrides
 
 
-def _add_platform_scope(cfg: Configuration, name: str, path: str, writable: bool = True) -> None:
+def _add_platform_scope(
+    cfg: Configuration, name: str, path: str, priority: ConfigScopePriority, writable: bool = True
+) -> None:
     """Add a platform-specific subdirectory for the current platform."""
+    import spack.platforms  # circular dependency
+
     platform = spack.platforms.host().name
     scope = DirectoryConfigScope(
         f"{name}/{platform}", os.path.join(path, platform), writable=writable
     )
-    cfg.push_scope(scope)
+    cfg.push_scope(scope, priority=priority)
+
+
+#: Class for the relevance of an optional path conditioned on a limited
+#: python code that evaluates to a boolean and or explicit specification
+#: as optional.
+class IncludePath(NamedTuple):
+    path: str
+    when: str
+    sha256: str
+    optional: bool
+
+
+def included_path(entry: Union[str, dict]) -> IncludePath:
+    """Convert the included path entry into an IncludePath.
+
+    Args:
+        entry: include configuration entry
+
+    Returns: converted entry, where an empty ``when`` means the path is
+        not conditionally included
+    """
+    if isinstance(entry, str):
+        return IncludePath(path=entry, sha256="", when="", optional=False)
+
+    path = entry["path"]
+    sha256 = entry.get("sha256", "")
+    when = entry.get("when", "")
+    optional = entry.get("optional", False)
+    return IncludePath(path=path, sha256=sha256, when=when, optional=optional)
+
+
+def include_path_scope(include: IncludePath) -> Optional[ConfigScope]:
+    """Instantiate an appropriate configuration scope for the given path.
+
+    Args:
+        include: optional include path
+
+    Returns: configuration scope
+
+    Raises:
+        ValueError: included path has an unsupported URL scheme, is required
+            but does not exist; configuration stage directory argument is missing
+        ConfigFileError: unable to access remote configuration file(s)
+    """
+    # circular dependencies
+    import spack.spec
+
+    if (not include.when) or spack.spec.eval_conditional(include.when):
+        config_path = rfc_util.local_path(include.path, include.sha256, _include_cache_location)
+        if not config_path:
+            raise ConfigFileError(f"Unable to fetch remote configuration from {include.path}")
+
+        if os.path.isdir(config_path):
+            # directories are treated as regular ConfigScopes
+            config_name = f"{INCLUDE_SCOPE_PREFIX}:{os.path.basename(config_path)}"
+            tty.debug(f"Creating DirectoryConfigScope {config_name} for '{config_path}'")
+            return DirectoryConfigScope(config_name, config_path)
+
+        if os.path.exists(config_path):
+            # files are assumed to be SingleFileScopes
+            config_name = f"{INCLUDE_SCOPE_PREFIX}:{config_path}"
+            tty.debug(f"Creating SingleFileScope {config_name} for '{config_path}'")
+            return SingleFileScope(config_name, config_path, spack.schema.merged.schema)
+
+        if not include.optional:
+            path = f" at ({config_path})" if config_path != include.path else ""
+            raise ValueError(f"Required path ({include.path}) does not exist{path}")
+
+    return None
 
 
 def config_paths_from_entry_points() -> List[Tuple[str, str]]:
@@ -806,18 +926,17 @@ def config_paths_from_entry_points() -> List[Tuple[str, str]]:
     return config_paths
 
 
-def create() -> Configuration:
+def create_incremental() -> Generator[Configuration, None, None]:
     """Singleton Configuration instance.
 
     This constructs one instance associated with this module and returns
     it. It is bundled inside a function so that configuration can be
     initialized lazily.
     """
-    cfg = Configuration()
-
     # first do the builtin, hardcoded defaults
-    builtin = InternalConfigScope("_builtin", CONFIG_DEFAULTS)
-    cfg.push_scope(builtin)
+    cfg = create_from(
+        (ConfigScopePriority.BUILTIN, InternalConfigScope("_builtin", CONFIG_DEFAULTS))
+    )
 
     # Builtin paths to configuration files in Spack
     configuration_paths = [
@@ -847,16 +966,29 @@ def create() -> Configuration:
 
     # add each scope and its platform-specific directory
     for name, path in configuration_paths:
-        cfg.push_scope(DirectoryConfigScope(name, path))
+        cfg.push_scope(DirectoryConfigScope(name, path), priority=ConfigScopePriority.CONFIG_FILES)
+        # Each scope can have per-platform overrides in subdirectories
+        _add_platform_scope(cfg, name, path, priority=ConfigScopePriority.CONFIG_FILES)
 
-        # Each scope can have per-platfom overrides in subdirectories
-        _add_platform_scope(cfg, name, path)
+        # yield the config incrementally so that each config level's init code can get
+        # data from the one below. This can be tricky, but it enables us to have a
+        # single unified config system.
+        #
+        # TODO: think about whether we want to restrict what types of config can be used
+        #     at each level. e.g., we may want to just more forcibly disallow remote
+        #     config (which uses ssl and other config options) for some of the scopes,
+        #     to make the bootstrap issues more explicit, even if allowing config scope
+        #     init to reference lower scopes is more flexible.
+        yield cfg
 
-    return cfg
+
+def create() -> Configuration:
+    """Create a configuration using create_incremental(), return the last yielded result."""
+    return list(create_incremental())[-1]
 
 
 #: This is the singleton configuration instance for Spack.
-CONFIG: Configuration = lang.Singleton(create)  # type: ignore
+CONFIG: Configuration = lang.Singleton(create_incremental)  # type: ignore
 
 
 def add_from_file(filename: str, scope: Optional[str] = None) -> None:
@@ -952,10 +1084,11 @@ def set(path: str, value: Any, scope: Optional[str] = None) -> None:
 
     Accepts the path syntax described in ``get()``.
     """
-    return CONFIG.set(path, value, scope)
+    result = CONFIG.set(path, value, scope)
+    return result
 
 
-def scopes() -> Dict[str, ConfigScope]:
+def scopes() -> lang.PriorityOrderedMapping[str, ConfigScope]:
     """Convenience function to get list of configuration scopes."""
     return CONFIG.scopes
 
@@ -1409,7 +1542,7 @@ def ensure_latest_format_fn(section: str) -> Callable[[YamlConfigDict], bool]:
 
 @contextlib.contextmanager
 def use_configuration(
-    *scopes_or_paths: Union[ConfigScope, str]
+    *scopes_or_paths: Union[ScopeWithOptionalPriority, str]
 ) -> Generator[Configuration, None, None]:
     """Use the configuration scopes passed as arguments within the context manager.
 
@@ -1424,7 +1557,7 @@ def use_configuration(
     global CONFIG
 
     # Normalize input and construct a Configuration object
-    configuration = _config_from(scopes_or_paths)
+    configuration = create_from(*scopes_or_paths)
     CONFIG.clear_caches(), configuration.clear_caches()
 
     saved_config, CONFIG = CONFIG, configuration
@@ -1435,137 +1568,44 @@ def use_configuration(
         CONFIG = saved_config
 
 
+def _normalize_input(entry: Union[ScopeWithOptionalPriority, str]) -> ScopeWithPriority:
+    if isinstance(entry, tuple):
+        return entry
+
+    default_priority = ConfigScopePriority.CONFIG_FILES
+    if isinstance(entry, ConfigScope):
+        return default_priority, entry
+
+    # Otherwise we need to construct it
+    path = os.path.normpath(entry)
+    assert os.path.isdir(path), f'"{path}" must be a directory'
+    name = os.path.basename(path)
+    return default_priority, DirectoryConfigScope(name, path)
+
+
 @lang.memoized
-def _config_from(scopes_or_paths: List[Union[ConfigScope, str]]) -> Configuration:
-    scopes = []
-    for scope_or_path in scopes_or_paths:
-        # If we have a config scope we are already done
-        if isinstance(scope_or_path, ConfigScope):
-            scopes.append(scope_or_path)
-            continue
-
-        # Otherwise we need to construct it
-        path = os.path.normpath(scope_or_path)
-        assert os.path.isdir(path), f'"{path}" must be a directory'
-        name = os.path.basename(path)
-        scopes.append(DirectoryConfigScope(name, path))
-
-    configuration = Configuration(*scopes)
-    return configuration
-
-
-def raw_github_gitlab_url(url: str) -> str:
-    """Transform a github URL to the raw form to avoid undesirable html.
+def create_from(*scopes_or_paths: Union[ScopeWithOptionalPriority, str]) -> Configuration:
+    """Creates a configuration object from the scopes passed in input.
 
     Args:
-        url: url to be converted to raw form
+        *scopes_or_paths: either a tuple of (priority, ConfigScope), or a ConfigScope, or a string
+            If priority is not given, it is assumed to be ConfigScopePriority.CONFIG_FILES. If a
+            string is given, a DirectoryConfigScope is created from it.
 
-    Returns:
-        Raw github/gitlab url or the original url
+    Examples:
+
+        >>> builtin_scope = InternalConfigScope("_builtin", {"config": {"build_jobs": 1}})
+        >>> cl_scope = InternalConfigScope("command_line", {"config": {"build_jobs": 10}})
+        >>> cfg = create_from(
+        ...     (ConfigScopePriority.COMMAND_LINE, cl_scope),
+        ...     (ConfigScopePriority.BUILTIN, builtin_scope)
+        ... )
     """
-    # Note we rely on GitHub to redirect the 'raw' URL returned here to the
-    # actual URL under https://raw.githubusercontent.com/ with '/blob'
-    # removed and or, '/blame' if needed.
-    if "github" in url or "gitlab" in url:
-        return url.replace("/blob/", "/raw/")
-
-    return url
-
-
-def collect_urls(base_url: str) -> list:
-    """Return a list of configuration URLs.
-
-    Arguments:
-        base_url: URL for a configuration (yaml) file or a directory
-            containing yaml file(s)
-
-    Returns:
-        List of configuration file(s) or empty list if none
-    """
-    if not base_url:
-        return []
-
-    extension = ".yaml"
-
-    if base_url.endswith(extension):
-        return [base_url]
-
-    # Collect configuration URLs if the base_url is a "directory".
-    _, links = web_util.spider(base_url, 0)
-    return [link for link in links if link.endswith(extension)]
-
-
-def fetch_remote_configs(url: str, dest_dir: str, skip_existing: bool = True) -> str:
-    """Retrieve configuration file(s) at the specified URL.
-
-    Arguments:
-        url: URL for a configuration (yaml) file or a directory containing
-            yaml file(s)
-        dest_dir: destination directory
-        skip_existing: Skip files that already exist in dest_dir if
-            ``True``; otherwise, replace those files
-
-    Returns:
-        Path to the corresponding file if URL is or contains a
-        single file and it is the only file in the destination directory or
-        the root (dest_dir) directory if multiple configuration files exist
-        or are retrieved.
-    """
-
-    def _fetch_file(url):
-        raw = raw_github_gitlab_url(url)
-        tty.debug(f"Reading config from url {raw}")
-        return web_util.fetch_url_text(raw, dest_dir=dest_dir)
-
-    if not url:
-        raise ConfigFileError("Cannot retrieve configuration without a URL")
-
-    # Return the local path to the cached configuration file OR to the
-    # directory containing the cached configuration files.
-    config_links = collect_urls(url)
-    existing_files = os.listdir(dest_dir) if os.path.isdir(dest_dir) else []
-
-    paths = []
-    for config_url in config_links:
-        basename = os.path.basename(config_url)
-        if skip_existing and basename in existing_files:
-            tty.warn(
-                f"Will not fetch configuration from {config_url} since a "
-                f"version already exists in {dest_dir}"
-            )
-            path = os.path.join(dest_dir, basename)
-        else:
-            path = _fetch_file(config_url)
-
-        if path:
-            paths.append(path)
-
-    if paths:
-        return dest_dir if len(paths) > 1 else paths[0]
-
-    raise ConfigFileError(f"Cannot retrieve configuration (yaml) from {url}")
-
-
-def get_mark_from_yaml_data(obj):
-    """Try to get ``spack.util.spack_yaml`` mark from YAML data.
-
-    We try the object, and if that fails we try its first member (if it's a container).
-
-    Returns:
-        mark if one is found, otherwise None.
-    """
-    # mark of object itelf
-    mark = getattr(obj, "_start_mark", None)
-    if mark:
-        return mark
-
-    # mark of first member if it is a container
-    if isinstance(obj, (list, dict)):
-        first_member = next(iter(obj), None)
-        if first_member:
-            mark = getattr(first_member, "_start_mark", None)
-
-    return mark
+    scopes_with_priority = [_normalize_input(x) for x in scopes_or_paths]
+    result = Configuration()
+    for priority, scope in scopes_with_priority:
+        result.push_scope(scope, priority=priority)
+    return result
 
 
 def determine_number_of_jobs(
@@ -1672,3 +1712,7 @@ def get_path(path, data):
 
         # give up and return None if nothing worked
         return None
+
+
+class RecursiveIncludeError(spack.error.SpackError):
+    """Too many levels of recursive includes."""

lib/spack/spack/database.py

@@ -1126,7 +1126,7 @@ def _add(
             installation_time:
                 Date and time of installation
             allow_missing: if True, don't warn when installation is not found on on disk
-                This is useful when installing specs without build deps.
+                This is useful when installing specs without build/test deps.
         """
         if not spec.concrete:
             raise NonConcreteSpecAddError("Specs added to DB must be concrete.")
@@ -1146,10 +1146,8 @@ def _add(
                 edge.spec,
                 explicit=False,
                 installation_time=installation_time,
-                # allow missing build-only deps. This prevents excessive warnings when a spec is
-                # installed, and its build dep is missing a build dep; there's no need to install
-                # the build dep's build dep first, and there's no need to warn about it missing.
-                allow_missing=allow_missing or edge.depflag == dt.BUILD,
+                # allow missing build / test only deps
+                allow_missing=allow_missing or edge.depflag & (dt.BUILD | dt.TEST) == edge.depflag,
             )
 
         # Make sure the directory layout agrees whether the spec is installed

lib/spack/spack/detection/path.py

@@ -7,6 +7,7 @@
 import collections
 import concurrent.futures
 import os
+import pathlib
 import re
 import sys
 import traceback
@@ -15,6 +16,7 @@
 
 import llnl.util.filesystem
 import llnl.util.lang
+import llnl.util.symlink
 import llnl.util.tty
 
 import spack.error
@@ -70,13 +72,21 @@ def dedupe_paths(paths: List[str]) -> List[str]:
     """Deduplicate paths based on inode and device number. In case the list contains first a
     symlink and then the directory it points to, the symlink is replaced with the directory path.
     This ensures that we pick for example ``/usr/bin`` over ``/bin`` if the latter is a symlink to
-    the former`."""
+    the former."""
     seen: Dict[Tuple[int, int], str] = {}
+
+    linked_parent_check = lambda x: any(
+        [llnl.util.symlink.islink(str(y)) for y in pathlib.Path(x).parents]
+    )
+
     for path in paths:
         identifier = file_identifier(path)
         if identifier not in seen:
             seen[identifier] = path
-        elif not os.path.islink(path):
+        # we also want to deprioritize paths if they contain a symlink in any parent
+        # (not just the basedir): e.g. oneapi has "latest/bin",
+        # where "latest" is a symlink to 2025.0"
+        elif not (llnl.util.symlink.islink(path) or linked_parent_check(path)):
             seen[identifier] = path
     return list(seen.values())
 

lib/spack/spack/directory_layout.py

@@ -25,7 +25,7 @@
 }
 
 
-def _check_concrete(spec):
+def _check_concrete(spec: "spack.spec.Spec") -> None:
     """If the spec is not concrete, raise a ValueError"""
     if not spec.concrete:
         raise ValueError("Specs passed to a DirectoryLayout must be concrete!")
@@ -51,7 +51,7 @@ def specs_from_metadata_dirs(root: str) -> List["spack.spec.Spec"]:
         spec = _get_spec(prefix)
 
         if spec:
-            spec.prefix = prefix
+            spec.set_prefix(prefix)
             specs.append(spec)
             continue
 
@@ -84,7 +84,7 @@ class DirectoryLayout:
 
     def __init__(
         self,
-        root,
+        root: str,
         *,
         projections: Optional[Dict[str, str]] = None,
         hash_length: Optional[int] = None,
@@ -120,17 +120,17 @@ def __init__(
         self.manifest_file_name = "install_manifest.json"
 
     @property
-    def hidden_file_regexes(self):
+    def hidden_file_regexes(self) -> Tuple[str]:
         return ("^{0}$".format(re.escape(self.metadata_dir)),)
 
-    def relative_path_for_spec(self, spec):
+    def relative_path_for_spec(self, spec: "spack.spec.Spec") -> str:
         _check_concrete(spec)
 
         projection = spack.projections.get_projection(self.projections, spec)
         path = spec.format_path(projection)
         return str(Path(path))
 
-    def write_spec(self, spec, path):
+    def write_spec(self, spec: "spack.spec.Spec", path: str) -> None:
         """Write a spec out to a file."""
         _check_concrete(spec)
         with open(path, "w", encoding="utf-8") as f:
@@ -138,7 +138,7 @@ def write_spec(self, spec, path):
             # the full provenance, so it's availabe if we want it later
             spec.to_json(f, hash=ht.dag_hash)
 
-    def write_host_environment(self, spec):
+    def write_host_environment(self, spec: "spack.spec.Spec") -> None:
         """The host environment is a json file with os, kernel, and spack
         versioning. We use it in the case that an analysis later needs to
         easily access this information.
@@ -148,7 +148,7 @@ def write_host_environment(self, spec):
         with open(env_file, "w", encoding="utf-8") as fd:
             sjson.dump(environ, fd)
 
-    def read_spec(self, path):
+    def read_spec(self, path: str) -> "spack.spec.Spec":
         """Read the contents of a file and parse them as a spec"""
         try:
             with open(path, encoding="utf-8") as f:
@@ -159,26 +159,28 @@ def read_spec(self, path):
                     # Too late for conversion; spec_file_path() already called.
                     spec = spack.spec.Spec.from_yaml(f)
                 else:
-                    raise SpecReadError(
-                        "Did not recognize spec file extension:" " {0}".format(extension)
-                    )
+                    raise SpecReadError(f"Did not recognize spec file extension: {extension}")
         except Exception as e:
             if spack.config.get("config:debug"):
                 raise
-            raise SpecReadError("Unable to read file: %s" % path, "Cause: " + str(e))
+            raise SpecReadError(f"Unable to read file: {path}", f"Cause: {e}")
 
         # Specs read from actual installations are always concrete
         spec._mark_concrete()
         return spec
 
-    def spec_file_path(self, spec):
+    def spec_file_path(self, spec: "spack.spec.Spec") -> str:
         """Gets full path to spec file"""
         _check_concrete(spec)
         yaml_path = os.path.join(self.metadata_path(spec), self._spec_file_name_yaml)
         json_path = os.path.join(self.metadata_path(spec), self.spec_file_name)
         return yaml_path if os.path.exists(yaml_path) else json_path
 
-    def deprecated_file_path(self, deprecated_spec, deprecator_spec=None):
+    def deprecated_file_path(
+        self,
+        deprecated_spec: "spack.spec.Spec",
+        deprecator_spec: Optional["spack.spec.Spec"] = None,
+    ) -> str:
         """Gets full path to spec file for deprecated spec
 
         If the deprecator_spec is provided, use that. Otherwise, assume
@@ -212,16 +214,16 @@ def deprecated_file_path(self, deprecated_spec, deprecator_spec=None):
 
         return yaml_path if os.path.exists(yaml_path) else json_path
 
-    def metadata_path(self, spec):
+    def metadata_path(self, spec: "spack.spec.Spec") -> str:
         return os.path.join(spec.prefix, self.metadata_dir)
 
-    def env_metadata_path(self, spec):
+    def env_metadata_path(self, spec: "spack.spec.Spec") -> str:
         return os.path.join(self.metadata_path(spec), "install_environment.json")
 
-    def build_packages_path(self, spec):
+    def build_packages_path(self, spec: "spack.spec.Spec") -> str:
         return os.path.join(self.metadata_path(spec), self.packages_dir)
 
-    def create_install_directory(self, spec):
+    def create_install_directory(self, spec: "spack.spec.Spec") -> None:
         _check_concrete(spec)
 
         # Create install directory with properly configured permissions
@@ -239,7 +241,7 @@ def create_install_directory(self, spec):
 
         self.write_spec(spec, self.spec_file_path(spec))
 
-    def ensure_installed(self, spec):
+    def ensure_installed(self, spec: "spack.spec.Spec") -> None:
         """
         Throws InconsistentInstallDirectoryError if:
         1. spec prefix does not exist
@@ -266,7 +268,7 @@ def ensure_installed(self, spec):
                 "Spec file in %s does not match hash!" % spec_file_path
             )
 
-    def path_for_spec(self, spec):
+    def path_for_spec(self, spec: "spack.spec.Spec") -> str:
         """Return absolute path from the root to a directory for the spec."""
         _check_concrete(spec)
 
@@ -277,23 +279,13 @@ def path_for_spec(self, spec):
         assert not path.startswith(self.root)
         return os.path.join(self.root, path)
 
-    def remove_install_directory(self, spec, deprecated=False):
+    def remove_install_directory(self, spec: "spack.spec.Spec", deprecated: bool = False) -> None:
         """Removes a prefix and any empty parent directories from the root.
         Raised RemoveFailedError if something goes wrong.
         """
         path = self.path_for_spec(spec)
         assert path.startswith(self.root)
 
-        # Windows readonly files cannot be removed by Python
-        # directly, change permissions before attempting to remove
-        if sys.platform == "win32":
-            kwargs = {
-                "ignore_errors": False,
-                "onerror": fs.readonly_file_handler(ignore_errors=False),
-            }
-        else:
-            kwargs = {}  # the default value for ignore_errors is false
-
         if deprecated:
             if os.path.exists(path):
                 try:
@@ -304,7 +296,16 @@ def remove_install_directory(self, spec, deprecated=False):
                     raise RemoveFailedError(spec, path, e) from e
         elif os.path.exists(path):
             try:
-                shutil.rmtree(path, **kwargs)
+                if sys.platform == "win32":
+                    # Windows readonly files cannot be removed by Python
+                    # directly, change permissions before attempting to remove
+                    shutil.rmtree(
+                        path,
+                        ignore_errors=False,
+                        onerror=fs.readonly_file_handler(ignore_errors=False),
+                    )
+                else:
+                    shutil.rmtree(path)
             except OSError as e:
                 raise RemoveFailedError(spec, path, e) from e
 

lib/spack/spack/enums.py

@@ -12,3 +12,13 @@ class InstallRecordStatus(enum.Flag):
     DEPRECATED = enum.auto()
     MISSING = enum.auto()
     ANY = INSTALLED | DEPRECATED | MISSING
+
+
+class ConfigScopePriority(enum.IntEnum):
+    """Priorities of the different kind of config scopes used by Spack"""
+
+    BUILTIN = 0
+    CONFIG_FILES = 1
+    CUSTOM = 2
+    ENVIRONMENT = 3
+    COMMAND_LINE = 4

lib/spack/spack/environment/depfile.py

@@ -166,7 +166,7 @@ def __init__(
                 " ".join(self._install_target(s.safe_name()) for s in item.prereqs),
                 item.target.spec_hash(),
                 item.target.unsafe_format(
-                    "{name}{@version}{%compiler}{variants}{arch=architecture}"
+                    "{name}{@version}{variants}{ arch=architecture} {%compiler}"
                 ),
                 item.buildcache_flag,
             )

lib/spack/spack/environment/environment.py

@@ -10,8 +10,6 @@
 import re
 import shutil
 import stat
-import urllib.parse
-import urllib.request
 import warnings
 from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union
 
@@ -32,7 +30,6 @@
 import spack.paths
 import spack.repo
 import spack.schema.env
-import spack.schema.merged
 import spack.spec
 import spack.spec_list
 import spack.store
@@ -43,7 +40,6 @@
 import spack.util.path
 import spack.util.spack_json as sjson
 import spack.util.spack_yaml as syaml
-import spack.util.url
 from spack import traverse
 from spack.installer import PackageInstaller
 from spack.schema.env import TOP_LEVEL_KEY
@@ -51,6 +47,8 @@
 from spack.spec_list import SpecList
 from spack.util.path import substitute_path_variables
 
+from ..enums import ConfigScopePriority
+
 SpecPair = spack.concretize.SpecPair
 
 #: environment variable used to indicate the active environment
@@ -387,6 +385,7 @@ def create_in_dir(
                 # dev paths in this environment to refer to their original
                 # locations.
                 _rewrite_relative_dev_paths_on_relocation(env, init_file_dir)
+                _rewrite_relative_repos_paths_on_relocation(env, init_file_dir)
 
     return env
 
@@ -403,8 +402,8 @@ def _rewrite_relative_dev_paths_on_relocation(env, init_file_dir):
             dev_path = substitute_path_variables(entry["path"])
             expanded_path = spack.util.path.canonicalize_path(dev_path, default_wd=init_file_dir)
 
-            # Skip if the expanded path is the same (e.g. when absolute)
-            if dev_path == expanded_path:
+            # Skip if the substituted and expanded path is the same (e.g. when absolute)
+            if entry["path"] == expanded_path:
                 continue
 
             tty.debug("Expanding develop path for {0} to {1}".format(name, expanded_path))
@@ -419,6 +418,34 @@ def _rewrite_relative_dev_paths_on_relocation(env, init_file_dir):
         env._re_read()
 
 
+def _rewrite_relative_repos_paths_on_relocation(env, init_file_dir):
+    """When initializing the environment from a manifest file and we plan
+    to store the environment in a different directory, we have to rewrite
+    relative repo paths to absolute ones and expand environment variables."""
+    with env:
+        repos_specs = spack.config.get("repos", default={}, scope=env.scope_name)
+        if not repos_specs:
+            return
+        for i, entry in enumerate(repos_specs):
+            repo_path = substitute_path_variables(entry)
+            expanded_path = spack.util.path.canonicalize_path(repo_path, default_wd=init_file_dir)
+
+            # Skip if the substituted and expanded path is the same (e.g. when absolute)
+            if entry == expanded_path:
+                continue
+
+            tty.debug("Expanding repo path for {0} to {1}".format(entry, expanded_path))
+
+            repos_specs[i] = expanded_path
+
+        spack.config.set("repos", repos_specs, scope=env.scope_name)
+
+        env.repos_specs = None
+        # If we changed the environment's spack.yaml scope, that will not be reflected
+        # in the manifest that we read
+        env._re_read()
+
+
 def environment_dir_from_name(name: str, exists_ok: bool = True) -> str:
     """Returns the directory associated with a named environment.
 
@@ -546,13 +573,6 @@ def _write_yaml(data, str_or_file):
     syaml.dump_config(data, str_or_file, default_flow_style=False)
 
 
-def _eval_conditional(string):
-    """Evaluate conditional definitions using restricted variable scope."""
-    valid_variables = spack.spec.get_host_environment()
-    valid_variables.update({"re": re, "env": os.environ})
-    return eval(string, valid_variables)
-
-
 def _is_dev_spec_and_has_changed(spec):
     """Check if the passed spec is a dev build and whether it has changed since the
     last installation"""
@@ -985,7 +1005,7 @@ def _process_definition(self, entry):
         """Process a single spec definition item."""
         when_string = entry.get("when")
         if when_string is not None:
-            when = _eval_conditional(when_string)
+            when = spack.spec.eval_conditional(when_string)
             assert len([x for x in entry if x != "when"]) == 1
         else:
             when = True
@@ -1530,9 +1550,6 @@ def _get_specs_to_concretize(
         return new_user_specs, kept_user_specs, specs_to_concretize
 
     def _concretize_together_where_possible(self, tests: bool = False) -> Sequence[SpecPair]:
-        # Avoid cyclic dependency
-        import spack.solver.asp
-
         # Exit early if the set of concretized specs is the set of user specs
         new_user_specs, _, specs_to_concretize = self._get_specs_to_concretize()
         if not new_user_specs:
@@ -2392,6 +2409,8 @@ def invalidate_repository_cache(self):
 
     def __enter__(self):
         self._previous_active = _active_environment
+        if self._previous_active:
+            deactivate()
         activate(self)
         return self
 
@@ -2641,20 +2660,23 @@ def _ensure_env_dir():
         # error handling for bad manifests is handled on other code paths
         return
 
+    # TODO: make this recursive
     includes = manifest[TOP_LEVEL_KEY].get("include", [])
     for include in includes:
-        if os.path.isabs(include):
+        included_path = spack.config.included_path(include)
+        path = included_path.path
+        if os.path.isabs(path):
             continue
 
-        abspath = pathlib.Path(os.path.normpath(environment_dir / include))
+        abspath = pathlib.Path(os.path.normpath(environment_dir / path))
         common_path = pathlib.Path(os.path.commonpath([environment_dir, abspath]))
         if common_path != environment_dir:
-            tty.debug(f"Will not copy relative include from outside environment: {include}")
+            tty.debug(f"Will not copy relative include file from outside environment: {path}")
             continue
 
-        orig_abspath = os.path.normpath(envfile.parent / include)
+        orig_abspath = os.path.normpath(envfile.parent / path)
         if not os.path.exists(orig_abspath):
-            tty.warn(f"Included file does not exist; will not copy: '{include}'")
+            tty.warn(f"Included file does not exist; will not copy: '{path}'")
             continue
 
         fs.touchp(abspath)
@@ -2877,7 +2899,7 @@ def extract_name(_item):
                 continue
 
             condition_str = item.get("when", "True")
-            if not _eval_conditional(condition_str):
+            if not spack.spec.eval_conditional(condition_str):
                 continue
 
             yield idx, item
@@ -2938,127 +2960,20 @@ def __iter__(self):
     def __str__(self):
         return str(self.manifest_file)
 
-    @property
-    def included_config_scopes(self) -> List[spack.config.ConfigScope]:
-        """List of included configuration scopes from the manifest.
-
-        Scopes are listed in the YAML file in order from highest to
-        lowest precedence, so configuration from earlier scope will take
-        precedence over later ones.
-
-        This routine returns them in the order they should be pushed onto
-        the internal scope stack (so, in reverse, from lowest to highest).
-
-        Returns: Configuration scopes associated with the environment manifest
-
-        Raises:
-            SpackEnvironmentError: if the manifest includes a remote file but
-                no configuration stage directory has been identified
-        """
-        scopes: List[spack.config.ConfigScope] = []
-
-        # load config scopes added via 'include:', in reverse so that
-        # highest-precedence scopes are last.
-        includes = self[TOP_LEVEL_KEY].get("include", [])
-        missing = []
-        for i, config_path in enumerate(reversed(includes)):
-            # allow paths to contain spack config/environment variables, etc.
-            config_path = substitute_path_variables(config_path)
-            include_url = urllib.parse.urlparse(config_path)
-
-            # If scheme is not valid, config_path is not a url
-            # of a type Spack is generally aware
-            if spack.util.url.validate_scheme(include_url.scheme):
-                # Transform file:// URLs to direct includes.
-                if include_url.scheme == "file":
-                    config_path = urllib.request.url2pathname(include_url.path)
-
-                # Any other URL should be fetched.
-                elif include_url.scheme in ("http", "https", "ftp"):
-                    # Stage any remote configuration file(s)
-                    staged_configs = (
-                        os.listdir(self.config_stage_dir)
-                        if os.path.exists(self.config_stage_dir)
-                        else []
-                    )
-                    remote_path = urllib.request.url2pathname(include_url.path)
-                    basename = os.path.basename(remote_path)
-                    if basename in staged_configs:
-                        # Do NOT re-stage configuration files over existing
-                        # ones with the same name since there is a risk of
-                        # losing changes (e.g., from 'spack config update').
-                        tty.warn(
-                            "Will not re-stage configuration from {0} to avoid "
-                            "losing changes to the already staged file of the "
-                            "same name.".format(remote_path)
-                        )
-
-                        # Recognize the configuration stage directory
-                        # is flattened to ensure a single copy of each
-                        # configuration file.
-                        config_path = self.config_stage_dir
-                        if basename.endswith(".yaml"):
-                            config_path = os.path.join(config_path, basename)
-                    else:
-                        staged_path = spack.config.fetch_remote_configs(
-                            config_path, str(self.config_stage_dir), skip_existing=True
-                        )
-                        if not staged_path:
-                            raise SpackEnvironmentError(
-                                "Unable to fetch remote configuration {0}".format(config_path)
-                            )
-                        config_path = staged_path
-
-                elif include_url.scheme:
-                    raise ValueError(
-                        f"Unsupported URL scheme ({include_url.scheme}) for "
-                        f"environment include: {config_path}"
-                    )
-
-            # treat relative paths as relative to the environment
-            if not os.path.isabs(config_path):
-                config_path = os.path.join(self.manifest_dir, config_path)
-                config_path = os.path.normpath(os.path.realpath(config_path))
-
-            if os.path.isdir(config_path):
-                # directories are treated as regular ConfigScopes
-                config_name = f"env:{self.name}:{os.path.basename(config_path)}"
-                tty.debug(f"Creating DirectoryConfigScope {config_name} for '{config_path}'")
-                scopes.append(spack.config.DirectoryConfigScope(config_name, config_path))
-            elif os.path.exists(config_path):
-                # files are assumed to be SingleFileScopes
-                config_name = f"env:{self.name}:{config_path}"
-                tty.debug(f"Creating SingleFileScope {config_name} for '{config_path}'")
-                scopes.append(
-                    spack.config.SingleFileScope(
-                        config_name, config_path, spack.schema.merged.schema
-                    )
-                )
-            else:
-                missing.append(config_path)
-                continue
-
-        if missing:
-            msg = "Detected {0} missing include path(s):".format(len(missing))
-            msg += "\n   {0}".format("\n   ".join(missing))
-            raise spack.config.ConfigFileError(msg)
-
-        return scopes
-
     @property
     def env_config_scopes(self) -> List[spack.config.ConfigScope]:
         """A list of all configuration scopes for the environment manifest. On the first call this
         instantiates all the scopes, on subsequent calls it returns the cached list."""
         if self._config_scopes is not None:
             return self._config_scopes
+
         scopes: List[spack.config.ConfigScope] = [
-            *self.included_config_scopes,
             spack.config.SingleFileScope(
                 self.scope_name,
                 str(self.manifest_file),
                 spack.schema.env.schema,
                 yaml_path=[TOP_LEVEL_KEY],
-            ),
+            )
         ]
         ensure_no_disallowed_env_config_mods(scopes)
         self._config_scopes = scopes
@@ -3067,14 +2982,12 @@ def env_config_scopes(self) -> List[spack.config.ConfigScope]:
     def prepare_config_scope(self) -> None:
         """Add the manifest's scopes to the global configuration search path."""
         for scope in self.env_config_scopes:
-            spack.config.CONFIG.push_scope(scope)
-        spack.config.CONFIG.ensure_scope_ordering()
+            spack.config.CONFIG.push_scope(scope, priority=ConfigScopePriority.ENVIRONMENT)
 
     def deactivate_config_scope(self) -> None:
         """Remove any of the manifest's scopes from the global config path."""
         for scope in self.env_config_scopes:
             spack.config.CONFIG.remove_scope(scope.name)
-        spack.config.CONFIG.ensure_scope_ordering()
 
     @contextlib.contextmanager
     def use_config(self):

lib/spack/spack/filesystem_view.py

@@ -10,7 +10,7 @@
 import stat
 import sys
 import tempfile
-from typing import Callable, Dict, Optional
+from typing import Callable, Dict, List, Optional
 
 from typing_extensions import Literal
 
@@ -78,7 +78,7 @@ def view_copy(
 
     # Order of this dict is somewhat irrelevant
     prefix_to_projection = {
-        s.prefix: view.get_projection_for_spec(s)
+        str(s.prefix): view.get_projection_for_spec(s)
         for s in spec.traverse(root=True, order="breadth")
         if not s.external
     }
@@ -185,7 +185,7 @@ def __init__(
     def link(self, src: str, dst: str, spec: Optional[spack.spec.Spec] = None) -> None:
         self._link(src, dst, self, spec)
 
-    def add_specs(self, *specs, **kwargs):
+    def add_specs(self, *specs: spack.spec.Spec, **kwargs) -> None:
         """
         Add given specs to view.
 
@@ -200,19 +200,19 @@ def add_specs(self, *specs, **kwargs):
         """
         raise NotImplementedError
 
-    def add_standalone(self, spec):
+    def add_standalone(self, spec: spack.spec.Spec) -> bool:
         """
         Add (link) a standalone package into this view.
         """
         raise NotImplementedError
 
-    def check_added(self, spec):
+    def check_added(self, spec: spack.spec.Spec) -> bool:
         """
         Check if the given concrete spec is active in this view.
         """
         raise NotImplementedError
 
-    def remove_specs(self, *specs, **kwargs):
+    def remove_specs(self, *specs: spack.spec.Spec, **kwargs) -> None:
         """
         Removes given specs from view.
 
@@ -231,25 +231,25 @@ def remove_specs(self, *specs, **kwargs):
         """
         raise NotImplementedError
 
-    def remove_standalone(self, spec):
+    def remove_standalone(self, spec: spack.spec.Spec) -> None:
         """
         Remove (unlink) a standalone package from this view.
         """
         raise NotImplementedError
 
-    def get_projection_for_spec(self, spec):
+    def get_projection_for_spec(self, spec: spack.spec.Spec) -> str:
         """
         Get the projection in this view for a spec.
         """
         raise NotImplementedError
 
-    def get_all_specs(self):
+    def get_all_specs(self) -> List[spack.spec.Spec]:
         """
         Get all specs currently active in this view.
         """
         raise NotImplementedError
 
-    def get_spec(self, spec):
+    def get_spec(self, spec: spack.spec.Spec) -> Optional[spack.spec.Spec]:
         """
         Return the actual spec linked in this view (i.e. do not look it up
         in the database by name).
@@ -263,7 +263,7 @@ def get_spec(self, spec):
         """
         raise NotImplementedError
 
-    def print_status(self, *specs, **kwargs):
+    def print_status(self, *specs: spack.spec.Spec, **kwargs) -> None:
         """
         Print a short summary about the given specs, detailing whether..
             * ..they are active in the view.
@@ -643,7 +643,7 @@ def print_status(self, *specs, **kwargs):
                 specs.sort()
 
                 abbreviated = [
-                    s.cformat("{name}{@version}{%compiler}{compiler_flags}{variants}")
+                    s.cformat("{name}{@version}{compiler_flags}{variants}{%compiler}")
                     for s in specs
                 ]
 
@@ -694,7 +694,7 @@ def _sanity_check_view_projection(self, specs):
                 raise ConflictingSpecsError(current_spec, conflicting_spec)
             seen[metadata_dir] = current_spec
 
-    def add_specs(self, *specs: spack.spec.Spec) -> None:
+    def add_specs(self, *specs, **kwargs) -> None:
         """Link a root-to-leaf topologically ordered list of specs into the view."""
         assert all((s.concrete for s in specs))
         if len(specs) == 0:
@@ -831,7 +831,7 @@ def get_projection_for_spec(self, spec):
 #####################
 # utility functions #
 #####################
-def get_spec_from_file(filename):
+def get_spec_from_file(filename) -> Optional[spack.spec.Spec]:
     try:
         with open(filename, "r", encoding="utf-8") as f:
             return spack.spec.Spec.from_yaml(f)

lib/spack/spack/graph.py

@@ -482,7 +482,7 @@ class SimpleDAG(DotGraphBuilder):
     """Simple DOT graph, with nodes colored uniformly and edges without properties"""
 
     def node_entry(self, node):
-        format_option = "{name}{@version}{%compiler}{/hash:7}"
+        format_option = "{name}{@version}{/hash:7}{%compiler}"
         return node.dag_hash(), f'[label="{node.format(format_option)}"]'
 
     def edge_entry(self, edge):
@@ -515,7 +515,7 @@ def visit(self, edge):
         super().visit(edge)
 
     def node_entry(self, node):
-        node_str = node.format("{name}{@version}{%compiler}{/hash:7}")
+        node_str = node.format("{name}{@version}{/hash:7}{%compiler}")
         options = f'[label="{node_str}", group="build_dependencies", fillcolor="coral"]'
         if node.dag_hash() in self.main_unified_space:
             options = f'[label="{node_str}", group="main_psid"]'

lib/spack/spack/hash_types.py

@@ -6,7 +6,7 @@
 import spack.deptypes as dt
 import spack.repo
 
-hashes = []
+HASHES = []
 
 
 class SpecHashDescriptor:
@@ -23,7 +23,7 @@ def __init__(self, depflag: dt.DepFlag, package_hash, name, override=None):
         self.depflag = depflag
         self.package_hash = package_hash
         self.name = name
-        hashes.append(self)
+        HASHES.append(self)
         # Allow spec hashes to have an alternate computation method
         self.override = override
 
@@ -43,13 +43,9 @@ def __repr__(self):
         )
 
 
-#: Spack's deployment hash. Includes all inputs that can affect how a package is built.
-dag_hash = SpecHashDescriptor(depflag=dt.BUILD | dt.LINK | dt.RUN, package_hash=True, name="hash")
-
-
-#: Hash descriptor used only to transfer a DAG, as is, across processes
-process_hash = SpecHashDescriptor(
-    depflag=dt.BUILD | dt.LINK | dt.RUN | dt.TEST, package_hash=True, name="process_hash"
+#: The DAG hash includes all inputs that can affect how a package is built.
+dag_hash = SpecHashDescriptor(
+    depflag=dt.BUILD | dt.LINK | dt.RUN | dt.TEST, package_hash=True, name="hash"
 )
 
 

lib/spack/spack/hooks/resolve_shared_libraries.py

@@ -2,200 +2,14 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import fnmatch
 import io
-import os
-import re
-from typing import Dict, List, Union
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import BaseDirectoryVisitor, visit_directory_tree
-from llnl.util.lang import stable_partition
+from llnl.util.filesystem import visit_directory_tree
 
 import spack.config
 import spack.error
-import spack.util.elf as elf
-
-#: Patterns for names of libraries that are allowed to be unresolved when *just* looking at RPATHs
-#: added by Spack. These are libraries outside of Spack's control, and assumed to be located in
-#: default search paths of the dynamic linker.
-ALLOW_UNRESOLVED = [
-    # kernel
-    "linux-vdso.so.*",
-    "libselinux.so.*",
-    # musl libc
-    "ld-musl-*.so.*",
-    # glibc
-    "ld-linux*.so.*",
-    "ld64.so.*",
-    "libanl.so.*",
-    "libc.so.*",
-    "libdl.so.*",
-    "libm.so.*",
-    "libmemusage.so.*",
-    "libmvec.so.*",
-    "libnsl.so.*",
-    "libnss_compat.so.*",
-    "libnss_db.so.*",
-    "libnss_dns.so.*",
-    "libnss_files.so.*",
-    "libnss_hesiod.so.*",
-    "libpcprofile.so.*",
-    "libpthread.so.*",
-    "libresolv.so.*",
-    "librt.so.*",
-    "libSegFault.so.*",
-    "libthread_db.so.*",
-    "libutil.so.*",
-    # gcc -- this is required even with gcc-runtime, because e.g. libstdc++ depends on libgcc_s,
-    # but the binaries we copy from the compiler don't have an $ORIGIN rpath.
-    "libasan.so.*",
-    "libatomic.so.*",
-    "libcc1.so.*",
-    "libgcc_s.so.*",
-    "libgfortran.so.*",
-    "libgomp.so.*",
-    "libitm.so.*",
-    "liblsan.so.*",
-    "libquadmath.so.*",
-    "libssp.so.*",
-    "libstdc++.so.*",
-    "libtsan.so.*",
-    "libubsan.so.*",
-    # systemd
-    "libudev.so.*",
-    # cuda driver
-    "libcuda.so.*",
-    # intel-oneapi-runtime
-    "libur_loader.so.*",
-]
-
-
-def is_compatible(parent: elf.ElfFile, child: elf.ElfFile) -> bool:
-    return (
-        child.elf_hdr.e_type == elf.ELF_CONSTANTS.ET_DYN
-        and parent.is_little_endian == child.is_little_endian
-        and parent.is_64_bit == child.is_64_bit
-        and parent.elf_hdr.e_machine == child.elf_hdr.e_machine
-    )
-
-
-def candidate_matches(current_elf: elf.ElfFile, candidate_path: bytes) -> bool:
-    try:
-        with open(candidate_path, "rb") as g:
-            return is_compatible(current_elf, elf.parse_elf(g))
-    except (OSError, elf.ElfParsingError):
-        return False
-
-
-class Problem:
-    def __init__(
-        self, resolved: Dict[bytes, bytes], unresolved: List[bytes], relative_rpaths: List[bytes]
-    ) -> None:
-        self.resolved = resolved
-        self.unresolved = unresolved
-        self.relative_rpaths = relative_rpaths
-
-
-class ResolveSharedElfLibDepsVisitor(BaseDirectoryVisitor):
-    def __init__(self, allow_unresolved_patterns: List[str]) -> None:
-        self.problems: Dict[str, Problem] = {}
-        self._allow_unresolved_regex = re.compile(
-            "|".join(fnmatch.translate(x) for x in allow_unresolved_patterns)
-        )
-
-    def allow_unresolved(self, needed: bytes) -> bool:
-        try:
-            name = needed.decode("utf-8")
-        except UnicodeDecodeError:
-            return False
-        return bool(self._allow_unresolved_regex.match(name))
-
-    def visit_file(self, root: str, rel_path: str, depth: int) -> None:
-        # We work with byte strings for paths.
-        path = os.path.join(root, rel_path).encode("utf-8")
-
-        # For $ORIGIN interpolation: should not have trailing dir seperator.
-        origin = os.path.dirname(path)
-
-        # Retrieve the needed libs + rpaths.
-        try:
-            with open(path, "rb") as f:
-                parsed_elf = elf.parse_elf(f, interpreter=False, dynamic_section=True)
-        except (OSError, elf.ElfParsingError):
-            # Not dealing with an invalid ELF file.
-            return
-
-        # If there's no needed libs all is good
-        if not parsed_elf.has_needed:
-            return
-
-        # Get the needed libs and rpaths (notice: byte strings)
-        # Don't force an encoding cause paths are just a bag of bytes.
-        needed_libs = parsed_elf.dt_needed_strs
-
-        rpaths = parsed_elf.dt_rpath_str.split(b":") if parsed_elf.has_rpath else []
-
-        # We only interpolate $ORIGIN, not $LIB and $PLATFORM, they're not really
-        # supported in general. Also remove empty paths.
-        rpaths = [x.replace(b"$ORIGIN", origin) for x in rpaths if x]
-
-        # Do not allow relative rpaths (they are relative to the current working directory)
-        rpaths, relative_rpaths = stable_partition(rpaths, os.path.isabs)
-
-        # If there's a / in the needed lib, it's opened directly, otherwise it needs
-        # a search.
-        direct_libs, search_libs = stable_partition(needed_libs, lambda x: b"/" in x)
-
-        # Do not allow relative paths in direct libs (they are relative to the current working
-        # directory)
-        direct_libs, unresolved = stable_partition(direct_libs, os.path.isabs)
-
-        resolved: Dict[bytes, bytes] = {}
-
-        for lib in search_libs:
-            if self.allow_unresolved(lib):
-                continue
-            for rpath in rpaths:
-                candidate = os.path.join(rpath, lib)
-                if candidate_matches(parsed_elf, candidate):
-                    resolved[lib] = candidate
-                    break
-            else:
-                unresolved.append(lib)
-
-        # Check if directly opened libs are compatible
-        for lib in direct_libs:
-            if candidate_matches(parsed_elf, lib):
-                resolved[lib] = lib
-            else:
-                unresolved.append(lib)
-
-        if unresolved or relative_rpaths:
-            self.problems[rel_path] = Problem(resolved, unresolved, relative_rpaths)
-
-    def visit_symlinked_file(self, root: str, rel_path: str, depth: int) -> None:
-        pass
-
-    def before_visit_dir(self, root: str, rel_path: str, depth: int) -> bool:
-        # There can be binaries in .spack/test which shouldn't be checked.
-        if rel_path == ".spack":
-            return False
-        return True
-
-    def before_visit_symlinked_dir(self, root: str, rel_path: str, depth: int) -> bool:
-        return False
-
-
-class CannotLocateSharedLibraries(spack.error.SpackError):
-    pass
-
-
-def maybe_decode(byte_str: bytes) -> Union[str, bytes]:
-    try:
-        return byte_str.decode("utf-8")
-    except UnicodeDecodeError:
-        return byte_str
+import spack.verify_libraries
 
 
 def post_install(spec, explicit):
@@ -206,36 +20,23 @@ def post_install(spec, explicit):
     if policy == "ignore" or spec.external or spec.platform not in ("linux", "freebsd"):
         return
 
-    visitor = ResolveSharedElfLibDepsVisitor(
-        [*ALLOW_UNRESOLVED, *spec.package.unresolved_libraries]
+    visitor = spack.verify_libraries.ResolveSharedElfLibDepsVisitor(
+        [*spack.verify_libraries.ALLOW_UNRESOLVED, *spec.package.unresolved_libraries]
     )
     visit_directory_tree(spec.prefix, visitor)
 
-    # All good?
     if not visitor.problems:
         return
 
-    # For now just list the issues (print it in ldd style, except we don't recurse)
-    output = io.StringIO()
-    output.write("not all executables and libraries can resolve their dependencies:\n")
-    for path, problem in visitor.problems.items():
-        output.write(path)
-        output.write("\n")
-        for needed, full_path in problem.resolved.items():
-            output.write("        ")
-            if needed == full_path:
-                output.write(maybe_decode(needed))
-            else:
-                output.write(f"{maybe_decode(needed)} => {maybe_decode(full_path)}")
-            output.write("\n")
-        for not_found in problem.unresolved:
-            output.write(f"        {maybe_decode(not_found)} => not found\n")
-        for relative_rpath in problem.relative_rpaths:
-            output.write(f"        {maybe_decode(relative_rpath)} => relative rpath\n")
-
+    output = io.StringIO("not all executables and libraries can resolve their dependencies:\n")
+    visitor.write(output)
     message = output.getvalue().strip()
 
     if policy == "error":
         raise CannotLocateSharedLibraries(message)
 
     tty.warn(message)
+
+
+class CannotLocateSharedLibraries(spack.error.SpackError):
+    pass

lib/spack/spack/install_test.py

@@ -21,7 +21,6 @@
 from llnl.util.lang import nullcontext
 from llnl.util.tty.color import colorize
 
-import spack.build_environment
 import spack.config
 import spack.error
 import spack.package_base
@@ -398,7 +397,7 @@ def stand_alone_tests(self, kwargs):
         Args:
             kwargs (dict): arguments to be used by the test process
         """
-        import spack.build_environment
+        import spack.build_environment  # avoid circular dependency
 
         spack.build_environment.start_build_process(self.pkg, test_process, kwargs)
 
@@ -463,6 +462,8 @@ def write_tested_status(self):
 
 @contextlib.contextmanager
 def test_part(pkg: Pb, test_name: str, purpose: str, work_dir: str = ".", verbose: bool = False):
+    import spack.build_environment  # avoid circular dependency
+
     wdir = "." if work_dir is None else work_dir
     tester = pkg.tester
     assert test_name and test_name.startswith(

lib/spack/spack/main.py

@@ -47,6 +47,8 @@
 import spack.util.environment
 import spack.util.lock
 
+from .enums import ConfigScopePriority
+
 #: names of profile statistics
 stat_names = pstats.Stats.sort_arg_dict_default
 
@@ -872,14 +874,19 @@ def add_command_line_scopes(
         scopes = ev.environment_path_scopes(name, path)
         if scopes is None:
             if os.path.isdir(path):  # directory with config files
-                cfg.push_scope(spack.config.DirectoryConfigScope(name, path, writable=False))
-                spack.config._add_platform_scope(cfg, name, path, writable=False)
+                cfg.push_scope(
+                    spack.config.DirectoryConfigScope(name, path, writable=False),
+                    priority=ConfigScopePriority.CUSTOM,
+                )
+                spack.config._add_platform_scope(
+                    cfg, name, path, priority=ConfigScopePriority.CUSTOM, writable=False
+                )
                 continue
             else:
                 raise spack.error.ConfigError(f"Invalid configuration scope: {path}")
 
         for scope in scopes:
-            cfg.push_scope(scope)
+            cfg.push_scope(scope, priority=ConfigScopePriority.CUSTOM)
 
 
 def _main(argv=None):
@@ -952,7 +959,9 @@ def _main(argv=None):
     # Push scopes from the command line last
     if args.config_scopes:
         add_command_line_scopes(spack.config.CONFIG, args.config_scopes)
-    spack.config.CONFIG.push_scope(spack.config.InternalConfigScope("command_line"))
+    spack.config.CONFIG.push_scope(
+        spack.config.InternalConfigScope("command_line"), priority=ConfigScopePriority.COMMAND_LINE
+    )
     setup_main_options(args)
 
     # ------------------------------------------------------------------------
@@ -998,6 +1007,7 @@ def finish_parse_and_run(parser, cmd_name, main_args, env_format_error):
     args, unknown = parser.parse_known_args(main_args.command)
     # we need to inherit verbose since the install command checks for it
     args.verbose = main_args.verbose
+    args.lines = main_args.lines
 
     # Now that we know what command this is and what its args are, determine
     # whether we can continue with a bad environment and raise if not.

lib/spack/spack/modules/common.py

@@ -564,6 +564,12 @@ def __init__(self, configuration):
     def spec(self):
         return self.conf.spec
 
+    @tengine.context_property
+    def tags(self):
+        if not hasattr(self.spec.package, "tags"):
+            return []
+        return self.spec.package.tags
+
     @tengine.context_property
     def timestamp(self):
         return datetime.datetime.now()

lib/spack/spack/package_base.py

@@ -125,9 +125,10 @@ def windows_establish_runtime_linkage(self):
         # Spack should in general not modify things it has not installed
         # we can reasonably expect externals to have their link interface properly established
         if sys.platform == "win32" and not self.spec.external:
-            self.win_rpath.add_library_dependent(*self.win_add_library_dependent())
-            self.win_rpath.add_rpath(*self.win_add_rpath())
-            self.win_rpath.establish_link()
+            win_rpath = fsys.WindowsSimulatedRPath(self)
+            win_rpath.add_library_dependent(*self.win_add_library_dependent())
+            win_rpath.add_rpath(*self.win_add_rpath())
+            win_rpath.establish_link()
 
 
 #: Registers which are the detectable packages, by repo and package name
@@ -742,7 +743,6 @@ def __init__(self, spec):
         # Set up timing variables
         self._fetch_time = 0.0
 
-        self.win_rpath = fsys.WindowsSimulatedRPath(self)
         super().__init__()
 
     def __getitem__(self, key: str) -> "PackageBase":

lib/spack/spack/patch.py

@@ -83,6 +83,7 @@ def __init__(
         level: int,
         working_dir: str,
         reverse: bool = False,
+        ordering_key: Optional[Tuple[str, int]] = None,
     ) -> None:
         """Initialize a new Patch instance.
 
@@ -92,6 +93,7 @@ def __init__(
             level: patch level
             working_dir: relative path *within* the stage to change to
             reverse: reverse the patch
+            ordering_key: key used to ensure patches are applied in a consistent order
         """
         # validate level (must be an integer >= 0)
         if not isinstance(level, int) or not level >= 0:
@@ -105,6 +107,13 @@ def __init__(
         self.working_dir = working_dir
         self.reverse = reverse
 
+        # The ordering key is passed when executing package.py directives, and is only relevant
+        # after a solve to build concrete specs with consistently ordered patches. For concrete
+        # specs read from a file, we add patches in the order of its patches variants and the
+        # ordering_key is irrelevant. In that case, use a default value so we don't need to branch
+        # on whether ordering_key is None where it's used, just to make static analysis happy.
+        self.ordering_key: Tuple[str, int] = ordering_key or ("", 0)
+
     def apply(self, stage: "spack.stage.Stage") -> None:
         """Apply a patch to source in a stage.
 
@@ -202,9 +211,8 @@ def __init__(
             msg += "package %s.%s does not exist." % (pkg.namespace, pkg.name)
             raise ValueError(msg)
 
-        super().__init__(pkg, abs_path, level, working_dir, reverse)
+        super().__init__(pkg, abs_path, level, working_dir, reverse, ordering_key)
         self.path = abs_path
-        self.ordering_key = ordering_key
 
     @property
     def sha256(self) -> str:
@@ -266,13 +274,11 @@ def __init__(
             archive_sha256: sha256 sum of the *archive*, if the patch is compressed
                 (only required for compressed URL patches)
         """
-        super().__init__(pkg, url, level, working_dir, reverse)
+        super().__init__(pkg, url, level, working_dir, reverse, ordering_key)
 
         self.url = url
         self._stage: Optional["spack.stage.Stage"] = None
 
-        self.ordering_key = ordering_key
-
         if allowed_archive(self.url) and not archive_sha256:
             raise spack.error.PatchDirectiveError(
                 "Compressed patches require 'archive_sha256' "

lib/spack/spack/repo.py

@@ -32,6 +32,7 @@
 import llnl.util.tty as tty
 from llnl.util.filesystem import working_dir
 
+import spack
 import spack.caches
 import spack.config
 import spack.error
@@ -49,6 +50,8 @@
 #: Package modules are imported as spack.pkg.<repo-namespace>.<pkg-name>
 ROOT_PYTHON_NAMESPACE = "spack.pkg"
 
+_API_REGEX = re.compile(r"^v(\d+)\.(\d+)$")
+
 
 def python_package_for_repo(namespace):
     """Returns the full namespace of a repository, given its relative one
@@ -909,19 +912,52 @@ def __reduce__(self):
         return RepoPath.unmarshal, self.marshal()
 
 
+def _parse_package_api_version(
+    config: Dict[str, Any],
+    min_api: Tuple[int, int] = spack.min_package_api_version,
+    max_api: Tuple[int, int] = spack.package_api_version,
+) -> Tuple[int, int]:
+    api = config.get("api")
+    if api is None:
+        package_api = (1, 0)
+    else:
+        if not isinstance(api, str):
+            raise BadRepoError(f"Invalid Package API version '{api}'. Must be of the form vX.Y")
+        api_match = _API_REGEX.match(api)
+        if api_match is None:
+            raise BadRepoError(f"Invalid Package API version '{api}'. Must be of the form vX.Y")
+        package_api = (int(api_match.group(1)), int(api_match.group(2)))
+
+    if min_api <= package_api <= max_api:
+        return package_api
+
+    min_str = ".".join(str(i) for i in min_api)
+    max_str = ".".join(str(i) for i in max_api)
+    curr_str = ".".join(str(i) for i in package_api)
+    raise BadRepoError(
+        f"Package API v{curr_str} is not supported by this version of Spack ("
+        f"must be between v{min_str} and v{max_str})"
+    )
+
+
 class Repo:
     """Class representing a package repository in the filesystem.
 
-    Each package repository must have a top-level configuration file
-    called `repo.yaml`.
+    Each package repository must have a top-level configuration file called `repo.yaml`.
 
-    Currently, `repo.yaml` must define:
+    It contains the following keys:
 
     `namespace`:
         A Python namespace where the repository's packages should live.
 
     `subdirectory`:
         An optional subdirectory name where packages are placed
+
+    `api`:
+        A string of the form vX.Y that indicates the Package API version. The default is "v1.0".
+        For the repo to be compatible with the current version of Spack, the version must be
+        greater than or equal to :py:data:`spack.min_package_api_version` and less than or equal to
+        :py:data:`spack.package_api_version`.
     """
 
     def __init__(
@@ -958,7 +994,7 @@ def check(condition, msg):
             f"{os.path.join(root, repo_config_name)} must define a namespace.",
         )
 
-        self.namespace = config["namespace"]
+        self.namespace: str = config["namespace"]
         check(
             re.match(r"[a-zA-Z][a-zA-Z0-9_.]+", self.namespace),
             f"Invalid namespace '{self.namespace}' in repo '{self.root}'. "
@@ -971,12 +1007,14 @@ def check(condition, msg):
         # Keep name components around for checking prefixes.
         self._names = self.full_namespace.split(".")
 
-        packages_dir = config.get("subdirectory", packages_dir_name)
+        packages_dir: str = config.get("subdirectory", packages_dir_name)
         self.packages_path = os.path.join(self.root, packages_dir)
         check(
             os.path.isdir(self.packages_path), f"No directory '{packages_dir}' found in '{root}'"
         )
 
+        self.package_api = _parse_package_api_version(config)
+
         # Class attribute overrides by package name
         self.overrides = overrides or {}
 
@@ -1026,7 +1064,7 @@ def is_prefix(self, fullname: str) -> bool:
         parts = fullname.split(".")
         return self._names[: len(parts)] == parts
 
-    def _read_config(self) -> Dict[str, str]:
+    def _read_config(self) -> Dict[str, Any]:
         """Check for a YAML config file in this db's root directory."""
         try:
             with open(self.config_file, encoding="utf-8") as reponame_file:
@@ -1179,9 +1217,8 @@ def all_package_paths(self) -> Generator[str, None, None]:
             yield self.package_path(name)
 
     def packages_with_tags(self, *tags: str) -> Set[str]:
-        v = set(self.tag_index[tags[0].lower()])
-        for tag in tags[1:]:
-            v.intersection_update(self.tag_index[tag.lower()])
+        v = set(self.all_package_names())
+        v.intersection_update(*(self.tag_index[tag.lower()] for tag in tags))
         return v
 
     def all_package_classes(self) -> Generator[Type["spack.package_base.PackageBase"], None, None]:
@@ -1369,6 +1406,8 @@ def create_repo(root, namespace=None, subdir=packages_dir_name):
             config.write(f"  namespace: '{namespace}'\n")
             if subdir != packages_dir_name:
                 config.write(f"  subdirectory: '{subdir}'\n")
+            x, y = spack.package_api_version
+            config.write(f"  api: v{x}.{y}\n")
 
     except OSError as e:
         # try to clean up.

lib/spack/spack/schema/__init__.py

@@ -7,8 +7,7 @@
 import warnings
 
 import jsonschema
-
-import llnl.util.lang
+import jsonschema.validators
 
 from spack.error import SpecSyntaxError
 
@@ -18,59 +17,59 @@ class DeprecationMessage(typing.NamedTuple):
     error: bool
 
 
-# jsonschema is imported lazily as it is heavy to import
-# and increases the start-up time
-def _make_validator():
-    def _validate_spec(validator, is_spec, instance, schema):
-        """Check if the attributes on instance are valid specs."""
-        import spack.spec_parser
+def _validate_spec(validator, is_spec, instance, schema):
+    """Check if all additional keys are valid specs."""
+    import spack.spec_parser
 
-        if not validator.is_type(instance, "object"):
-            return
+    if not validator.is_type(instance, "object"):
+        return
 
-        for spec_str in instance:
-            try:
-                spack.spec_parser.parse(spec_str)
-            except SpecSyntaxError:
-                yield jsonschema.ValidationError(f"the key '{spec_str}' is not a valid spec")
+    properties = schema.get("properties") or {}
 
-    def _deprecated_properties(validator, deprecated, instance, schema):
-        if not (validator.is_type(instance, "object") or validator.is_type(instance, "array")):
-            return
-
-        if not deprecated:
-            return
-
-        deprecations = {
-            name: DeprecationMessage(message=x["message"], error=x["error"])
-            for x in deprecated
-            for name in x["names"]
-        }
-
-        # Get a list of the deprecated properties, return if there is none
-        issues = [entry for entry in instance if entry in deprecations]
-        if not issues:
-            return
-
-        # Process issues
-        errors = []
-        for name in issues:
-            msg = deprecations[name].message.format(name=name)
-            if deprecations[name].error:
-                errors.append(msg)
-            else:
-                warnings.warn(msg)
-
-        if errors:
-            yield jsonschema.ValidationError("\n".join(errors))
-
-    return jsonschema.validators.extend(
-        jsonschema.Draft7Validator,
-        {"validate_spec": _validate_spec, "deprecatedProperties": _deprecated_properties},
-    )
+    for spec_str in instance:
+        if spec_str in properties:
+            continue
+        try:
+            spack.spec_parser.parse(spec_str)
+        except SpecSyntaxError:
+            yield jsonschema.ValidationError(f"the key '{spec_str}' is not a valid spec")
 
 
-Validator = llnl.util.lang.Singleton(_make_validator)
+def _deprecated_properties(validator, deprecated, instance, schema):
+    if not (validator.is_type(instance, "object") or validator.is_type(instance, "array")):
+        return
+
+    if not deprecated:
+        return
+
+    deprecations = {
+        name: DeprecationMessage(message=x["message"], error=x["error"])
+        for x in deprecated
+        for name in x["names"]
+    }
+
+    # Get a list of the deprecated properties, return if there is none
+    issues = [entry for entry in instance if entry in deprecations]
+    if not issues:
+        return
+
+    # Process issues
+    errors = []
+    for name in issues:
+        msg = deprecations[name].message.format(name=name)
+        if deprecations[name].error:
+            errors.append(msg)
+        else:
+            warnings.warn(msg)
+
+    if errors:
+        yield jsonschema.ValidationError("\n".join(errors))
+
+
+Validator = jsonschema.validators.extend(
+    jsonschema.Draft7Validator,
+    {"additionalKeysAreSpecs": _validate_spec, "deprecatedProperties": _deprecated_properties},
+)
 
 
 def _append(string: str) -> bool:

lib/spack/spack/schema/env.py

@@ -29,11 +29,7 @@
             # merged configuration scope schemas
             spack.schema.merged.properties,
             # extra environment schema properties
-            {
-                "include": {"type": "array", "default": [], "items": {"type": "string"}},
-                "specs": spec_list_schema,
-                "include_concrete": include_concrete,
-            },
+            {"specs": spec_list_schema, "include_concrete": include_concrete},
         ),
     }
 }

lib/spack/spack/schema/include.py

@@ -0,0 +1,41 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Schema for include.yaml configuration file.
+
+.. literalinclude:: _spack_root/lib/spack/spack/schema/include.py
+   :lines: 12-
+"""
+from typing import Any, Dict
+
+#: Properties for inclusion in other schemas
+properties: Dict[str, Any] = {
+    "include": {
+        "type": "array",
+        "default": [],
+        "additionalProperties": False,
+        "items": {
+            "anyOf": [
+                {
+                    "type": "object",
+                    "properties": {
+                        "when": {"type": "string"},
+                        "path": {"type": "string"},
+                        "sha256": {"type": "string"},
+                        "optional": {"type": "boolean"},
+                    },
+                    "required": ["path"],
+                    "additionalProperties": False,
+                },
+                {"type": "string"},
+            ]
+        },
+    }
+}
+
+#: Full schema with metadata
+schema = {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Spack include configuration file schema",
+    "properties": properties,
+}

lib/spack/spack/schema/merged.py

@@ -21,6 +21,7 @@
 import spack.schema.definitions
 import spack.schema.develop
 import spack.schema.env_vars
+import spack.schema.include
 import spack.schema.mirrors
 import spack.schema.modules
 import spack.schema.packages
@@ -40,6 +41,7 @@
     spack.schema.definitions.properties,
     spack.schema.develop.properties,
     spack.schema.env_vars.properties,
+    spack.schema.include.properties,
     spack.schema.mirrors.properties,
     spack.schema.modules.properties,
     spack.schema.packages.properties,
@@ -48,7 +50,6 @@
     spack.schema.view.properties,
 )
 
-
 #: Full schema with metadata
 schema = {
     "$schema": "http://json-schema.org/draft-07/schema#",

lib/spack/spack/schema/modules.py

@@ -39,7 +39,7 @@
         "load": array_of_strings,
         "suffixes": {
             "type": "object",
-            "validate_spec": True,
+            "additionalKeysAreSpecs": True,
             "additionalProperties": {"type": "string"},  # key
         },
         "environment": spack.schema.environment.definition,
@@ -48,40 +48,44 @@
 
 projections_scheme = spack.schema.projections.properties["projections"]
 
-module_type_configuration: Dict = {
+common_props = {
+    "verbose": {"type": "boolean", "default": False},
+    "hash_length": {"type": "integer", "minimum": 0, "default": 7},
+    "include": array_of_strings,
+    "exclude": array_of_strings,
+    "exclude_implicits": {"type": "boolean", "default": False},
+    "defaults": array_of_strings,
+    "hide_implicits": {"type": "boolean", "default": False},
+    "naming_scheme": {"type": "string"},
+    "projections": projections_scheme,
+    "all": module_file_configuration,
+}
+
+tcl_configuration = {
     "type": "object",
     "default": {},
-    "validate_spec": True,
-    "properties": {
-        "verbose": {"type": "boolean", "default": False},
-        "hash_length": {"type": "integer", "minimum": 0, "default": 7},
-        "include": array_of_strings,
-        "exclude": array_of_strings,
-        "exclude_implicits": {"type": "boolean", "default": False},
-        "defaults": array_of_strings,
-        "hide_implicits": {"type": "boolean", "default": False},
-        "naming_scheme": {"type": "string"},
-        "projections": projections_scheme,
-        "all": module_file_configuration,
-    },
+    "additionalKeysAreSpecs": True,
+    "properties": {**common_props},
     "additionalProperties": module_file_configuration,
 }
 
-tcl_configuration = module_type_configuration.copy()
-
-lmod_configuration = module_type_configuration.copy()
-lmod_configuration["properties"].update(
-    {
+lmod_configuration = {
+    "type": "object",
+    "default": {},
+    "additionalKeysAreSpecs": True,
+    "properties": {
+        **common_props,
         "core_compilers": array_of_strings,
         "hierarchy": array_of_strings,
         "core_specs": array_of_strings,
         "filter_hierarchy_specs": {
             "type": "object",
-            "validate_spec": True,
+            "additionalKeysAreSpecs": True,
             "additionalProperties": array_of_strings,
         },
-    }
-)
+    },
+    "additionalProperties": module_file_configuration,
+}
 
 module_config_properties = {
     "use_view": {"anyOf": [{"type": "string"}, {"type": "boolean"}]},

lib/spack/spack/solver/asp.py

@@ -36,6 +36,7 @@
 import spack.error
 import spack.package_base
 import spack.package_prefs
+import spack.patch
 import spack.platforms
 import spack.repo
 import spack.solver.splicing
@@ -43,6 +44,7 @@
 import spack.store
 import spack.util.crypto
 import spack.util.libc
+import spack.util.module_cmd as md
 import spack.util.path
 import spack.util.timer
 import spack.variant as vt
@@ -1656,13 +1658,16 @@ def track_dependencies(input_spec, requirements):
                     return requirements + [fn.attr("track_dependencies", input_spec.name)]
 
                 def dependency_holds(input_spec, requirements):
-                    return remove_node(input_spec, requirements) + [
+                    result = remove_node(input_spec, requirements) + [
                         fn.attr(
                             "dependency_holds", pkg.name, input_spec.name, dt.flag_to_string(t)
                         )
                         for t in dt.ALL_FLAGS
                         if t & depflag
                     ]
+                    if input_spec.name not in pkg.extendees:
+                        return result
+                    return result + [fn.attr("extends", pkg.name, input_spec.name)]
 
                 context = ConditionContext()
                 context.source = ConstraintOrigin.append_type_suffix(
@@ -3702,11 +3707,11 @@ def build_specs(self, function_tuples):
         roots = [spec.root for spec in self._specs.values()]
         roots = dict((id(r), r) for r in roots)
         for root in roots.values():
-            spack.spec.Spec.inject_patches_variant(root)
+            _inject_patches_variant(root)
 
         # Add external paths to specs with just external modules
         for s in self._specs.values():
-            spack.spec.Spec.ensure_external_path_if_external(s)
+            _ensure_external_path_if_external(s)
 
         for s in self._specs.values():
             _develop_specs_from_env(s, ev.active_environment())
@@ -3778,6 +3783,92 @@ def execute_explicit_splices(self):
         return specs
 
 
+def _inject_patches_variant(root: spack.spec.Spec) -> None:
+    # This dictionary will store object IDs rather than Specs as keys
+    # since the Spec __hash__ will change as patches are added to them
+    spec_to_patches: Dict[int, Set[spack.patch.Patch]] = {}
+    for s in root.traverse():
+        # After concretizing, assign namespaces to anything left.
+        # Note that this doesn't count as a "change".  The repository
+        # configuration is constant throughout a spack run, and
+        # normalize and concretize evaluate Packages using Repo.get(),
+        # which respects precedence.  So, a namespace assignment isn't
+        # changing how a package name would have been interpreted and
+        # we can do it as late as possible to allow as much
+        # compatibility across repositories as possible.
+        if s.namespace is None:
+            s.namespace = spack.repo.PATH.repo_for_pkg(s.name).namespace
+
+        if s.concrete:
+            continue
+
+        # Add any patches from the package to the spec.
+        node_patches = {
+            patch
+            for cond, patch_list in spack.repo.PATH.get_pkg_class(s.fullname).patches.items()
+            if s.satisfies(cond)
+            for patch in patch_list
+        }
+        if node_patches:
+            spec_to_patches[id(s)] = node_patches
+
+    # Also record all patches required on dependencies by depends_on(..., patch=...)
+    for dspec in root.traverse_edges(deptype=dt.ALL, cover="edges", root=False):
+        if dspec.spec.concrete:
+            continue
+
+        pkg_deps = spack.repo.PATH.get_pkg_class(dspec.parent.fullname).dependencies
+
+        edge_patches: List[spack.patch.Patch] = []
+        for cond, deps_by_name in pkg_deps.items():
+            if not dspec.parent.satisfies(cond):
+                continue
+
+            dependency = deps_by_name.get(dspec.spec.name)
+            if not dependency:
+                continue
+
+            for pcond, patch_list in dependency.patches.items():
+                if dspec.spec.satisfies(pcond):
+                    edge_patches.extend(patch_list)
+
+        if edge_patches:
+            spec_to_patches.setdefault(id(dspec.spec), set()).update(edge_patches)
+
+    for spec in root.traverse():
+        if id(spec) not in spec_to_patches:
+            continue
+
+        patches = list(spec_to_patches[id(spec)])
+        variant: vt.MultiValuedVariant = spec.variants.setdefault(
+            "patches", vt.MultiValuedVariant("patches", ())
+        )
+        variant.value = tuple(p.sha256 for p in patches)
+        # FIXME: Monkey patches variant to store patches order
+        ordered_hashes = [(*p.ordering_key, p.sha256) for p in patches if p.ordering_key]
+        ordered_hashes.sort()
+        tty.debug(
+            f"Ordered hashes [{spec.name}]: "
+            + ", ".join("/".join(str(e) for e in t) for t in ordered_hashes)
+        )
+        setattr(
+            variant, "_patches_in_order_of_appearance", [sha256 for _, _, sha256 in ordered_hashes]
+        )
+
+
+def _ensure_external_path_if_external(spec: spack.spec.Spec) -> None:
+    if not spec.external_modules or spec.external_path:
+        return
+
+    # Get the path from the module the package can override the default
+    # (this is mostly needed for Cray)
+    pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)
+    package = pkg_cls(spec)
+    spec.external_path = getattr(package, "external_prefix", None) or md.path_from_modules(
+        spec.external_modules
+    )
+
+
 def _develop_specs_from_env(spec, env):
     dev_info = env.dev_specs.get(spec.name, {}) if env else {}
     if not dev_info:

lib/spack/spack/solver/concretize.lp

@@ -524,6 +524,16 @@ error(10, "'{0}' is not a valid dependency for any package in the DAG", Package)
   :- attr("node", node(ID, Package)),
      not needed(node(ID, Package)).
 
+
+% Extensions depending on each other must all extend the same node (e.g. all Python packages
+% depending on each other must depend on the same Python interpreter)
+error(100, "{0} and {1} must depend on the same {2}", ExtensionParent, ExtensionChild, ExtendeePackage)
+  :- depends_on(ExtensionParent, ExtensionChild),
+     attr("extends", ExtensionParent, ExtendeePackage),
+     depends_on(ExtensionParent, node(X, ExtendeePackage)),
+     depends_on(ExtensionChild,  node(Y, ExtendeePackage)),
+     X != Y.
+
 #defined dependency_type/2.
 
 %-----------------------------------------------------------------------------

lib/spack/spack/solver/requirements.py

@@ -10,7 +10,7 @@
 import spack.error
 import spack.package_base
 import spack.spec
-from spack.config import get_mark_from_yaml_data
+from spack.util.spack_yaml import get_mark_from_yaml_data
 
 
 class RequirementKind(enum.Enum):
@@ -165,7 +165,8 @@ def _rules_from_requirements(
 
                 # validate specs from YAML first, and fail with line numbers if parsing fails.
                 constraints = [
-                    parse_spec_from_yaml_string(constraint) for constraint in constraints
+                    parse_spec_from_yaml_string(constraint, named=kind == RequirementKind.VIRTUAL)
+                    for constraint in constraints
                 ]
                 when_str = requirement.get("when")
                 when = parse_spec_from_yaml_string(when_str) if when_str else spack.spec.Spec()
@@ -203,7 +204,7 @@ def reject_requirement_constraint(
                 s.validate_or_raise()
             except spack.error.SpackError as e:
                 tty.debug(
-                    f"[SETUP] Rejecting the default '{constraint}' requirement "
+                    f"[{__name__}] Rejecting the default '{constraint}' requirement "
                     f"on '{pkg_name}': {str(e)}",
                     level=2,
                 )
@@ -211,21 +212,37 @@ def reject_requirement_constraint(
         return False
 
 
-def parse_spec_from_yaml_string(string: str) -> spack.spec.Spec:
+def parse_spec_from_yaml_string(string: str, *, named: bool = False) -> spack.spec.Spec:
     """Parse a spec from YAML and add file/line info to errors, if it's available.
 
     Parse a ``Spec`` from the supplied string, but also intercept any syntax errors and
     add file/line information for debugging using file/line annotations from the string.
 
-    Arguments:
+    Args:
         string: a string representing a ``Spec`` from config YAML.
-
+        named: if True, the spec must have a name
     """
     try:
-        return spack.spec.Spec(string)
+        result = spack.spec.Spec(string)
     except spack.error.SpecSyntaxError as e:
         mark = get_mark_from_yaml_data(string)
         if mark:
             msg = f"{mark.name}:{mark.line + 1}: {str(e)}"
             raise spack.error.SpecSyntaxError(msg) from e
         raise e
+
+    if named is True and not result.name:
+        msg = f"expected a named spec, but got '{string}' instead"
+        mark = get_mark_from_yaml_data(string)
+
+        # Add a hint in case it's dependencies
+        deps = result.dependencies()
+        if len(deps) == 1:
+            msg = f"{msg}. Did you mean '{deps[0]}'?"
+
+        if mark:
+            msg = f"{mark.name}:{mark.line + 1}: {msg}"
+
+        raise spack.error.SpackError(msg)
+
+    return result

lib/spack/spack/spec.py

@@ -99,7 +99,6 @@
 import spack.traverse
 import spack.util.executable
 import spack.util.hash
-import spack.util.module_cmd as md
 import spack.util.prefix
 import spack.util.spack_json as sjson
 import spack.util.spack_yaml as syaml
@@ -176,15 +175,17 @@
 #:     Spec(Spec("string").format()) == Spec("string)"
 DEFAULT_FORMAT = (
     "{name}{@versions}"
-    "{%compiler.name}{@compiler.versions}{compiler_flags}"
+    "{compiler_flags}"
     "{variants}{ namespace=namespace_if_anonymous}{ arch=architecture}{/abstract_hash}"
+    " {%compiler.name}{@compiler.versions}"
 )
 
 #: Display format, which eliminates extra `@=` in the output, for readability.
 DISPLAY_FORMAT = (
     "{name}{@version}"
-    "{%compiler.name}{@compiler.version}{compiler_flags}"
+    "{compiler_flags}"
     "{variants}{ namespace=namespace_if_anonymous}{ arch=architecture}{/abstract_hash}"
+    " {%compiler.name}{@compiler.version}"
 )
 
 #: Regular expression to pull spec contents out of clearsigned signature
@@ -799,7 +800,7 @@ def update_deptypes(self, depflag: dt.DepFlag) -> bool:
         self.depflag = new
         return True
 
-    def update_virtuals(self, virtuals: Tuple[str, ...]) -> bool:
+    def update_virtuals(self, virtuals: Iterable[str]) -> bool:
         """Update the list of provided virtuals"""
         old = self.virtuals
         self.virtuals = tuple(sorted(set(virtuals).union(self.virtuals)))
@@ -1514,7 +1515,7 @@ def __init__(self, spec_like=None, *, external_path=None, external_modules=None)
         self.abstract_hash = None
 
         # initial values for all spec hash types
-        for h in ht.hashes:
+        for h in ht.HASHES:
             setattr(self, h.attr, None)
 
         # cache for spec's prefix, computed lazily by prefix property
@@ -2107,32 +2108,34 @@ def long_spec(self):
     def short_spec(self):
         """Returns a version of the spec with the dependencies hashed
         instead of completely enumerated."""
-        spec_format = "{name}{@version}{%compiler.name}{@compiler.version}"
-        spec_format += "{variants}{ arch=architecture}{/hash:7}"
-        return self.format(spec_format)
+        return self.format(
+            "{name}{@version}{variants}{ arch=architecture}"
+            "{/hash:7}{%compiler.name}{@compiler.version}"
+        )
 
     @property
     def cshort_spec(self):
         """Returns an auto-colorized version of ``self.short_spec``."""
-        spec_format = "{name}{@version}{%compiler.name}{@compiler.version}"
-        spec_format += "{variants}{ arch=architecture}{/hash:7}"
-        return self.cformat(spec_format)
+        return self.cformat(
+            "{name}{@version}{variants}{ arch=architecture}"
+            "{/hash:7}{%compiler.name}{@compiler.version}"
+        )
 
     @property
-    def prefix(self):
+    def prefix(self) -> spack.util.prefix.Prefix:
         if not self._concrete:
-            raise spack.error.SpecError("Spec is not concrete: " + str(self))
+            raise spack.error.SpecError(f"Spec is not concrete: {self}")
 
         if self._prefix is None:
-            upstream, record = spack.store.STORE.db.query_by_spec_hash(self.dag_hash())
+            _, record = spack.store.STORE.db.query_by_spec_hash(self.dag_hash())
             if record and record.path:
-                self.prefix = record.path
+                self.set_prefix(record.path)
             else:
-                self.prefix = spack.store.STORE.layout.path_for_spec(self)
+                self.set_prefix(spack.store.STORE.layout.path_for_spec(self))
+        assert self._prefix is not None
         return self._prefix
 
-    @prefix.setter
-    def prefix(self, value):
+    def set_prefix(self, value: str) -> None:
         self._prefix = spack.util.prefix.Prefix(llnl.path.convert_to_platform_path(value))
 
     def spec_hash(self, hash):
@@ -2195,30 +2198,16 @@ def package_hash(self):
     def dag_hash(self, length=None):
         """This is Spack's default hash, used to identify installations.
 
-        Same as the full hash (includes package hash and build/link/run deps).
-        Tells us when package files and any dependencies have changes.
-
         NOTE: Versions of Spack prior to 0.18 only included link and run deps.
+        NOTE: Versions of Spack prior to 1.0 only did not include test deps.
 
         """
         return self._cached_hash(ht.dag_hash, length)
 
-    def process_hash(self, length=None):
-        """Hash used to transfer specs among processes.
-
-        This hash includes build and test dependencies and is only used to
-        serialize a spec and pass it around among processes.
-        """
-        return self._cached_hash(ht.process_hash, length)
-
     def dag_hash_bit_prefix(self, bits):
         """Get the first <bits> bits of the DAG hash as an integer type."""
         return spack.util.hash.base32_prefix_bits(self.dag_hash(), bits)
 
-    def process_hash_bit_prefix(self, bits):
-        """Get the first <bits> bits of the DAG hash as an integer type."""
-        return spack.util.hash.base32_prefix_bits(self.process_hash(), bits)
-
     def _lookup_hash(self):
         """Lookup just one spec with an abstract hash, returning a spec from the the environment,
         store, or finally, binary caches."""
@@ -2738,7 +2727,7 @@ def spec_and_dependency_types(
         return spec_builder(spec_dict)
 
     @staticmethod
-    def from_dict(data):
+    def from_dict(data) -> "Spec":
         """Construct a spec from JSON/YAML.
 
         Args:
@@ -2761,7 +2750,7 @@ def from_dict(data):
         return spec
 
     @staticmethod
-    def from_yaml(stream):
+    def from_yaml(stream) -> "Spec":
         """Construct a spec from YAML.
 
         Args:
@@ -2771,7 +2760,7 @@ def from_yaml(stream):
         return Spec.from_dict(data)
 
     @staticmethod
-    def from_json(stream):
+    def from_json(stream) -> "Spec":
         """Construct a spec from JSON.
 
         Args:
@@ -2781,7 +2770,7 @@ def from_json(stream):
             data = sjson.load(stream)
             return Spec.from_dict(data)
         except Exception as e:
-            raise sjson.SpackJSONError("error parsing JSON spec:", str(e)) from e
+            raise sjson.SpackJSONError("error parsing JSON spec:", e) from e
 
     @staticmethod
     def extract_json_from_clearsig(data):
@@ -2845,94 +2834,6 @@ def _patches_assigned(self):
 
         return True
 
-    @staticmethod
-    def inject_patches_variant(root):
-        # This dictionary will store object IDs rather than Specs as keys
-        # since the Spec __hash__ will change as patches are added to them
-        spec_to_patches = {}
-        for s in root.traverse():
-            # After concretizing, assign namespaces to anything left.
-            # Note that this doesn't count as a "change".  The repository
-            # configuration is constant throughout a spack run, and
-            # normalize and concretize evaluate Packages using Repo.get(),
-            # which respects precedence.  So, a namespace assignment isn't
-            # changing how a package name would have been interpreted and
-            # we can do it as late as possible to allow as much
-            # compatibility across repositories as possible.
-            if s.namespace is None:
-                s.namespace = spack.repo.PATH.repo_for_pkg(s.name).namespace
-
-            if s.concrete:
-                continue
-
-            # Add any patches from the package to the spec.
-            patches = set()
-            for cond, patch_list in spack.repo.PATH.get_pkg_class(s.fullname).patches.items():
-                if s.satisfies(cond):
-                    for patch in patch_list:
-                        patches.add(patch)
-            if patches:
-                spec_to_patches[id(s)] = patches
-
-        # Also record all patches required on dependencies by
-        # depends_on(..., patch=...)
-        for dspec in root.traverse_edges(deptype=all, cover="edges", root=False):
-            if dspec.spec.concrete:
-                continue
-
-            pkg_deps = spack.repo.PATH.get_pkg_class(dspec.parent.fullname).dependencies
-
-            patches = []
-            for cond, deps_by_name in pkg_deps.items():
-                if not dspec.parent.satisfies(cond):
-                    continue
-
-                dependency = deps_by_name.get(dspec.spec.name)
-                if not dependency:
-                    continue
-
-                for pcond, patch_list in dependency.patches.items():
-                    if dspec.spec.satisfies(pcond):
-                        patches.extend(patch_list)
-
-            if patches:
-                all_patches = spec_to_patches.setdefault(id(dspec.spec), set())
-                for patch in patches:
-                    all_patches.add(patch)
-
-        for spec in root.traverse():
-            if id(spec) not in spec_to_patches:
-                continue
-
-            patches = list(lang.dedupe(spec_to_patches[id(spec)]))
-            mvar = spec.variants.setdefault("patches", vt.MultiValuedVariant("patches", ()))
-            mvar.value = tuple(p.sha256 for p in patches)
-            # FIXME: Monkey patches mvar to store patches order
-            full_order_keys = list(tuple(p.ordering_key) + (p.sha256,) for p in patches)
-            ordered_hashes = sorted(full_order_keys)
-            tty.debug(
-                "Ordered hashes [{0}]: ".format(spec.name)
-                + ", ".join("/".join(str(e) for e in t) for t in ordered_hashes)
-            )
-            mvar._patches_in_order_of_appearance = list(t[-1] for t in ordered_hashes)
-
-    @staticmethod
-    def ensure_external_path_if_external(external_spec):
-        if external_spec.external_modules and not external_spec.external_path:
-            compiler = spack.compilers.compiler_for_spec(
-                external_spec.compiler, external_spec.architecture
-            )
-            for mod in compiler.modules:
-                md.load_module(mod)
-
-            # Get the path from the module the package can override the default
-            # (this is mostly needed for Cray)
-            pkg_cls = spack.repo.PATH.get_pkg_class(external_spec.name)
-            package = pkg_cls(external_spec)
-            external_spec.external_path = getattr(
-                package, "external_prefix", md.path_from_modules(external_spec.external_modules)
-            )
-
     @staticmethod
     def ensure_no_deprecated(root):
         """Raise if a deprecated spec is in the dag.
@@ -3673,11 +3574,11 @@ def _dup(self, other: "Spec", deps: Union[bool, dt.DepTypes, dt.DepFlag] = True)
 
         if self._concrete:
             self._dunder_hash = other._dunder_hash
-            for h in ht.hashes:
+            for h in ht.HASHES:
                 setattr(self, h.attr, getattr(other, h.attr, None))
         else:
             self._dunder_hash = None
-            for h in ht.hashes:
+            for h in ht.HASHES:
                 setattr(self, h.attr, None)
 
         return changed
@@ -3869,16 +3770,6 @@ def _cmp_iter(self):
         # serialized before the hash change and one after, are considered different.
         yield self.dag_hash() if self.concrete else None
 
-        # This needs to be in _cmp_iter so that no specs with different process hashes
-        # are considered the same by `__hash__` or `__eq__`.
-        #
-        # TODO: We should eventually unify the `_cmp_*` methods with `to_node_dict` so
-        # TODO: there aren't two sources of truth, but this needs some thought, since
-        # TODO: they exist for speed.  We should benchmark whether it's really worth
-        # TODO: having two types of hashing now that we use `json` instead of `yaml` for
-        # TODO: spec hashing.
-        yield self.process_hash() if self.concrete else None
-
         def deps():
             for dep in sorted(itertools.chain.from_iterable(self._dependencies.values())):
                 yield dep.spec.name
@@ -4532,7 +4423,7 @@ def clear_caches(self, ignore: Tuple[str, ...] = ()) -> None:
         """
         Clears all cached hashes in a Spec, while preserving other properties.
         """
-        for h in ht.hashes:
+        for h in ht.HASHES:
             if h.attr not in ignore:
                 if hasattr(self, h.attr):
                     setattr(self, h.attr, None)
@@ -4541,18 +4432,12 @@ def clear_caches(self, ignore: Tuple[str, ...] = ()) -> None:
                 setattr(self, attr, None)
 
     def __hash__(self):
-        # If the spec is concrete, we leverage the process hash and just use
-        # a 64-bit prefix of it. The process hash has the advantage that it's
-        # computed once per concrete spec, and it's saved -- so if we read
-        # concrete specs we don't need to recompute the whole hash. This is
-        # good for large, unchanging specs.
-        #
-        # We use the process hash instead of the DAG hash here because the DAG
-        # hash includes the package hash, which can cause infinite recursion,
-        # and which isn't defined unless the spec has a known package.
+        # If the spec is concrete, we leverage the dag hash and just use a 64-bit prefix of it.
+        # The dag hash has the advantage that it's computed once per concrete spec, and it's saved
+        # -- so if we read concrete specs we don't need to recompute the whole hash.
         if self.concrete:
             if not self._dunder_hash:
-                self._dunder_hash = self.process_hash_bit_prefix(64)
+                self._dunder_hash = self.dag_hash_bit_prefix(64)
             return self._dunder_hash
 
         # This is the normal hash for lazy_lexicographic_ordering. It's
@@ -4561,7 +4446,7 @@ def __hash__(self):
         return hash(lang.tuplify(self._cmp_iter))
 
     def __reduce__(self):
-        return Spec.from_dict, (self.to_dict(hash=ht.process_hash),)
+        return Spec.from_dict, (self.to_dict(hash=ht.dag_hash),)
 
     def attach_git_version_lookup(self):
         # Add a git lookup method for GitVersions
@@ -4704,17 +4589,6 @@ def constrain(self, other: "VariantMap") -> bool:
 
         return changed
 
-    @property
-    def concrete(self):
-        """Returns True if the spec is concrete in terms of variants.
-
-        Returns:
-            bool: True or False
-        """
-        return self.spec._concrete or all(
-            v in self for v in spack.repo.PATH.get_pkg_class(self.spec.fullname).variant_names()
-        )
-
     def copy(self) -> "VariantMap":
         clone = VariantMap(self.spec)
         for name, variant in self.items():
@@ -4841,33 +4715,51 @@ def merge_abstract_anonymous_specs(*abstract_specs: Spec):
     return merged_spec
 
 
-def reconstruct_virtuals_on_edges(spec):
-    """Reconstruct virtuals on edges. Used to read from old DB and reindex.
+def reconstruct_virtuals_on_edges(spec: Spec) -> None:
+    """Reconstruct virtuals on edges. Used to read from old DB and reindex."""
+    virtuals_needed: Dict[str, Set[str]] = {}
+    virtuals_provided: Dict[str, Set[str]] = {}
+    for edge in spec.traverse_edges(cover="edges", root=False):
+        parent_key = edge.parent.dag_hash()
+        if parent_key not in virtuals_needed:
+            # Construct which virtuals are needed by parent
+            virtuals_needed[parent_key] = set()
+            try:
+                parent_pkg = edge.parent.package
+            except Exception as e:
+                warnings.warn(
+                    f"cannot reconstruct virtual dependencies on {edge.parent.name}: {e}"
+                )
+                continue
 
-    Args:
-        spec: spec on which we want to reconstruct virtuals
-    """
-    # Collect all possible virtuals
-    possible_virtuals = set()
-    for node in spec.traverse():
-        try:
-            possible_virtuals.update(
-                {x for x in node.package.dependencies if spack.repo.PATH.is_virtual(x)}
+            virtuals_needed[parent_key].update(
+                name
+                for name, when_deps in parent_pkg.dependencies_by_name(when=True).items()
+                if spack.repo.PATH.is_virtual(name)
+                and any(edge.parent.satisfies(x) for x in when_deps)
             )
-        except Exception as e:
-            warnings.warn(f"cannot reconstruct virtual dependencies on package {node.name}: {e}")
+
+        if not virtuals_needed[parent_key]:
             continue
 
-    # Assume all incoming edges to provider are marked with virtuals=
-    for vspec in possible_virtuals:
-        try:
-            provider = spec[vspec]
-        except KeyError:
-            # Virtual not in the DAG
+        child_key = edge.spec.dag_hash()
+        if child_key not in virtuals_provided:
+            virtuals_provided[child_key] = set()
+            try:
+                child_pkg = edge.spec.package
+            except Exception as e:
+                warnings.warn(
+                    f"cannot reconstruct virtual dependencies on {edge.parent.name}: {e}"
+                )
+                continue
+            virtuals_provided[child_key].update(x.name for x in child_pkg.virtuals_provided)
+
+        if not virtuals_provided[child_key]:
             continue
 
-        for edge in provider.edges_from_dependents():
-            edge.update_virtuals([vspec])
+        virtuals_to_add = virtuals_needed[parent_key] & virtuals_provided[child_key]
+        if virtuals_to_add:
+            edge.update_virtuals(virtuals_to_add)
 
 
 class SpecfileReaderBase:
@@ -4876,7 +4768,7 @@ def from_node_dict(cls, node):
         spec = Spec()
 
         name, node = cls.name_and_data(node)
-        for h in ht.hashes:
+        for h in ht.HASHES:
             setattr(spec, h.attr, node.get(h.name, None))
 
         spec.name = name
@@ -5059,7 +4951,7 @@ def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):
         """
         for dep_name, elt in deps.items():
             if isinstance(elt, dict):
-                for h in ht.hashes:
+                for h in ht.HASHES:
                     if h.name in elt:
                         dep_hash, deptypes = elt[h.name], elt["type"]
                         hash_type = h.name
@@ -5102,7 +4994,7 @@ def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):
             dep_name = dep["name"]
             if isinstance(elt, dict):
                 # new format: elements of dependency spec are keyed.
-                for h in ht.hashes:
+                for h in ht.HASHES:
                     if h.name in elt:
                         dep_hash, deptypes, hash_type, virtuals = cls.extract_info_from_dep(elt, h)
                         break
@@ -5212,6 +5104,13 @@ def get_host_environment() -> Dict[str, Any]:
     }
 
 
+def eval_conditional(string):
+    """Evaluate conditional definitions using restricted variable scope."""
+    valid_variables = get_host_environment()
+    valid_variables.update({"re": re, "env": os.environ})
+    return eval(string, valid_variables)
+
+
 class SpecParseError(spack.error.SpecError):
     """Wrapper for ParseError for when we're parsing specs."""
 
@@ -5370,8 +5269,10 @@ def __init__(self, spec):
 
 class AmbiguousHashError(spack.error.SpecError):
     def __init__(self, msg, *specs):
-        spec_fmt = "{namespace}.{name}{@version}{%compiler}{compiler_flags}"
-        spec_fmt += "{variants}{ arch=architecture}{/hash:7}"
+        spec_fmt = (
+            "{namespace}.{name}{@version}{compiler_flags}{variants}"
+            "{ arch=architecture}{/hash:7}{%compiler}"
+        )
         specs_str = "\n  " + "\n  ".join(spec.format(spec_fmt) for spec in specs)
         super().__init__(msg + specs_str)
 

lib/spack/spack/spec_parser.py

@@ -60,13 +60,17 @@
 import pathlib
 import re
 import sys
-from typing import Iterator, List, Optional
+import traceback
+import warnings
+from typing import Iterator, List, Optional, Tuple
 
 from llnl.util.tty import color
 
 import spack.deptypes
 import spack.error
+import spack.paths
 import spack.spec
+import spack.util.spack_yaml
 import spack.version
 from spack.tokenize import Token, TokenBase, Tokenizer
 
@@ -204,6 +208,32 @@ def __init__(self, tokens: List[Token], text: str):
         super().__init__(message)
 
 
+def _warn_about_variant_after_compiler(literal_str: str, issues: List[str]):
+    """Issue a warning if variant or other token is preceded by a compiler token. The warning is
+    only issued if it's actionable: either we know the config file it originates from, or we have
+    call site that's not internal to Spack."""
+    ignore = [spack.paths.lib_path, spack.paths.bin_path]
+    mark = spack.util.spack_yaml.get_mark_from_yaml_data(literal_str)
+    issue_str = ", ".join(issues)
+    error = f"{issue_str} in `{literal_str}`"
+
+    # warning from config file
+    if mark:
+        warnings.warn(f"{mark.name}:{mark.line + 1}: {error}")
+        return
+
+    # warning from hopefully package.py
+    for frame in reversed(traceback.extract_stack()):
+        if frame.lineno and not any(frame.filename.startswith(path) for path in ignore):
+            warnings.warn_explicit(
+                error,
+                category=spack.error.SpackAPIWarning,
+                filename=frame.filename,
+                lineno=frame.lineno,
+            )
+            return
+
+
 class SpecParser:
     """Parse text into specs"""
 
@@ -242,26 +272,31 @@ def add_dependency(dep, **edge_properties):
                 raise SpecParsingError(str(e), self.ctx.current_token, self.literal_str) from e
 
         initial_spec = initial_spec or spack.spec.Spec()
-        root_spec = SpecNodeParser(self.ctx, self.literal_str).parse(initial_spec)
+        root_spec, parser_warnings = SpecNodeParser(self.ctx, self.literal_str).parse(initial_spec)
         while True:
             if self.ctx.accept(SpecTokens.START_EDGE_PROPERTIES):
                 edge_properties = EdgeAttributeParser(self.ctx, self.literal_str).parse()
                 edge_properties.setdefault("depflag", 0)
                 edge_properties.setdefault("virtuals", ())
-                dependency = self._parse_node(root_spec)
+                dependency, warnings = self._parse_node(root_spec)
+                parser_warnings.extend(warnings)
                 add_dependency(dependency, **edge_properties)
 
             elif self.ctx.accept(SpecTokens.DEPENDENCY):
-                dependency = self._parse_node(root_spec)
+                dependency, warnings = self._parse_node(root_spec)
+                parser_warnings.extend(warnings)
                 add_dependency(dependency, depflag=0, virtuals=())
 
             else:
                 break
 
+        if parser_warnings:
+            _warn_about_variant_after_compiler(self.literal_str, parser_warnings)
+
         return root_spec
 
     def _parse_node(self, root_spec):
-        dependency = SpecNodeParser(self.ctx, self.literal_str).parse()
+        dependency, parser_warnings = SpecNodeParser(self.ctx, self.literal_str).parse()
         if dependency is None:
             msg = (
                 "the dependency sigil and any optional edge attributes must be followed by a "
@@ -270,7 +305,7 @@ def _parse_node(self, root_spec):
             raise SpecParsingError(msg, self.ctx.current_token, self.literal_str)
         if root_spec.concrete:
             raise spack.spec.RedundantSpecError(root_spec, "^" + str(dependency))
-        return dependency
+        return dependency, parser_warnings
 
     def all_specs(self) -> List["spack.spec.Spec"]:
         """Return all the specs that remain to be parsed"""
@@ -290,7 +325,7 @@ def __init__(self, ctx, literal_str):
 
     def parse(
         self, initial_spec: Optional["spack.spec.Spec"] = None
-    ) -> Optional["spack.spec.Spec"]:
+    ) -> Tuple["spack.spec.Spec", List[str]]:
         """Parse a single spec node from a stream of tokens
 
         Args:
@@ -299,12 +334,15 @@ def parse(
         Return
             The object passed as argument
         """
-        if not self.ctx.next_token or self.ctx.expect(SpecTokens.DEPENDENCY):
-            return initial_spec
+        parser_warnings: List[str] = []
+        last_compiler = None
 
         if initial_spec is None:
             initial_spec = spack.spec.Spec()
 
+        if not self.ctx.next_token or self.ctx.expect(SpecTokens.DEPENDENCY):
+            return initial_spec, parser_warnings
+
         # If we start with a package name we have a named spec, we cannot
         # accept another package name afterwards in a node
         if self.ctx.accept(SpecTokens.UNQUALIFIED_PACKAGE_NAME):
@@ -318,7 +356,7 @@ def parse(
             initial_spec.namespace = namespace
 
         elif self.ctx.accept(SpecTokens.FILENAME):
-            return FileParser(self.ctx).parse(initial_spec)
+            return FileParser(self.ctx).parse(initial_spec), parser_warnings
 
         def raise_parsing_error(string: str, cause: Optional[Exception] = None):
             """Raise a spec parsing error with token context."""
@@ -331,6 +369,12 @@ def add_flag(name: str, value: str, propagate: bool):
             except Exception as e:
                 raise_parsing_error(str(e), e)
 
+        def warn_if_after_compiler(token: str):
+            """Register a warning for %compiler followed by +variant that will in the future apply
+            to the compiler instead of the current root."""
+            if last_compiler:
+                parser_warnings.append(f"`{token}` should go before `{last_compiler}`")
+
         while True:
             if self.ctx.accept(SpecTokens.COMPILER):
                 if self.has_compiler:
@@ -339,6 +383,7 @@ def add_flag(name: str, value: str, propagate: bool):
                 compiler_name = self.ctx.current_token.value[1:]
                 initial_spec.compiler = spack.spec.CompilerSpec(compiler_name.strip(), ":")
                 self.has_compiler = True
+                last_compiler = self.ctx.current_token.value
 
             elif self.ctx.accept(SpecTokens.COMPILER_AND_VERSION):
                 if self.has_compiler:
@@ -349,6 +394,7 @@ def add_flag(name: str, value: str, propagate: bool):
                     compiler_name.strip(), compiler_version
                 )
                 self.has_compiler = True
+                last_compiler = self.ctx.current_token.value
 
             elif (
                 self.ctx.accept(SpecTokens.VERSION_HASH_PAIR)
@@ -363,14 +409,17 @@ def add_flag(name: str, value: str, propagate: bool):
                 )
                 initial_spec.attach_git_version_lookup()
                 self.has_version = True
+                warn_if_after_compiler(self.ctx.current_token.value)
 
             elif self.ctx.accept(SpecTokens.BOOL_VARIANT):
                 variant_value = self.ctx.current_token.value[0] == "+"
                 add_flag(self.ctx.current_token.value[1:].strip(), variant_value, propagate=False)
+                warn_if_after_compiler(self.ctx.current_token.value)
 
             elif self.ctx.accept(SpecTokens.PROPAGATED_BOOL_VARIANT):
                 variant_value = self.ctx.current_token.value[0:2] == "++"
                 add_flag(self.ctx.current_token.value[2:].strip(), variant_value, propagate=True)
+                warn_if_after_compiler(self.ctx.current_token.value)
 
             elif self.ctx.accept(SpecTokens.KEY_VALUE_PAIR):
                 match = SPLIT_KVP.match(self.ctx.current_token.value)
@@ -378,6 +427,7 @@ def add_flag(name: str, value: str, propagate: bool):
 
                 name, _, value = match.groups()
                 add_flag(name, strip_quotes_and_unescape(value), propagate=False)
+                warn_if_after_compiler(self.ctx.current_token.value)
 
             elif self.ctx.accept(SpecTokens.PROPAGATED_KEY_VALUE_PAIR):
                 match = SPLIT_KVP.match(self.ctx.current_token.value)
@@ -385,17 +435,19 @@ def add_flag(name: str, value: str, propagate: bool):
 
                 name, _, value = match.groups()
                 add_flag(name, strip_quotes_and_unescape(value), propagate=True)
+                warn_if_after_compiler(self.ctx.current_token.value)
 
             elif self.ctx.expect(SpecTokens.DAG_HASH):
                 if initial_spec.abstract_hash:
                     break
                 self.ctx.accept(SpecTokens.DAG_HASH)
                 initial_spec.abstract_hash = self.ctx.current_token.value[1:]
+                warn_if_after_compiler(self.ctx.current_token.value)
 
             else:
                 break
 
-        return initial_spec
+        return initial_spec, parser_warnings
 
 
 class FileParser:
@@ -485,23 +537,18 @@ def parse_one_or_raise(
         text (str): text to be parsed
         initial_spec: buffer where to parse the spec. If None a new one will be created.
     """
-    stripped_text = text.strip()
-    parser = SpecParser(stripped_text)
+    parser = SpecParser(text)
     result = parser.next_spec(initial_spec)
-    last_token = parser.ctx.current_token
+    next_token = parser.ctx.next_token
 
-    if last_token is not None and last_token.end != len(stripped_text):
-        message = "a single spec was requested, but parsed more than one:"
-        message += f"\n{text}"
-        if last_token is not None:
-            underline = f"\n{' ' * last_token.end}{'^' * (len(text) - last_token.end)}"
-            message += color.colorize(f"@*r{{{underline}}}")
+    if next_token:
+        message = f"expected a single spec, but got more:\n{text}"
+        underline = f"\n{' ' * next_token.start}{'^' * len(next_token.value)}"
+        message += color.colorize(f"@*r{{{underline}}}")
         raise ValueError(message)
 
     if result is None:
-        message = "a single spec was requested, but none was parsed:"
-        message += f"\n{text}"
-        raise ValueError(message)
+        raise ValueError("expected a single spec, but got none")
 
     return result
 

lib/spack/spack/test/architecture.py

@@ -129,7 +129,7 @@ def test_satisfy_strict_constraint_when_not_concrete(architecture_tuple, constra
 )
 def test_concretize_target_ranges(root_target_range, dep_target_range, result, monkeypatch):
     spec = Spec(
-        f"pkg-a %gcc@10 foobar=bar target={root_target_range} ^pkg-b target={dep_target_range}"
+        f"pkg-a foobar=bar target={root_target_range} %gcc@10 ^pkg-b target={dep_target_range}"
     )
     with spack.concretize.disable_compiler_existence_check():
         spec = spack.concretize.concretize_one(spec)

lib/spack/spack/test/bindist.py

@@ -200,7 +200,11 @@ def dummy_prefix(tmpdir):
 @pytest.mark.requires_executables(*required_executables)
 @pytest.mark.maybeslow
 @pytest.mark.usefixtures(
-    "default_config", "cache_directory", "install_dir_default_layout", "temporary_mirror"
+    "default_config",
+    "cache_directory",
+    "install_dir_default_layout",
+    "temporary_mirror",
+    "mutable_mock_env_path",
 )
 def test_default_rpaths_create_install_default_layout(temporary_mirror_dir):
     """
@@ -272,7 +276,11 @@ def test_default_rpaths_install_nondefault_layout(temporary_mirror_dir):
 @pytest.mark.maybeslow
 @pytest.mark.nomockstage
 @pytest.mark.usefixtures(
-    "default_config", "cache_directory", "install_dir_default_layout", "temporary_mirror"
+    "default_config",
+    "cache_directory",
+    "install_dir_default_layout",
+    "temporary_mirror",
+    "mutable_mock_env_path",
 )
 def test_relative_rpaths_install_default_layout(temporary_mirror_dir):
     """
@@ -569,7 +577,6 @@ def test_FetchCacheError_only_accepts_lists_of_errors():
 def test_FetchCacheError_pretty_printing_multiple():
     e = bindist.FetchCacheError([RuntimeError("Oops!"), TypeError("Trouble!")])
     str_e = str(e)
-    print("'" + str_e + "'")
     assert "Multiple errors" in str_e
     assert "Error 1: RuntimeError: Oops!" in str_e
     assert "Error 2: TypeError: Trouble!" in str_e
@@ -1133,7 +1140,7 @@ def test_get_valid_spec_file_no_json(tmp_path, filename):
 
 def test_download_tarball_with_unsupported_layout_fails(tmp_path, mutable_config, capsys):
     layout_version = bindist.CURRENT_BUILD_CACHE_LAYOUT_VERSION + 1
-    spec = Spec("gmake@4.4.1%gcc@13.1.0 arch=linux-ubuntu23.04-zen2")
+    spec = Spec("gmake@4.4.1 arch=linux-ubuntu23.04-zen2 %gcc@13.1.0")
     spec._mark_concrete()
     spec_dict = spec.to_dict()
     spec_dict["buildcache_layout_version"] = layout_version

lib/spack/spack/test/build_environment.py

@@ -388,7 +388,7 @@ def test_wrapper_variables(
     root = spack.concretize.concretize_one("dt-diamond")
 
     for s in root.traverse():
-        s.prefix = "/{0}-prefix/".format(s.name)
+        s.set_prefix(f"/{s.name}-prefix/")
 
     dep_pkg = root["dt-diamond-left"].package
     dep_lib_paths = ["/test/path/to/ex1.so", "/test/path/to/subdir/ex2.so"]
@@ -396,7 +396,7 @@ def test_wrapper_variables(
     dep_libs = LibraryList(dep_lib_paths)
 
     dep2_pkg = root["dt-diamond-right"].package
-    dep2_pkg.spec.prefix = str(installation_dir_with_headers)
+    dep2_pkg.spec.set_prefix(str(installation_dir_with_headers))
 
     setattr(dep_pkg, "libs", dep_libs)
     try:
@@ -542,7 +542,7 @@ def test_build_jobs_sequential_is_sequential():
         spack.config.determine_number_of_jobs(
             parallel=False,
             max_cpus=8,
-            config=spack.config.Configuration(
+            config=spack.config.create_from(
                 spack.config.InternalConfigScope("command_line", {"config": {"build_jobs": 8}}),
                 spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 8}}),
             ),
@@ -556,7 +556,7 @@ def test_build_jobs_command_line_overrides():
         spack.config.determine_number_of_jobs(
             parallel=True,
             max_cpus=1,
-            config=spack.config.Configuration(
+            config=spack.config.create_from(
                 spack.config.InternalConfigScope("command_line", {"config": {"build_jobs": 10}}),
                 spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 1}}),
             ),
@@ -567,7 +567,7 @@ def test_build_jobs_command_line_overrides():
         spack.config.determine_number_of_jobs(
             parallel=True,
             max_cpus=100,
-            config=spack.config.Configuration(
+            config=spack.config.create_from(
                 spack.config.InternalConfigScope("command_line", {"config": {"build_jobs": 10}}),
                 spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 100}}),
             ),
@@ -581,7 +581,7 @@ def test_build_jobs_defaults():
         spack.config.determine_number_of_jobs(
             parallel=True,
             max_cpus=10,
-            config=spack.config.Configuration(
+            config=spack.config.create_from(
                 spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 1}})
             ),
         )
@@ -591,7 +591,7 @@ def test_build_jobs_defaults():
         spack.config.determine_number_of_jobs(
             parallel=True,
             max_cpus=10,
-            config=spack.config.Configuration(
+            config=spack.config.create_from(
                 spack.config.InternalConfigScope("defaults", {"config": {"build_jobs": 100}})
             ),
         )

lib/spack/spack/test/build_systems.py

@@ -403,8 +403,8 @@ def test_autoreconf_search_path_args_multiple(default_mock_concretization, tmpdi
     aclocal_fst = str(tmpdir.mkdir("fst").mkdir("share").mkdir("aclocal"))
     aclocal_snd = str(tmpdir.mkdir("snd").mkdir("share").mkdir("aclocal"))
     build_dep_one, build_dep_two = spec.dependencies(deptype="build")
-    build_dep_one.prefix = str(tmpdir.join("fst"))
-    build_dep_two.prefix = str(tmpdir.join("snd"))
+    build_dep_one.set_prefix(str(tmpdir.join("fst")))
+    build_dep_two.set_prefix(str(tmpdir.join("snd")))
     assert spack.build_systems.autotools._autoreconf_search_path_args(spec) == [
         "-I",
         aclocal_fst,
@@ -422,8 +422,8 @@ def test_autoreconf_search_path_args_skip_automake(default_mock_concretization,
     aclocal_snd = str(tmpdir.mkdir("snd").mkdir("share").mkdir("aclocal"))
     build_dep_one, build_dep_two = spec.dependencies(deptype="build")
     build_dep_one.name = "automake"
-    build_dep_one.prefix = str(tmpdir.join("fst"))
-    build_dep_two.prefix = str(tmpdir.join("snd"))
+    build_dep_one.set_prefix(str(tmpdir.join("fst")))
+    build_dep_two.set_prefix(str(tmpdir.join("snd")))
     assert spack.build_systems.autotools._autoreconf_search_path_args(spec) == ["-I", aclocal_snd]
 
 
@@ -434,7 +434,7 @@ def test_autoreconf_search_path_args_external_order(default_mock_concretization,
     aclocal_snd = str(tmpdir.mkdir("snd").mkdir("share").mkdir("aclocal"))
     build_dep_one, build_dep_two = spec.dependencies(deptype="build")
     build_dep_one.external_path = str(tmpdir.join("fst"))
-    build_dep_two.prefix = str(tmpdir.join("snd"))
+    build_dep_two.set_prefix(str(tmpdir.join("snd")))
     assert spack.build_systems.autotools._autoreconf_search_path_args(spec) == [
         "-I",
         aclocal_snd,
@@ -447,8 +447,8 @@ def test_autoreconf_search_path_skip_nonexisting(default_mock_concretization, tm
     """Skip -I flags for non-existing directories"""
     spec = default_mock_concretization("dttop")
     build_dep_one, build_dep_two = spec.dependencies(deptype="build")
-    build_dep_one.prefix = str(tmpdir.join("fst"))
-    build_dep_two.prefix = str(tmpdir.join("snd"))
+    build_dep_one.set_prefix(str(tmpdir.join("fst")))
+    build_dep_two.set_prefix(str(tmpdir.join("snd")))
     assert spack.build_systems.autotools._autoreconf_search_path_args(spec) == []
 
 

lib/spack/spack/test/cc.py

@@ -210,7 +210,6 @@ def check_args_contents(cc, args, must_contain, must_not_contain):
     """
     with set_env(SPACK_TEST_COMMAND="dump-args"):
         cc_modified_args = cc(*args, output=str).strip().split("\n")
-        print(cc_modified_args)
         for a in must_contain:
             assert a in cc_modified_args
         for a in must_not_contain:

lib/spack/spack/test/ci.py

@@ -347,7 +347,6 @@ def test_get_spec_filter_list(mutable_mock_env_path, mutable_mock_repo):
     for key, val in expectations.items():
         affected_specs = ci.get_spec_filter_list(e1, touched, dependent_traverse_depth=key)
         affected_pkg_names = set([s.name for s in affected_specs])
-        print(f"{key}: {affected_pkg_names}")
         assert affected_pkg_names == val
 
 

lib/spack/spack/test/cmd/buildcache.py

@@ -214,9 +214,7 @@ def verify_mirror_contents():
             if in_env_pkg in p:
                 found_pkg = True
 
-        if not found_pkg:
-            print("Expected to find {0} in {1}".format(in_env_pkg, dest_mirror_dir))
-            assert False
+        assert found_pkg, f"Expected to find {in_env_pkg} in {dest_mirror_dir}"
 
     # Install a package and put it in the buildcache
     s = spack.concretize.concretize_one(out_env_pkg)

lib/spack/spack/test/cmd/ci.py

@@ -867,7 +867,7 @@ def test_push_to_build_cache(
             logs_dir = scratch / "logs_dir"
             logs_dir.mkdir()
             ci.copy_stage_logs_to_artifacts(concrete_spec, str(logs_dir))
-            assert "spack-build-out.txt" in os.listdir(logs_dir)
+            assert "spack-build-out.txt.gz" in os.listdir(logs_dir)
 
             dl_dir = scratch / "download_dir"
             buildcache_cmd("download", "--spec-file", json_path, "--path", str(dl_dir))

lib/spack/spack/test/cmd/commands.py

@@ -5,6 +5,7 @@
 import filecmp
 import os
 import shutil
+import textwrap
 
 import pytest
 
@@ -259,15 +260,25 @@ def test_update_completion_arg(shell, tmpdir, monkeypatch):
 def test_updated_completion_scripts(shell, tmpdir):
     """Make sure our shell tab completion scripts remain up-to-date."""
 
-    msg = (
+    width = 72
+    lines = textwrap.wrap(
         "It looks like Spack's command-line interface has been modified. "
-        "Please update Spack's shell tab completion scripts by running:\n\n"
-        "    spack commands --update-completion\n\n"
-        "and adding the changed files to your pull request."
+        "If differences are more than your global 'include:' scopes, please "
+        "update Spack's shell tab completion scripts by running:",
+        width,
     )
+    lines.append("\n    spack commands --update-completion\n")
+    lines.extend(
+        textwrap.wrap(
+            "and adding the changed files (minus your global 'include:' scopes) "
+            "to your pull request.",
+            width,
+        )
+    )
+    msg = "\n".join(lines)
 
     header = os.path.join(spack.paths.share_path, shell, f"spack-completion.{shell}")
-    script = "spack-completion.{0}".format(shell)
+    script = f"spack-completion.{shell}"
     old_script = os.path.join(spack.paths.share_path, script)
     new_script = str(tmpdir.join(script))
 

lib/spack/spack/test/cmd/config.py

@@ -213,7 +213,7 @@ def test_config_add_update_dict(mutable_empty_config):
 
 def test_config_with_c_argument(mutable_empty_config):
     # I don't know how to add a spack argument to a Spack Command, so we test this way
-    config_file = "config:install_root:root:/path/to/config.yaml"
+    config_file = "config:install_tree:root:/path/to/config.yaml"
     parser = spack.main.make_argument_parser()
     args = parser.parse_args(["-c", config_file])
     assert config_file in args.config_vars
@@ -221,7 +221,7 @@ def test_config_with_c_argument(mutable_empty_config):
     # Add the path to the config
     config("add", args.config_vars[0], scope="command_line")
     output = config("get", "config")
-    assert "config:\n  install_root:\n    root: /path/to/config.yaml" in output
+    assert "config:\n  install_tree:\n    root: /path/to/config.yaml" in output
 
 
 def test_config_add_ordered_dict(mutable_empty_config):

lib/spack/spack/test/cmd/deprecate.py

@@ -15,6 +15,9 @@
 deprecate = SpackCommand("deprecate")
 find = SpackCommand("find")
 
+# Unit tests should not be affected by the user's managed environments
+pytestmark = pytest.mark.usefixtures("mutable_mock_env_path")
+
 
 def test_deprecate(mock_packages, mock_archive, mock_fetch, install_mockery):
     install("--fake", "libelf@0.8.13")

lib/spack/spack/test/cmd/env.py

@@ -1067,13 +1067,17 @@ def test_init_from_yaml_relative_includes(tmp_path):
         assert os.path.exists(os.path.join(e2.path, f))
 
 
+# TODO: Should we be supporting relative path rewrites when creating new env from existing?
+# TODO: If so, then this should confirm that the absolute include paths in the new env exist.
 def test_init_from_yaml_relative_includes_outside_env(tmp_path):
-    files = ["../outside_env_not_copied/repos.yaml"]
+    """Ensure relative includes to files outside the environment fail."""
+    files = ["../outside_env/repos.yaml"]
 
     manifest = f"""
 spack:
   specs: []
-  include: {files}
+  include:
+  - path: {files[0]}
 """
 
     # subdir to ensure parent of environment dir is not shared
@@ -1086,7 +1090,7 @@ def test_init_from_yaml_relative_includes_outside_env(tmp_path):
     for f in files:
         fs.touchp(e1_path / f)
 
-    with pytest.raises(spack.config.ConfigFileError, match="Detected 1 missing include"):
+    with pytest.raises(ValueError, match="does not exist"):
         _ = _env_create("test2", init_file=e1_manifest)
 
 
@@ -1186,14 +1190,14 @@ def test_env_with_config(environment_from_manifest):
 
 
 def test_with_config_bad_include_create(environment_from_manifest):
-    """Confirm missing include paths raise expected exception and error."""
-    with pytest.raises(spack.config.ConfigFileError, match="2 missing include path"):
+    """Confirm missing required include raises expected exception."""
+    err = "does not exist"
+    with pytest.raises(ValueError, match=err):
         environment_from_manifest(
             """
 spack:
   include:
   - /no/such/directory
-  - no/such/file.yaml
 """
         )
 
@@ -1203,34 +1207,25 @@ def test_with_config_bad_include_activate(environment_from_manifest, tmpdir):
     include1 = env_root / "include1.yaml"
     include1.touch()
 
-    abs_include_path = os.path.abspath(tmpdir.join("subdir").ensure("include2.yaml"))
-
     spack_yaml = env_root / ev.manifest_name
     spack_yaml.write_text(
-        f"""
+        """
 spack:
   include:
   - ./include1.yaml
-  - {abs_include_path}
 """
     )
 
     with ev.Environment(env_root) as e:
         e.concretize()
 
-    # we've created an environment with some included config files (which do
-    # in fact exist): now we remove them and check that we get a sensible
-    # error message
+    # We've created an environment with included config file (which does
+    # exist). Now we remove it and check that we get a sensible error.
 
-    os.remove(abs_include_path)
     os.remove(include1)
-    with pytest.raises(spack.config.ConfigFileError) as exc:
+    with pytest.raises(ValueError, match="does not exist"):
         ev.activate(ev.Environment(env_root))
 
-    err = exc.value.message
-    assert "missing include" in err
-    assert abs_include_path in err
-    assert "include1.yaml" in err
     assert ev.active_environment() is None
 
 
@@ -1338,8 +1333,10 @@ def test_config_change_existing(mutable_mock_env_path, tmp_path, mock_packages,
     included file scope.
     """
 
+    env_path = tmp_path / "test_config"
+    fs.mkdirp(env_path)
     included_file = "included-packages.yaml"
-    included_path = tmp_path / included_file
+    included_path = env_path / included_file
     with open(included_path, "w", encoding="utf-8") as f:
         f.write(
             """\
@@ -1355,7 +1352,7 @@ def test_config_change_existing(mutable_mock_env_path, tmp_path, mock_packages,
 """
         )
 
-    spack_yaml = tmp_path / ev.manifest_name
+    spack_yaml = env_path / ev.manifest_name
     spack_yaml.write_text(
         f"""\
 spack:
@@ -1369,7 +1366,8 @@ def test_config_change_existing(mutable_mock_env_path, tmp_path, mock_packages,
 """
     )
 
-    e = ev.Environment(tmp_path)
+    mutable_config.set("config:misc_cache", str(tmp_path / "cache"))
+    e = ev.Environment(env_path)
     with e:
         # List of requirements, flip a variant
         config("change", "packages:mpich:require:~debug")
@@ -1459,19 +1457,6 @@ def test_env_with_included_config_file_url(tmpdir, mutable_empty_config, package
     assert cfg["mpileaks"]["version"] == ["2.2"]
 
 
-def test_env_with_included_config_missing_file(tmpdir, mutable_empty_config):
-    """Test inclusion of a missing configuration file raises FetchError
-    noting missing file."""
-
-    spack_yaml = tmpdir.join("spack.yaml")
-    missing_file = tmpdir.join("packages.yaml")
-    with spack_yaml.open("w") as f:
-        f.write("spack:\n  include:\n    - {0}\n".format(missing_file.strpath))
-
-    with pytest.raises(spack.error.ConfigError, match="missing include path"):
-        ev.Environment(tmpdir.strpath)
-
-
 def test_env_with_included_config_scope(mutable_mock_env_path, packages_file):
     """Test inclusion of a package file from the environment's configuration
     stage directory. This test is intended to represent a case where a remote
@@ -1566,7 +1551,7 @@ def test_env_with_included_config_precedence(tmp_path):
 
 
 def test_env_with_included_configs_precedence(tmp_path):
-    """Test precendence of multiple included configuration files."""
+    """Test precedence of multiple included configuration files."""
     file1 = "high-config.yaml"
     file2 = "low-config.yaml"
 
@@ -1794,7 +1779,7 @@ def test_roots_display_with_variants():
     with ev.read("test"):
         out = find(output=str)
 
-    assert "boost +shared" in out
+    assert "boost+shared" in out
 
 
 def test_uninstall_keeps_in_env(mock_stage, mock_fetch, install_mockery):
@@ -4277,21 +4262,31 @@ def test_unify_when_possible_works_around_conflicts():
     assert len([x for x in e.all_specs() if x.satisfies("mpich")]) == 1
 
 
+# Using mock_include_cache to ensure the "remote" file is cached in a temporary
+# location and not polluting the user cache.
 def test_env_include_packages_url(
-    tmpdir, mutable_empty_config, mock_spider_configs, mock_curl_configs
+    tmpdir, mutable_empty_config, mock_fetch_url_text, mock_curl_configs, mock_include_cache
 ):
     """Test inclusion of a (GitHub) URL."""
     develop_url = "https://github.com/fake/fake/blob/develop/"
     default_packages = develop_url + "etc/fake/defaults/packages.yaml"
+    sha256 = "a422e35b3a18869d0611a4137b37314131749ecdc070a7cd7183f488da81201a"
     spack_yaml = tmpdir.join("spack.yaml")
     with spack_yaml.open("w") as f:
-        f.write("spack:\n  include:\n    - {0}\n".format(default_packages))
-    assert os.path.isfile(spack_yaml.strpath)
+        f.write(
+            f"""\
+spack:
+  include:
+  - path: {default_packages}
+    sha256: {sha256}
+"""
+        )
 
     with spack.config.override("config:url_fetch_method", "curl"):
         env = ev.Environment(tmpdir.strpath)
         ev.activate(env)
 
+        # Make sure a setting from test/data/config/packages.yaml is present
         cfg = spack.config.get("packages")
         assert "mpich" in cfg["all"]["providers"]["mpi"]
 
@@ -4360,7 +4355,7 @@ def test_env_view_disabled(tmp_path, mutable_mock_env_path):
 
 
 @pytest.mark.parametrize("first", ["false", "true", "custom"])
-def test_env_include_mixed_views(tmp_path, mutable_mock_env_path, mutable_config, first):
+def test_env_include_mixed_views(tmp_path, mutable_config, mutable_mock_env_path, first):
     """Ensure including path and boolean views in different combinations result
     in the creation of only the first view if it is not disabled."""
     false_yaml = tmp_path / "false-view.yaml"

lib/spack/spack/test/cmd/find.py

@@ -320,7 +320,7 @@ def test_find_very_long(database, config):
 @pytest.mark.db
 def test_find_show_compiler(database, config):
     output = find("--no-groups", "--show-full-compiler", "mpileaks")
-    assert "mpileaks@2.3%gcc@10.2.1" in output
+    assert "mpileaks@2.3 %gcc@10.2.1" in output
 
 
 @pytest.mark.db
@@ -464,7 +464,7 @@ def test_environment_with_version_range_in_compiler_doesnt_fail(tmp_path):
 
     with test_environment:
         output = find()
-    assert "zlib%gcc@12.1.0" in output
+    assert "zlib %gcc@12.1.0" in output
 
 
 _pkga = (

lib/spack/spack/test/cmd/install.py

@@ -718,10 +718,11 @@ def test_install_deps_then_package(tmpdir, mock_fetch, install_mockery):
     assert os.path.exists(root.prefix)
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701")
 @pytest.mark.regression("12002")
 def test_install_only_dependencies_in_env(
-    tmpdir, mock_fetch, install_mockery, mutable_mock_env_path
+    tmpdir, mutable_mock_env_path, mock_fetch, install_mockery
 ):
     env("create", "test")
 
@@ -735,9 +736,10 @@ def test_install_only_dependencies_in_env(
         assert not os.path.exists(root.prefix)
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.regression("12002")
 def test_install_only_dependencies_of_all_in_env(
-    tmpdir, mock_fetch, install_mockery, mutable_mock_env_path
+    tmpdir, mutable_mock_env_path, mock_fetch, install_mockery
 ):
     env("create", "--without-view", "test")
 
@@ -757,7 +759,8 @@ def test_install_only_dependencies_of_all_in_env(
                 assert os.path.exists(dep.prefix)
 
 
-def test_install_no_add_in_env(tmpdir, mock_fetch, install_mockery, mutable_mock_env_path):
+# Unit tests should not be affected by the user's managed environments
+def test_install_no_add_in_env(tmpdir, mutable_mock_env_path, mock_fetch, install_mockery):
     # To test behavior of --add option, we create the following environment:
     #
     #     mpileaks
@@ -898,7 +901,6 @@ def test_cdash_configure_warning(tmpdir, mock_fetch, install_mockery, capfd):
         specfile = "./spec.json"
         with open(specfile, "w", encoding="utf-8") as f:
             f.write(spec.to_json())
-        print(spec.to_json())
         install("--log-file=cdash_reports", "--log-format=cdash", specfile)
         # Verify Configure.xml exists with expected contents.
         report_dir = tmpdir.join("cdash_reports")
@@ -933,9 +935,10 @@ def test_install_fails_no_args_suggests_env_activation(tmpdir):
     assert "using the `spack.yaml` in this directory" in output
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701")
 def test_install_env_with_tests_all(
-    tmpdir, mock_packages, mock_fetch, install_mockery, mutable_mock_env_path
+    tmpdir, mutable_mock_env_path, mock_packages, mock_fetch, install_mockery
 ):
     env("create", "test")
     with ev.read("test"):
@@ -945,9 +948,10 @@ def test_install_env_with_tests_all(
         assert os.path.exists(test_dep.prefix)
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701")
 def test_install_env_with_tests_root(
-    tmpdir, mock_packages, mock_fetch, install_mockery, mutable_mock_env_path
+    tmpdir, mutable_mock_env_path, mock_packages, mock_fetch, install_mockery
 ):
     env("create", "test")
     with ev.read("test"):
@@ -957,9 +961,10 @@ def test_install_env_with_tests_root(
         assert not os.path.exists(test_dep.prefix)
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701")
 def test_install_empty_env(
-    tmpdir, mock_packages, mock_fetch, install_mockery, mutable_mock_env_path
+    tmpdir, mutable_mock_env_path, mock_packages, mock_fetch, install_mockery
 ):
     env_name = "empty"
     env("create", env_name)
@@ -995,9 +1000,17 @@ def test_installation_fail_tests(install_mockery, mock_fetch, name, method):
     assert "See test log for details" in output
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.not_on_windows("Buildcache not supported on windows")
 def test_install_use_buildcache(
-    capsys, mock_packages, mock_fetch, mock_archive, mock_binary_index, tmpdir, install_mockery
+    capsys,
+    mutable_mock_env_path,
+    mock_packages,
+    mock_fetch,
+    mock_archive,
+    mock_binary_index,
+    tmpdir,
+    install_mockery,
 ):
     """
     Make sure installing with use-buildcache behaves correctly.

lib/spack/spack/test/cmd/mark.py

@@ -12,6 +12,9 @@
 install = SpackCommand("install")
 uninstall = SpackCommand("uninstall")
 
+# Unit tests should not be affected by the user's managed environments
+pytestmark = pytest.mark.usefixtures("mutable_mock_env_path")
+
 
 @pytest.mark.db
 def test_mark_mode_required(mutable_database):

lib/spack/spack/test/cmd/mirror.py

@@ -38,8 +38,9 @@ def test_regression_8083(tmpdir, capfd, mock_packages, mock_fetch, config):
     assert "as it is an external spec" in output
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.regression("12345")
-def test_mirror_from_env(tmp_path, mock_packages, mock_fetch, mutable_mock_env_path):
+def test_mirror_from_env(mutable_mock_env_path, tmp_path, mock_packages, mock_fetch):
     mirror_dir = str(tmp_path / "mirror")
     env_name = "test"
 
@@ -342,8 +343,16 @@ def test_mirror_name_collision(mutable_config):
         mirror("add", "first", "1")
 
 
+# Unit tests should not be affected by the user's managed environments
 def test_mirror_destroy(
-    install_mockery, mock_packages, mock_fetch, mock_archive, mutable_config, monkeypatch, tmpdir
+    mutable_mock_env_path,
+    install_mockery,
+    mock_packages,
+    mock_fetch,
+    mock_archive,
+    mutable_config,
+    monkeypatch,
+    tmpdir,
 ):
     # Create a temp mirror directory for buildcache usage
     mirror_dir = tmpdir.join("mirror_dir")

lib/spack/spack/test/cmd/repo.py

@@ -5,9 +5,13 @@
 
 import pytest
 
+import spack.config
+import spack.environment as ev
 import spack.main
+from spack.main import SpackCommand
 
 repo = spack.main.SpackCommand("repo")
+env = SpackCommand("env")
 
 
 def test_help_option():
@@ -33,3 +37,33 @@ def test_create_add_list_remove(mutable_config, tmpdir):
     repo("remove", "--scope=site", str(tmpdir))
     output = repo("list", "--scope=site", output=str)
     assert "mockrepo" not in output
+
+
+def test_env_repo_path_vars_substitution(
+    tmpdir, install_mockery, mutable_mock_env_path, monkeypatch
+):
+    """Test Spack correctly substitues repo paths with environment variables when creating an
+    environment from a manifest file."""
+
+    monkeypatch.setenv("CUSTOM_REPO_PATH", ".")
+
+    # setup environment from spack.yaml
+    envdir = tmpdir.mkdir("env")
+    with envdir.as_cwd():
+        with open("spack.yaml", "w", encoding="utf-8") as f:
+            f.write(
+                """\
+spack:
+  specs: []
+
+  repos:
+    - $CUSTOM_REPO_PATH
+"""
+            )
+        # creating env from manifest file
+        env("create", "test", "./spack.yaml")
+        # check that repo path was correctly substituted with the environment variable
+        current_dir = os.getcwd()
+        with ev.read("test") as newenv:
+            repos_specs = spack.config.get("repos", default={}, scope=newenv.scope_name)
+            assert current_dir in repos_specs

lib/spack/spack/test/cmd/spec.py

@@ -13,7 +13,10 @@
 import spack.store
 from spack.main import SpackCommand, SpackCommandError
 
-pytestmark = pytest.mark.usefixtures("mutable_config", "mutable_mock_repo")
+# Unit tests should not be affected by the user's managed environments
+pytestmark = pytest.mark.usefixtures(
+    "mutable_mock_env_path", "mutable_config", "mutable_mock_repo"
+)
 
 spec = SpackCommand("spec")
 

lib/spack/spack/test/cmd/style.py

@@ -409,3 +409,108 @@ def test_case_sensitive_imports(tmp_path: pathlib.Path):
 def test_pkg_imports():
     assert spack.cmd.style._module_part(spack.paths.prefix, "spack.pkg.builtin.boost") is None
     assert spack.cmd.style._module_part(spack.paths.prefix, "spack.pkg") is None
+
+
+def test_spec_strings(tmp_path):
+    (tmp_path / "example.py").write_text(
+        """\
+def func(x):
+    print("dont fix %s me" % x, 3)
+    return x.satisfies("+foo %gcc +bar") and x.satisfies("%gcc +baz")
+"""
+    )
+    (tmp_path / "example.json").write_text(
+        """\
+{
+    "spec": [
+        "+foo %gcc +bar~nope   ^dep %clang +yup @3.2 target=x86_64 /abcdef ^another   %gcc   ",
+        "%gcc +baz"
+    ],
+    "%gcc x=y": 2
+}
+"""
+    )
+    (tmp_path / "example.yaml").write_text(
+        """\
+spec:
+  - "+foo   %gcc +bar"
+  - "%gcc +baz"
+  - "this is fine %clang"
+"%gcc x=y": 2
+"""
+    )
+
+    issues = set()
+
+    def collect_issues(path: str, line: int, col: int, old: str, new: str):
+        issues.add((path, line, col, old, new))
+
+    # check for issues with custom handler
+    spack.cmd.style._check_spec_strings(
+        [
+            str(tmp_path / "nonexistent.py"),
+            str(tmp_path / "example.py"),
+            str(tmp_path / "example.json"),
+            str(tmp_path / "example.yaml"),
+        ],
+        handler=collect_issues,
+    )
+
+    assert issues == {
+        (
+            str(tmp_path / "example.json"),
+            3,
+            9,
+            "+foo %gcc +bar~nope   ^dep %clang +yup @3.2 target=x86_64 /abcdef ^another   %gcc   ",
+            "+foo +bar~nope %gcc   ^dep +yup @3.2 target=x86_64 /abcdef %clang ^another   %gcc   ",
+        ),
+        (str(tmp_path / "example.json"), 4, 9, "%gcc +baz", "+baz %gcc"),
+        (str(tmp_path / "example.json"), 6, 5, "%gcc x=y", "x=y %gcc"),
+        (str(tmp_path / "example.py"), 3, 23, "+foo %gcc +bar", "+foo +bar %gcc"),
+        (str(tmp_path / "example.py"), 3, 57, "%gcc +baz", "+baz %gcc"),
+        (str(tmp_path / "example.yaml"), 2, 5, "+foo   %gcc +bar", "+foo +bar   %gcc"),
+        (str(tmp_path / "example.yaml"), 3, 5, "%gcc +baz", "+baz %gcc"),
+        (str(tmp_path / "example.yaml"), 5, 1, "%gcc x=y", "x=y %gcc"),
+    }
+
+    # fix the issues in the files
+    spack.cmd.style._check_spec_strings(
+        [
+            str(tmp_path / "nonexistent.py"),
+            str(tmp_path / "example.py"),
+            str(tmp_path / "example.json"),
+            str(tmp_path / "example.yaml"),
+        ],
+        handler=spack.cmd.style._spec_str_fix_handler,
+    )
+
+    assert (
+        (tmp_path / "example.json").read_text()
+        == """\
+{
+    "spec": [
+        "+foo +bar~nope %gcc   ^dep +yup @3.2 target=x86_64 /abcdef %clang ^another   %gcc   ",
+        "+baz %gcc"
+    ],
+    "x=y %gcc": 2
+}
+"""
+    )
+    assert (
+        (tmp_path / "example.py").read_text()
+        == """\
+def func(x):
+    print("dont fix %s me" % x, 3)
+    return x.satisfies("+foo +bar %gcc") and x.satisfies("+baz %gcc")
+"""
+    )
+    assert (
+        (tmp_path / "example.yaml").read_text()
+        == """\
+spec:
+  - "+foo +bar   %gcc"
+  - "+baz %gcc"
+  - "this is fine %clang"
+"x=y %gcc": 2
+"""
+    )

lib/spack/spack/test/cmd/uninstall.py

@@ -16,6 +16,9 @@
 uninstall = SpackCommand("uninstall")
 install = SpackCommand("install")
 
+# Unit tests should not be affected by the user's managed environments
+pytestmark = pytest.mark.usefixtures("mutable_mock_env_path")
+
 
 class MockArgs:
     def __init__(self, packages, all=False, force=False, dependents=False):
@@ -220,9 +223,7 @@ class TestUninstallFromEnv:
     find = SpackCommand("find")
 
     @pytest.fixture(scope="function")
-    def environment_setup(
-        self, mutable_mock_env_path, mock_packages, mutable_database, install_mockery
-    ):
+    def environment_setup(self, mock_packages, mutable_database, install_mockery):
         TestUninstallFromEnv.env("create", "e1")
         e1 = spack.environment.read("e1")
         with e1:

lib/spack/spack/test/cmd/unit_test.py

@@ -50,7 +50,7 @@ def test_list_long(capsys):
 def test_list_long_with_pytest_arg(capsys):
     with capsys.disabled():
         output = spack_test("--list-long", cmd_test_py)
-    print(output)
+
     assert "unit_test.py::\n" in output
     assert "test_list" in output
     assert "test_list_with_pytest_arg" in output

lib/spack/spack/test/cmd/verify.py

@@ -4,19 +4,31 @@
 
 """Tests for the `spack verify` command"""
 import os
+import platform
+
+import pytest
 
 import llnl.util.filesystem as fs
 
+import spack.cmd.verify
 import spack.concretize
+import spack.installer
 import spack.store
+import spack.util.executable
 import spack.util.spack_json as sjson
 import spack.verify
-from spack.main import SpackCommand
+from spack.main import SpackCommand, SpackCommandError
 
 verify = SpackCommand("verify")
 install = SpackCommand("install")
 
 
+def skip_unless_linux(f):
+    return pytest.mark.skipif(
+        str(platform.system()) != "Linux", reason="only tested on linux for now"
+    )(f)
+
+
 def test_single_file_verify_cmd(tmpdir):
     # Test the verify command interface to verifying a single file.
     filedir = os.path.join(str(tmpdir), "a", "b", "c", "d")
@@ -36,15 +48,14 @@ def test_single_file_verify_cmd(tmpdir):
     with open(manifest_file, "w", encoding="utf-8") as f:
         sjson.dump({filepath: data}, f)
 
-    results = verify("-f", filepath, fail_on_error=False)
-    print(results)
+    results = verify("manifest", "-f", filepath, fail_on_error=False)
     assert not results
 
     os.utime(filepath, (0, 0))
     with open(filepath, "w", encoding="utf-8") as f:
         f.write("I changed.")
 
-    results = verify("-f", filepath, fail_on_error=False)
+    results = verify("manifest", "-f", filepath, fail_on_error=False)
 
     expected = ["hash"]
     mtime = os.stat(filepath).st_mtime
@@ -55,7 +66,7 @@ def test_single_file_verify_cmd(tmpdir):
     assert filepath in results
     assert all(x in results for x in expected)
 
-    results = verify("-fj", filepath, fail_on_error=False)
+    results = verify("manifest", "-fj", filepath, fail_on_error=False)
     res = sjson.load(results)
     assert len(res) == 1
     errors = res.pop(filepath)
@@ -69,18 +80,68 @@ def test_single_spec_verify_cmd(tmpdir, mock_packages, mock_archive, mock_fetch,
     prefix = s.prefix
     hash = s.dag_hash()
 
-    results = verify("/%s" % hash, fail_on_error=False)
+    results = verify("manifest", "/%s" % hash, fail_on_error=False)
     assert not results
 
     new_file = os.path.join(prefix, "new_file_for_verify_test")
     with open(new_file, "w", encoding="utf-8") as f:
         f.write("New file")
 
-    results = verify("/%s" % hash, fail_on_error=False)
+    results = verify("manifest", "/%s" % hash, fail_on_error=False)
     assert new_file in results
     assert "added" in results
 
-    results = verify("-j", "/%s" % hash, fail_on_error=False)
+    results = verify("manifest", "-j", "/%s" % hash, fail_on_error=False)
     res = sjson.load(results)
     assert len(res) == 1
     assert res[new_file] == ["added"]
+
+
+@pytest.mark.requires_executables("gcc")
+@skip_unless_linux
+def test_libraries(tmp_path, install_mockery, mock_fetch):
+    gcc = spack.util.executable.which("gcc", required=True)
+    s = spack.concretize.concretize_one("libelf")
+    spack.installer.PackageInstaller([s.package]).install()
+    os.mkdir(s.prefix.bin)
+
+    # There are no ELF files so the verification should pass
+    verify("libraries", f"/{s.dag_hash()}")
+
+    # Now put main_with_rpath linking to libf.so inside the prefix and verify again. This should
+    # work because libf.so can be located in the rpath.
+    (tmp_path / "f.c").write_text("void f(void){return;}")
+    (tmp_path / "main.c").write_text("void f(void); int main(void){f();return 0;}")
+
+    gcc("-shared", "-fPIC", "-o", str(tmp_path / "libf.so"), str(tmp_path / "f.c"))
+    gcc(
+        "-o",
+        str(s.prefix.bin.main_with_rpath),
+        str(tmp_path / "main.c"),
+        "-L",
+        str(tmp_path),
+        f"-Wl,-rpath,{tmp_path}",
+        "-lf",
+    )
+    verify("libraries", f"/{s.dag_hash()}")
+
+    # Now put main_without_rpath linking to libf.so inside the prefix and verify again. This should
+    # fail because libf.so cannot be located in the rpath.
+    gcc(
+        "-o",
+        str(s.prefix.bin.main_without_rpath),
+        str(tmp_path / "main.c"),
+        "-L",
+        str(tmp_path),
+        "-lf",
+    )
+
+    with pytest.raises(SpackCommandError):
+        verify("libraries", f"/{s.dag_hash()}")
+
+    # Check the error message
+    msg = spack.cmd.verify._verify_libraries(s, [])
+    assert msg is not None and "libf.so => not found" in msg
+
+    # And check that we can make it pass by ignoring it.
+    assert spack.cmd.verify._verify_libraries(s, ["libf.so"]) is None

lib/spack/spack/test/cmd/versions.py

@@ -59,13 +59,6 @@ def mock_fetch_remote_versions(*args, **kwargs):
     assert v.strip(" \n\t") == "99.99.99\n  3.2.1"
 
 
-@pytest.mark.maybeslow
-def test_no_versions():
-    """Test a package for which no remote versions are available."""
-
-    versions("converge")
-
-
 @pytest.mark.maybeslow
 def test_no_unchecksummed_versions():
     """Test a package for which no unchecksummed versions are available."""

lib/spack/spack/test/concretization/compiler_runtimes.py

@@ -92,7 +92,7 @@ def test_external_nodes_do_not_have_runtimes(runtime_repo, mutable_config, tmp_p
         # Same as before, but tests that we can reuse from a more generic target
         pytest.param(
             "pkg-a%gcc@9.4.0",
-            "pkg-b%gcc@10.2.1 target=x86_64",
+            "pkg-b target=x86_64 %gcc@10.2.1",
             {"pkg-a": "gcc-runtime@9.4.0", "pkg-b": "gcc-runtime@9.4.0"},
             1,
             marks=pytest.mark.skipif(
@@ -101,7 +101,7 @@ def test_external_nodes_do_not_have_runtimes(runtime_repo, mutable_config, tmp_p
         ),
         pytest.param(
             "pkg-a%gcc@10.2.1",
-            "pkg-b%gcc@9.4.0 target=x86_64",
+            "pkg-b target=x86_64 %gcc@9.4.0",
             {
                 "pkg-a": "gcc-runtime@10.2.1 target=x86_64",
                 "pkg-b": "gcc-runtime@9.4.0 target=x86_64",

lib/spack/spack/test/concretization/core.py

@@ -121,7 +121,7 @@ def binary_compatibility(monkeypatch, request):
         "mpileaks ^mpi@1.2:2",
         # conflict not triggered
         "conflict",
-        "conflict%clang~foo",
+        "conflict~foo%clang",
         "conflict-parent%gcc",
     ]
 )
@@ -387,8 +387,8 @@ def test_different_compilers_get_different_flags(
         t = archspec.cpu.host().family
         client = spack.concretize.concretize_one(
             Spec(
-                f"cmake-client %gcc@11.1.0 platform=test os=redhat6 target={t}"
-                f" ^cmake %clang@12.2.0 platform=test os=redhat6 target={t}"
+                f"cmake-client platform=test os=redhat6 target={t} %gcc@11.1.0"
+                f" ^cmake platform=test os=redhat6 target={t} %clang@12.2.0"
             )
         )
         cmake = client["cmake"]
@@ -403,7 +403,7 @@ def test_spec_flags_maintain_order(self, mutable_config, gcc11_with_flags):
         for successive concretizations.
         """
         mutable_config.set("compilers", [gcc11_with_flags])
-        spec_str = "libelf %gcc@11.1.0 os=redhat6"
+        spec_str = "libelf os=redhat6 %gcc@11.1.0"
         for _ in range(3):
             s = spack.concretize.concretize_one(spec_str)
             assert all(
@@ -414,7 +414,7 @@ def test_compiler_flags_differ_identical_compilers(self, mutable_config, clang12
         mutable_config.set("compilers", [clang12_with_flags])
         # Correct arch to use test compiler that has flags
         t = archspec.cpu.host().family
-        spec = Spec(f"pkg-a %clang@12.2.0 platform=test os=redhat6 target={t}")
+        spec = Spec(f"pkg-a platform=test os=redhat6 target={t} %clang@12.2.0")
 
         # Get the compiler that matches the spec (
         compiler = spack.compilers.compiler_for_spec("clang@=12.2.0", spec.architecture)
@@ -488,13 +488,13 @@ def test_architecture_deep_inheritance(self, mock_targets, compiler_factory):
         # CNL compiler has no target attribute, and this is essential to make detection pass
         del cnl_compiler["compiler"]["target"]
         with spack.config.override("compilers", [cnl_compiler]):
-            spec_str = "mpileaks %gcc@4.5.0 os=CNL target=nocona ^dyninst os=CNL ^callpath os=CNL"
+            spec_str = "mpileaks os=CNL target=nocona %gcc@4.5.0 ^dyninst os=CNL ^callpath os=CNL"
             spec = spack.concretize.concretize_one(spec_str)
             for s in spec.traverse(root=False):
                 assert s.architecture.target == spec.architecture.target
 
     def test_compiler_flags_from_user_are_grouped(self):
-        spec = Spec('pkg-a%gcc cflags="-O -foo-flag foo-val" platform=test')
+        spec = Spec('pkg-a cflags="-O -foo-flag foo-val" platform=test %gcc')
         spec = spack.concretize.concretize_one(spec)
         cflags = spec.compiler_flags["cflags"]
         assert any(x == "-foo-flag foo-val" for x in cflags)
@@ -804,7 +804,7 @@ def test_external_and_virtual(self, mutable_config):
         assert spec["stuff"].compiler.satisfies("gcc")
 
     def test_compiler_child(self):
-        s = Spec("mpileaks%clang target=x86_64 ^dyninst%gcc")
+        s = Spec("mpileaks target=x86_64 %clang ^dyninst%gcc")
         s = spack.concretize.concretize_one(s)
         assert s["mpileaks"].satisfies("%clang")
         assert s["dyninst"].satisfies("%gcc")
@@ -826,7 +826,7 @@ def test_conflict_in_all_directives_true(self):
         with pytest.raises(spack.error.SpackError):
             s = spack.concretize.concretize_one(s)
 
-    @pytest.mark.parametrize("spec_str", ["conflict@10.0%clang+foo"])
+    @pytest.mark.parametrize("spec_str", ["conflict@10.0+foo%clang"])
     def test_no_conflict_in_external_specs(self, spec_str):
         # Modify the configuration to have the spec with conflict
         # registered as an external
@@ -940,9 +940,9 @@ def test_noversion_pkg(self, spec):
         "spec, best_achievable",
         [
             ("mpileaks%gcc@=4.4.7 ^dyninst@=10.2.1 target=x86_64:", "core2"),
-            ("mpileaks%gcc@=4.8 target=x86_64:", "haswell"),
-            ("mpileaks%gcc@=5.3.0 target=x86_64:", "broadwell"),
-            ("mpileaks%apple-clang@=5.1.0 target=x86_64:", "x86_64"),
+            ("mpileaks target=x86_64: %gcc@=4.8", "haswell"),
+            ("mpileaks target=x86_64: %gcc@=5.3.0", "broadwell"),
+            ("mpileaks target=x86_64: %apple-clang@=5.1.0", "x86_64"),
         ],
     )
     @pytest.mark.regression("13361", "20537")
@@ -1286,7 +1286,7 @@ def test_compiler_match_is_preferred_to_newer_version(self, compiler_factory):
         with spack.config.override(
             "compilers", [compiler_factory(spec="gcc@10.1.0", operating_system="redhat6")]
         ):
-            spec_str = "simple-inheritance+openblas %gcc@10.1.0 os=redhat6"
+            spec_str = "simple-inheritance+openblas os=redhat6 %gcc@10.1.0"
             s = spack.concretize.concretize_one(spec_str)
             assert "openblas@0.2.15" in s
             assert s["openblas"].satisfies("%gcc@10.1.0")
@@ -1319,7 +1319,7 @@ def test_custom_compiler_version(self, mutable_config, compiler_factory, monkeyp
             "compilers", [compiler_factory(spec="gcc@10foo", operating_system="redhat6")]
         )
         monkeypatch.setattr(spack.compiler.Compiler, "real_version", "10.2.1")
-        s = spack.concretize.concretize_one("pkg-a %gcc@10foo os=redhat6")
+        s = spack.concretize.concretize_one("pkg-a os=redhat6 %gcc@10foo")
         assert "%gcc@10foo" in s
 
     def test_all_patches_applied(self):
@@ -1531,8 +1531,8 @@ def test_external_with_non_default_variant_as_dependency(self):
             ("mpileaks", "os=debian6"),
             # To trigger the bug in 22871 we need to have the same compiler
             # spec available on both operating systems
-            ("mpileaks%gcc@10.2.1 platform=test os=debian6", "os=debian6"),
-            ("mpileaks%gcc@10.2.1 platform=test os=redhat6", "os=redhat6"),
+            ("mpileaks platform=test os=debian6 %gcc@10.2.1", "os=debian6"),
+            ("mpileaks platform=test os=redhat6 %gcc@10.2.1", "os=redhat6"),
         ],
     )
     def test_os_selection_when_multiple_choices_are_possible(
@@ -2018,7 +2018,6 @@ def test_git_ref_version_is_equivalent_to_specified_version(self, git_ref):
         s = Spec("develop-branch-version@git.%s=develop" % git_ref)
         c = spack.concretize.concretize_one(s)
         assert git_ref in str(c)
-        print(str(c))
         assert s.satisfies("@develop")
         assert s.satisfies("@0.1:")
 
@@ -2888,6 +2887,23 @@ def test_specifying_different_versions_build_deps(self):
         assert any(x.satisfies(hdf5_str) for x in result.specs)
         assert any(x.satisfies(pinned_str) for x in result.specs)
 
+    @pytest.mark.regression("44289")
+    def test_all_extensions_depend_on_same_extendee(self):
+        """Tests that we don't reuse dependencies that bring in a different extendee"""
+        setuptools = spack.concretize.concretize_one("py-setuptools ^python@3.10")
+
+        solver = spack.solver.asp.Solver()
+        setup = spack.solver.asp.SpackSolverSetup()
+        result, _, _ = solver.driver.solve(
+            setup, [Spec("py-floating ^python@3.11")], reuse=list(setuptools.traverse())
+        )
+        assert len(result.specs) == 1
+
+        floating = result.specs[0]
+        assert all(setuptools.dag_hash() != x.dag_hash() for x in floating.traverse())
+        pythons = [x for x in floating.traverse() if x.name == "python"]
+        assert len(pythons) == 1 and pythons[0].satisfies("@3.11")
+
 
 @pytest.mark.parametrize(
     "v_str,v_opts,checksummed",
@@ -2997,7 +3013,7 @@ def test_virtuals_provided_together_but_only_one_required_in_dag(self):
 
 
 def test_reusable_externals_match(mock_packages, tmpdir):
-    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0")
     spec.external_path = tmpdir.strpath
     spec.external_modules = ["mpich/4.1"]
     spec._mark_concrete()
@@ -3015,7 +3031,7 @@ def test_reusable_externals_match(mock_packages, tmpdir):
 
 
 def test_reusable_externals_match_virtual(mock_packages, tmpdir):
-    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0")
     spec.external_path = tmpdir.strpath
     spec.external_modules = ["mpich/4.1"]
     spec._mark_concrete()
@@ -3033,7 +3049,7 @@ def test_reusable_externals_match_virtual(mock_packages, tmpdir):
 
 
 def test_reusable_externals_different_prefix(mock_packages, tmpdir):
-    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0")
     spec.external_path = "/other/path"
     spec.external_modules = ["mpich/4.1"]
     spec._mark_concrete()
@@ -3052,7 +3068,7 @@ def test_reusable_externals_different_prefix(mock_packages, tmpdir):
 
 @pytest.mark.parametrize("modules", [None, ["mpich/4.1", "libfabric/1.19"]])
 def test_reusable_externals_different_modules(mock_packages, tmpdir, modules):
-    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0")
     spec.external_path = tmpdir.strpath
     spec.external_modules = modules
     spec._mark_concrete()
@@ -3070,7 +3086,7 @@ def test_reusable_externals_different_modules(mock_packages, tmpdir, modules):
 
 
 def test_reusable_externals_different_spec(mock_packages, tmpdir):
-    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0")
     spec.external_path = tmpdir.strpath
     spec._mark_concrete()
     assert not spack.solver.asp._is_reusable(

lib/spack/spack/test/concretization/flag_mixing.py

@@ -94,7 +94,7 @@ def test_mix_spec_and_compiler_cfg(concretize_scope, test_repo):
     conf_str = _compiler_cfg_one_entry_with_cflags("-Wall")
     update_concretize_scope(conf_str, "compilers")
 
-    s1 = spack.concretize.concretize_one('y %gcc@12.100.100 cflags="-O2"')
+    s1 = spack.concretize.concretize_one('y cflags="-O2" %gcc@12.100.100')
     assert s1.satisfies('cflags="-Wall -O2"')
 
 
@@ -182,7 +182,7 @@ def test_propagate_and_compiler_cfg(concretize_scope, test_repo):
     conf_str = _compiler_cfg_one_entry_with_cflags("-f2")
     update_concretize_scope(conf_str, "compilers")
 
-    root_spec = spack.concretize.concretize_one("v %gcc@12.100.100 cflags=='-f1'")
+    root_spec = spack.concretize.concretize_one("v cflags=='-f1' %gcc@12.100.100")
     assert root_spec["y"].satisfies("cflags='-f1 -f2'")
 
 
@@ -229,7 +229,7 @@ def test_dev_mix_flags(tmp_path, concretize_scope, mutable_mock_env_path, test_r
     env_content = f"""\
 spack:
   specs:
-  - y %gcc@12.100.100 cflags=='-fsanitize=address'
+  - y cflags=='-fsanitize=address' %gcc@12.100.100
   develop:
     y:
       spec: y cflags=='-fsanitize=address'

lib/spack/spack/test/concretization/requirements.py

@@ -359,10 +359,10 @@ def test_one_package_multiple_oneof_groups(concretize_scope, test_repo):
     update_packages_config(conf_str)
 
     s1 = spack.concretize.concretize_one("y@2.5")
-    assert s1.satisfies("%clang~shared")
+    assert s1.satisfies("~shared%clang")
 
     s2 = spack.concretize.concretize_one("y@2.4")
-    assert s2.satisfies("%gcc+shared")
+    assert s2.satisfies("+shared%gcc")
 
 
 @pytest.mark.regression("34241")
@@ -499,7 +499,7 @@ def test_default_requirements_with_all(spec_str, requirement_str, concretize_sco
     "requirements,expectations",
     [
         (("%gcc", "%clang"), ("%gcc", "%clang")),
-        (("%gcc~shared", "@1.0"), ("%gcc~shared", "@1.0+shared")),
+        (("~shared%gcc", "@1.0"), ("~shared%gcc", "@1.0+shared")),
     ],
 )
 def test_default_and_package_specific_requirements(
@@ -754,7 +754,7 @@ def test_skip_requirement_when_default_requirement_condition_cannot_be_met(
     update_packages_config(packages_yaml)
     s = spack.concretize.concretize_one("mpileaks")
 
-    assert s.satisfies("%clang+shared")
+    assert s.satisfies("+shared %clang")
     # Sanity checks that 'callpath' doesn't have the shared variant, but that didn't
     # cause failures during concretization.
     assert "shared" not in s["callpath"].variants
@@ -1125,3 +1125,46 @@ def test_strong_preferences_higher_priority_than_reuse(concretize_scope, mock_pa
         )
         ascent = result.specs[0]
     assert ascent["adios2"].dag_hash() == reused_spec.dag_hash(), ascent
+
+
+@pytest.mark.parametrize(
+    "packages_yaml,err_match",
+    [
+        (
+            """
+packages:
+  mpi:
+    require:
+    - "+bzip2"
+""",
+            "expected a named spec",
+        ),
+        (
+            """
+packages:
+  mpi:
+    require:
+    - one_of: ["+bzip2", openmpi]
+""",
+            "expected a named spec",
+        ),
+        (
+            """
+packages:
+  mpi:
+    require:
+    - "^mpich"
+""",
+            "Did you mean",
+        ),
+    ],
+)
+def test_anonymous_spec_cannot_be_used_in_virtual_requirements(
+    packages_yaml, err_match, concretize_scope, mock_packages
+):
+    """Tests that using anonymous specs in requirements for virtual packages raises an
+    appropriate error message.
+    """
+    update_packages_config(packages_yaml)
+    with pytest.raises(spack.error.SpackError, match=err_match):
+        spack.concretize.concretize_one("mpileaks")

lib/spack/spack/test/config.py

@@ -11,8 +11,7 @@
 
 import pytest
 
-import llnl.util.tty as tty
-from llnl.util.filesystem import join_path, touch, touchp
+from llnl.util.filesystem import join_path, touch
 
 import spack
 import spack.config
@@ -26,6 +25,7 @@
 import spack.schema.compilers
 import spack.schema.config
 import spack.schema.env
+import spack.schema.include
 import spack.schema.mirrors
 import spack.schema.repos
 import spack.spec
@@ -33,6 +33,8 @@
 import spack.util.path as spack_path
 import spack.util.spack_yaml as syaml
 
+from ..enums import ConfigScopePriority
+
 # sample config data
 config_low = {
     "config": {
@@ -49,22 +51,9 @@
 
 config_override_list = {"config": {"build_stage:": ["pathd", "pathe"]}}
 
-config_merge_dict = {"config": {"info": {"a": 3, "b": 4}}}
+config_merge_dict = {"config": {"aliases": {"ls": "find", "dev": "develop"}}}
 
-config_override_dict = {"config": {"info:": {"a": 7, "c": 9}}}
-
-
-@pytest.fixture()
-def write_config_file(tmpdir):
-    """Returns a function that writes a config file."""
-
-    def _write(config, data, scope):
-        config_yaml = tmpdir.join(scope, config + ".yaml")
-        config_yaml.ensure()
-        with config_yaml.open("w") as f:
-            syaml.dump_config(data, f)
-
-    return _write
+config_override_dict = {"config": {"aliases:": {"be": "build-env", "deps": "dependencies"}}}
 
 
 @pytest.fixture()
@@ -710,10 +699,7 @@ def assert_marked(obj):
 
 
 def test_internal_config_from_data():
-    config = spack.config.Configuration()
-
-    # add an internal config initialized from an inline dict
-    config.push_scope(
+    config = spack.config.create_from(
         spack.config.InternalConfigScope(
             "_builtin", {"config": {"verify_ssl": False, "build_jobs": 6}}
         )
@@ -1038,6 +1024,16 @@ def test_bad_config_yaml(tmpdir):
         )
 
 
+def test_bad_include_yaml(tmpdir):
+    with pytest.raises(spack.config.ConfigFormatError, match="is not of type"):
+        check_schema(
+            spack.schema.include.schema,
+            """\
+include: $HOME/include.yaml
+""",
+        )
+
+
 def test_bad_mirrors_yaml(tmpdir):
     with pytest.raises(spack.config.ConfigFormatError):
         check_schema(
@@ -1102,9 +1098,9 @@ def test_internal_config_section_override(mock_low_high_config, write_config_fil
 
 def test_internal_config_dict_override(mock_low_high_config, write_config_file):
     write_config_file("config", config_merge_dict, "low")
-    wanted_dict = config_override_dict["config"]["info:"]
+    wanted_dict = config_override_dict["config"]["aliases:"]
     mock_low_high_config.push_scope(spack.config.InternalConfigScope("high", config_override_dict))
-    assert mock_low_high_config.get("config:info") == wanted_dict
+    assert mock_low_high_config.get("config:aliases") == wanted_dict
 
 
 def test_internal_config_list_override(mock_low_high_config, write_config_file):
@@ -1136,10 +1132,10 @@ def test_set_list_override(mock_low_high_config, write_config_file):
 
 def test_set_dict_override(mock_low_high_config, write_config_file):
     write_config_file("config", config_merge_dict, "low")
-    wanted_dict = config_override_dict["config"]["info:"]
-    with spack.config.override("config:info:", wanted_dict):
-        assert wanted_dict == mock_low_high_config.get("config:info")
-    assert config_merge_dict["config"]["info"] == mock_low_high_config.get("config:info")
+    wanted_dict = config_override_dict["config"]["aliases:"]
+    with spack.config.override("config:aliases:", wanted_dict):
+        assert wanted_dict == mock_low_high_config.get("config:aliases")
+    assert config_merge_dict["config"]["aliases"] == mock_low_high_config.get("config:aliases")
 
 
 def test_set_bad_path(config):
@@ -1225,7 +1221,7 @@ def test_user_config_path_is_default_when_env_var_is_empty(working_env):
 
 
 def test_default_install_tree(monkeypatch, default_config):
-    s = spack.spec.Spec("nonexistent@x.y.z %none@a.b.c arch=foo-bar-baz")
+    s = spack.spec.Spec("nonexistent@x.y.z arch=foo-bar-baz %none@a.b.c")
     monkeypatch.setattr(s, "dag_hash", lambda length: "abc123")
     _, _, projections = spack.store.parse_install_tree(spack.config.get("config"))
     assert s.format(projections["all"]) == "foo-bar-baz/none-a.b.c/nonexistent-x.y.z-abc123"
@@ -1265,134 +1261,6 @@ def test_user_cache_path_is_default_when_env_var_is_empty(working_env):
     assert os.path.expanduser("~%s.spack" % os.sep) == spack.paths._get_user_cache_path()
 
 
-github_url = "https://github.com/fake/fake/{0}/develop"
-gitlab_url = "https://gitlab.fake.io/user/repo/-/blob/config/defaults"
-
-
-@pytest.mark.parametrize(
-    "url,isfile",
-    [
-        (github_url.format("tree"), False),
-        ("{0}/README.md".format(github_url.format("blob")), True),
-        ("{0}/etc/fake/defaults/packages.yaml".format(github_url.format("blob")), True),
-        (gitlab_url, False),
-        (None, False),
-    ],
-)
-def test_config_collect_urls(mutable_empty_config, mock_spider_configs, url, isfile):
-    with spack.config.override("config:url_fetch_method", "curl"):
-        urls = spack.config.collect_urls(url)
-        if url:
-            if isfile:
-                expected = 1 if url.endswith(".yaml") else 0
-                assert len(urls) == expected
-            else:
-                # Expect multiple configuration files for a "directory"
-                assert len(urls) > 1
-        else:
-            assert not urls
-
-
-@pytest.mark.parametrize(
-    "url,isfile,fail",
-    [
-        (github_url.format("tree"), False, False),
-        (gitlab_url, False, False),
-        ("{0}/README.md".format(github_url.format("blob")), True, True),
-        ("{0}/compilers.yaml".format(gitlab_url), True, False),
-        (None, False, True),
-    ],
-)
-def test_config_fetch_remote_configs(
-    tmpdir, mutable_empty_config, mock_collect_urls, mock_curl_configs, url, isfile, fail
-):
-    def _has_content(filename):
-        # The first element of all configuration files for this test happen to
-        # be the basename of the file so this check leverages that feature. If
-        # that changes, then this check will need to change accordingly.
-        element = "{0}:".format(os.path.splitext(os.path.basename(filename))[0])
-        with open(filename, "r", encoding="utf-8") as fd:
-            for line in fd:
-                if element in line:
-                    return True
-        tty.debug("Expected {0} in '{1}'".format(element, filename))
-        return False
-
-    dest_dir = join_path(tmpdir.strpath, "defaults")
-    if fail:
-        msg = "Cannot retrieve configuration"
-        with spack.config.override("config:url_fetch_method", "curl"):
-            with pytest.raises(spack.config.ConfigFileError, match=msg):
-                spack.config.fetch_remote_configs(url, dest_dir)
-    else:
-        with spack.config.override("config:url_fetch_method", "curl"):
-            path = spack.config.fetch_remote_configs(url, dest_dir)
-            assert os.path.exists(path)
-            if isfile:
-                # Ensure correct file is "fetched"
-                assert os.path.basename(path) == os.path.basename(url)
-                # Ensure contents of the file has expected config element
-                assert _has_content(path)
-            else:
-                for filename in os.listdir(path):
-                    assert _has_content(join_path(path, filename))
-
-
-@pytest.fixture(scope="function")
-def mock_collect_urls(mock_config_data, monkeypatch):
-    """Mock the collection of URLs to avoid mocking spider."""
-
-    _, config_files = mock_config_data
-
-    def _collect(base_url):
-        if not base_url:
-            return []
-
-        ext = os.path.splitext(base_url)[1]
-        if ext:
-            return [base_url] if ext == ".yaml" else []
-
-        return [join_path(base_url, f) for f in config_files]
-
-    monkeypatch.setattr(spack.config, "collect_urls", _collect)
-
-    yield
-
-
-@pytest.mark.parametrize(
-    "url,skip",
-    [(github_url.format("tree"), True), ("{0}/compilers.yaml".format(gitlab_url), True)],
-)
-def test_config_fetch_remote_configs_skip(
-    tmpdir, mutable_empty_config, mock_collect_urls, mock_curl_configs, url, skip
-):
-    """Ensure skip fetching remote config file if it already exists when
-    required and not skipping if replacing it."""
-
-    def check_contents(filename, expected):
-        with open(filename, "r", encoding="utf-8") as fd:
-            lines = fd.readlines()
-            if expected:
-                assert lines[0] == "compilers:"
-            else:
-                assert not lines
-
-    dest_dir = join_path(tmpdir.strpath, "defaults")
-    filename = "compilers.yaml"
-
-    # Create a stage directory with an empty configuration file
-    path = join_path(dest_dir, filename)
-    touchp(path)
-
-    # Do NOT replace the existing cached configuration file if skipping
-    expected = None if skip else "compilers:"
-
-    with spack.config.override("config:url_fetch_method", "curl"):
-        path = spack.config.fetch_remote_configs(url, dest_dir, skip)
-        result_filename = path if path.endswith(".yaml") else join_path(path, filename)
-        check_contents(result_filename, expected)
-
-
 def test_config_file_dir_failure(tmpdir, mutable_empty_config):
     with pytest.raises(spack.config.ConfigFileError, match="not a file"):
         spack.config.read_config_file(tmpdir.strpath)
@@ -1445,7 +1313,7 @@ def test_config_path_dsl(path, it_should_work, expected_parsed):
 
 
 @pytest.mark.regression("48254")
-def test_env_activation_preserves_config_scopes(mutable_mock_env_path):
+def test_env_activation_preserves_command_line_scope(mutable_mock_env_path):
     """Check that the "command_line" scope remains the highest priority scope, when we activate,
     or deactivate, environments.
     """
@@ -1469,3 +1337,51 @@ def test_env_activation_preserves_config_scopes(mutable_mock_env_path):
         assert spack.config.CONFIG.highest() == expected_cl_scope
 
     assert spack.config.CONFIG.highest() == expected_cl_scope
+
+
+@pytest.mark.regression("48414")
+@pytest.mark.regression("49188")
+def test_env_activation_preserves_config_scopes(mutable_mock_env_path):
+    """Check that the priority of scopes is respected when merging configuration files."""
+    custom_scope = spack.config.InternalConfigScope("custom_scope")
+    spack.config.CONFIG.push_scope(custom_scope, priority=ConfigScopePriority.CUSTOM)
+    expected_scopes_without_env = ["custom_scope", "command_line"]
+    expected_scopes_with_first_env = ["custom_scope", "env:test", "command_line"]
+    expected_scopes_with_second_env = ["custom_scope", "env:test-2", "command_line"]
+
+    def highest_priority_scopes(config, *, nscopes):
+        return list(config.scopes)[-nscopes:]
+
+    assert highest_priority_scopes(spack.config.CONFIG, nscopes=2) == expected_scopes_without_env
+    # Creating an environment pushes a new scope
+    ev.create("test")
+    with ev.read("test"):
+        assert (
+            highest_priority_scopes(spack.config.CONFIG, nscopes=3)
+            == expected_scopes_with_first_env
+        )
+
+        # No active environment pops the scope
+        with ev.no_active_environment():
+            assert (
+                highest_priority_scopes(spack.config.CONFIG, nscopes=2)
+                == expected_scopes_without_env
+            )
+        assert (
+            highest_priority_scopes(spack.config.CONFIG, nscopes=3)
+            == expected_scopes_with_first_env
+        )
+
+        # Switch the environment to another one
+        ev.create("test-2")
+        with ev.read("test-2"):
+            assert (
+                highest_priority_scopes(spack.config.CONFIG, nscopes=3)
+                == expected_scopes_with_second_env
+            )
+        assert (
+            highest_priority_scopes(spack.config.CONFIG, nscopes=3)
+            == expected_scopes_with_first_env
+        )
+
+    assert highest_priority_scopes(spack.config.CONFIG, nscopes=2) == expected_scopes_without_env

lib/spack/spack/test/conftest.py

@@ -30,7 +30,15 @@
 import llnl.util.lang
 import llnl.util.lock
 import llnl.util.tty as tty
-from llnl.util.filesystem import copy_tree, mkdirp, remove_linked_tree, touchp, working_dir
+from llnl.util.filesystem import (
+    copy,
+    copy_tree,
+    join_path,
+    mkdirp,
+    remove_linked_tree,
+    touchp,
+    working_dir,
+)
 
 import spack.binary_distribution
 import spack.bootstrap.core
@@ -65,6 +73,9 @@
 from spack.installer import PackageInstaller
 from spack.main import SpackCommand
 from spack.util.pattern import Bunch
+from spack.util.remote_file_cache import raw_github_gitlab_url
+
+from ..enums import ConfigScopePriority
 
 mirror_cmd = SpackCommand("mirror")
 
@@ -723,11 +734,23 @@ def configuration_dir(tmpdir_factory, linux_os):
 def _create_mock_configuration_scopes(configuration_dir):
     """Create the configuration scopes used in `config` and `mutable_config`."""
     return [
-        spack.config.InternalConfigScope("_builtin", spack.config.CONFIG_DEFAULTS),
-        spack.config.DirectoryConfigScope("site", str(configuration_dir.join("site"))),
-        spack.config.DirectoryConfigScope("system", str(configuration_dir.join("system"))),
-        spack.config.DirectoryConfigScope("user", str(configuration_dir.join("user"))),
-        spack.config.InternalConfigScope("command_line"),
+        (
+            ConfigScopePriority.BUILTIN,
+            spack.config.InternalConfigScope("_builtin", spack.config.CONFIG_DEFAULTS),
+        ),
+        (
+            ConfigScopePriority.CONFIG_FILES,
+            spack.config.DirectoryConfigScope("site", str(configuration_dir.join("site"))),
+        ),
+        (
+            ConfigScopePriority.CONFIG_FILES,
+            spack.config.DirectoryConfigScope("system", str(configuration_dir.join("system"))),
+        ),
+        (
+            ConfigScopePriority.CONFIG_FILES,
+            spack.config.DirectoryConfigScope("user", str(configuration_dir.join("user"))),
+        ),
+        (ConfigScopePriority.COMMAND_LINE, spack.config.InternalConfigScope("command_line")),
     ]
 
 
@@ -794,13 +817,11 @@ def mock_wsdk_externals(monkeypatch_session):
 def concretize_scope(mutable_config, tmpdir):
     """Adds a scope for concretization preferences"""
     tmpdir.ensure_dir("concretize")
-    mutable_config.push_scope(
+    with spack.config.override(
         spack.config.DirectoryConfigScope("concretize", str(tmpdir.join("concretize")))
-    )
+    ):
+        yield str(tmpdir.join("concretize"))
 
-    yield str(tmpdir.join("concretize"))
-
-    mutable_config.pop_scope()
     spack.repo.PATH._provider_index = None
 
 
@@ -1669,7 +1690,7 @@ def installation_dir_with_headers(tmpdir_factory):
 ##########
 
 
-@pytest.fixture(params=["conflict%clang+foo", "conflict-parent@0.9^conflict~foo"])
+@pytest.fixture(params=["conflict+foo%clang", "conflict-parent@0.9^conflict~foo"])
 def conflict_spec(request):
     """Specs which violate constraints specified with the "conflicts"
     directive in the "conflict" package.
@@ -1884,35 +1905,21 @@ def __call__(self, *args, **kwargs):
 
 
 @pytest.fixture(scope="function")
-def mock_spider_configs(mock_config_data, monkeypatch):
-    """
-    Mock retrieval of configuration file URLs from the web by grabbing
-    them from the test data configuration directory.
-    """
-    config_data_dir, config_files = mock_config_data
+def mock_fetch_url_text(tmpdir, mock_config_data, monkeypatch):
+    """Mock spack.util.web.fetch_url_text."""
 
-    def _spider(*args, **kwargs):
-        root_urls = args[0]
-        if not root_urls:
-            return [], set()
+    stage_dir, config_files = mock_config_data
 
-        root_urls = [root_urls] if isinstance(root_urls, str) else root_urls
+    def _fetch_text_file(url, dest_dir):
+        raw_url = raw_github_gitlab_url(url)
+        mkdirp(dest_dir)
+        basename = os.path.basename(raw_url)
+        src = join_path(stage_dir, basename)
+        dest = join_path(dest_dir, basename)
+        copy(src, dest)
+        return dest
 
-        # Any URL with an extension will be treated like a file; otherwise,
-        # it is considered a directory/folder and we'll grab all available
-        # files.
-        urls = []
-        for url in root_urls:
-            if os.path.splitext(url)[1]:
-                urls.append(url)
-            else:
-                urls.extend([os.path.join(url, f) for f in config_files])
-
-        return [], set(urls)
-
-    monkeypatch.setattr(spack.util.web, "spider", _spider)
-
-    yield
+    monkeypatch.setattr(spack.util.web, "fetch_url_text", _fetch_text_file)
 
 
 @pytest.fixture(scope="function")
@@ -2126,7 +2133,6 @@ def _c_compiler_always_exists():
 @pytest.fixture(scope="session")
 def mock_test_cache(tmp_path_factory):
     cache_dir = tmp_path_factory.mktemp("cache")
-    print(cache_dir)
     return spack.util.file_cache.FileCache(str(cache_dir))
 
 
@@ -2176,3 +2182,27 @@ def info(self):
 @pytest.fixture()
 def mock_runtimes(config, mock_packages):
     return mock_packages.packages_with_tags("runtime")
+
+
+@pytest.fixture()
+def write_config_file(tmpdir):
+    """Returns a function that writes a config file."""
+
+    def _write(config, data, scope):
+        config_yaml = tmpdir.join(scope, config + ".yaml")
+        config_yaml.ensure()
+        with config_yaml.open("w") as f:
+            syaml.dump_config(data, f)
+        return config_yaml
+
+    return _write
+
+
+def _include_cache_root():
+    return join_path(str(tempfile.mkdtemp()), "user_cache", "includes")
+
+
+@pytest.fixture()
+def mock_include_cache(monkeypatch):
+    """Override the include cache directory so tests don't pollute user cache."""
+    monkeypatch.setattr(spack.config, "_include_cache_location", _include_cache_root)

lib/spack/spack/test/env.py

@@ -12,6 +12,7 @@
 
 import spack.config
 import spack.environment as ev
+import spack.platforms
 import spack.solver.asp
 import spack.spec
 from spack.environment.environment import (
@@ -519,7 +520,9 @@ def test_error_message_when_using_too_new_lockfile(tmp_path):
         ("when_possible", True),
     ],
 )
-def test_environment_concretizer_scheme_used(tmp_path, unify_in_lower_scope, unify_in_spack_yaml):
+def test_environment_concretizer_scheme_used(
+    tmp_path, mutable_config, unify_in_lower_scope, unify_in_spack_yaml
+):
     """Tests that "unify" settings in spack.yaml always take precedence over settings in lower
     configuration scopes.
     """
@@ -533,10 +536,11 @@ def test_environment_concretizer_scheme_used(tmp_path, unify_in_lower_scope, uni
     unify: {str(unify_in_spack_yaml).lower()}
 """
     )
-
-    with spack.config.override("concretizer:unify", unify_in_lower_scope):
-        with ev.Environment(manifest.parent) as e:
-            assert e.unify == unify_in_spack_yaml
+    mutable_config.set("concretizer:unify", unify_in_lower_scope)
+    assert mutable_config.get("concretizer:unify") == unify_in_lower_scope
+    with ev.Environment(manifest.parent) as e:
+        assert mutable_config.get("concretizer:unify") == unify_in_spack_yaml
+        assert e.unify == unify_in_spack_yaml
 
 
 @pytest.mark.parametrize("unify_in_config", [True, False, "when_possible"])
@@ -918,3 +922,50 @@ def test_environment_from_name_or_dir(mock_packages, mutable_mock_env_path, tmp_
 
     with pytest.raises(ev.SpackEnvironmentError, match="no such environment"):
         _ = ev.environment_from_name_or_dir("fake-env")
+
+
+def test_env_include_configs(mutable_mock_env_path, mock_packages):
+    """check config and package values using new include schema"""
+    env_path = mutable_mock_env_path
+    env_path.mkdir()
+
+    this_os = spack.platforms.host().default_os
+    config_root = env_path / this_os
+    config_root.mkdir()
+    config_path = str(config_root / "config.yaml")
+    with open(config_path, "w", encoding="utf-8") as f:
+        f.write(
+            """\
+config:
+  verify_ssl: False
+"""
+        )
+
+    packages_path = str(env_path / "packages.yaml")
+    with open(packages_path, "w", encoding="utf-8") as f:
+        f.write(
+            """\
+packages:
+  python:
+    require:
+    - spec: "@3.11:"
+"""
+        )
+
+    spack_yaml = env_path / ev.manifest_name
+    spack_yaml.write_text(
+        f"""\
+spack:
+  include:
+  - path: {config_path}
+    optional: true
+  - path: {packages_path}
+"""
+    )
+
+    e = ev.Environment(env_path)
+    with e.manifest.use_config():
+        assert not spack.config.get("config:verify_ssl")
+        python_reqs = spack.config.get("packages")["python"]["require"]
+        req_specs = set(x["spec"] for x in python_reqs)
+        assert req_specs == set(["@3.11:"])

lib/spack/spack/test/installer.py

@@ -680,13 +680,19 @@ def test_install_spliced_build_spec_installed(install_mockery, capfd, mock_fetch
         assert node.build_spec.installed
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.not_on_windows("lacking windows support for binary installs")
 @pytest.mark.parametrize("transitive", [True, False])
 @pytest.mark.parametrize(
     "root_str", ["splice-t^splice-h~foo", "splice-h~foo", "splice-vt^splice-a"]
 )
 def test_install_splice_root_from_binary(
-    install_mockery, mock_fetch, mutable_temporary_mirror, transitive, root_str
+    mutable_mock_env_path,
+    install_mockery,
+    mock_fetch,
+    mutable_temporary_mirror,
+    transitive,
+    root_str,
 ):
     """Test installing a spliced spec with the root available in binary cache"""
     # Test splicing and rewiring a spec with the same name, different hash.
@@ -977,7 +983,6 @@ class MyBuildException(Exception):
 
 
 def _install_fail_my_build_exception(installer, task, install_status, **kwargs):
-    print(task, task.pkg.name)
     if task.pkg.name == "pkg-a":
         raise MyBuildException("mock internal package build error for pkg-a")
     else:

lib/spack/spack/test/llnl/util/lang.py

@@ -364,3 +364,44 @@ def test_fnmatch_multiple():
     assert not regex.match("libbar.so.1")
     assert not regex.match("libfoo.solibbar.so")
     assert not regex.match("libbaz.so")
+
+
+class TestPriorityOrderedMapping:
+    @pytest.mark.parametrize(
+        "elements,expected",
+        [
+            # Push out-of-order with explicit, and different, priorities
+            ([("b", 2), ("a", 1), ("d", 4), ("c", 3)], ["a", "b", "c", "d"]),
+            # Push in-order with priority=None
+            ([("a", None), ("b", None), ("c", None), ("d", None)], ["a", "b", "c", "d"]),
+            # Mix explicit and implicit priorities
+            ([("b", 2), ("c", None), ("a", 1), ("d", None)], ["a", "b", "c", "d"]),
+            ([("b", 10), ("c", None), ("a", -20), ("d", None)], ["a", "b", "c", "d"]),
+            ([("b", 10), ("c", None), ("a", 20), ("d", None)], ["b", "c", "a", "d"]),
+            # Adding the same key twice with different priorities
+            ([("b", 10), ("c", None), ("a", 20), ("d", None), ("a", -20)], ["a", "b", "c", "d"]),
+            # Adding the same key twice, no priorities
+            ([("b", None), ("a", None), ("b", None)], ["a", "b"]),
+        ],
+    )
+    def test_iteration_order(self, elements, expected):
+        """Tests that the iteration order respects priorities, no matter the insertion order."""
+        m = llnl.util.lang.PriorityOrderedMapping()
+        for key, priority in elements:
+            m.add(key, value=None, priority=priority)
+        assert list(m) == expected
+
+    def test_reverse_iteration(self):
+        """Tests that we can conveniently use reverse iteration"""
+        m = llnl.util.lang.PriorityOrderedMapping()
+        for key, value in [("a", 1), ("b", 2), ("c", 3)]:
+            m.add(key, value=value)
+
+        assert list(m) == ["a", "b", "c"]
+        assert list(reversed(m)) == ["c", "b", "a"]
+
+        assert list(m.keys()) == ["a", "b", "c"]
+        assert list(m.reversed_keys()) == ["c", "b", "a"]
+
+        assert list(m.values()) == [1, 2, 3]
+        assert list(m.reversed_values()) == [3, 2, 1]

lib/spack/spack/test/main.py

@@ -3,6 +3,9 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 
+import os
+import os.path
+
 import pytest
 
 import llnl.util.filesystem as fs
@@ -13,8 +16,10 @@
 import spack.error
 import spack.main
 import spack.paths
+import spack.platforms
 import spack.util.executable as exe
 import spack.util.git
+import spack.util.spack_yaml as syaml
 
 pytestmark = pytest.mark.not_on_windows(
     "Test functionality supported but tests are failing on Win"
@@ -167,3 +172,163 @@ def test_add_command_line_scope_env(tmp_path, mutable_mock_env_path):
     assert config.get("config:install_tree:root") == "/tmp/first"
 
     assert ev.active_environment() is None  # shouldn't cause an environment to be activated
+
+
+def test_include_cfg(mock_low_high_config, write_config_file, tmpdir):
+    cfg1_path = str(tmpdir.join("include1.yaml"))
+    with open(cfg1_path, "w", encoding="utf-8") as f:
+        f.write(
+            """\
+config:
+  verify_ssl: False
+  dirty: True
+packages:
+  python:
+    require:
+    - spec: "@3.11:"
+"""
+        )
+
+    def python_cfg(_spec):
+        return f"""\
+packages:
+  python:
+    require:
+    - spec: {_spec}
+"""
+
+    def write_python_cfg(_spec, _cfg_name):
+        cfg_path = str(tmpdir.join(_cfg_name))
+        with open(cfg_path, "w", encoding="utf-8") as f:
+            f.write(python_cfg(_spec))
+        return cfg_path
+
+    # This config will not be included
+    cfg2_path = write_python_cfg("+shared", "include2.yaml")
+
+    # The config will point to this using substitutable variables,
+    # namely $os; we expect that Spack resolves these variables
+    # into the actual path of the config
+    this_os = spack.platforms.host().default_os
+    cfg3_expanded_path = os.path.join(str(tmpdir), f"{this_os}", "include3.yaml")
+    fs.mkdirp(os.path.dirname(cfg3_expanded_path))
+    with open(cfg3_expanded_path, "w", encoding="utf-8") as f:
+        f.write(python_cfg("+ssl"))
+    cfg3_abstract_path = os.path.join(str(tmpdir), "$os", "include3.yaml")
+
+    # This will be included unconditionally
+    cfg4_path = write_python_cfg("+tk", "include4.yaml")
+
+    # This config will not exist, and the config will explicitly
+    # allow this
+    cfg5_path = os.path.join(str(tmpdir), "non-existent.yaml")
+
+    include_entries = [
+        {"path": f"{cfg1_path}", "when": f'os == "{this_os}"'},
+        {"path": f"{cfg2_path}", "when": "False"},
+        {"path": cfg3_abstract_path},
+        cfg4_path,
+        {"path": cfg5_path, "optional": True},
+    ]
+    include_cfg = {"include": include_entries}
+    filename = write_config_file("include", include_cfg, "low")
+
+    assert not spack.config.get("config:dirty")
+
+    spack.main.add_command_line_scopes(mock_low_high_config, [os.path.dirname(filename)])
+
+    assert spack.config.get("config:dirty")
+    python_reqs = spack.config.get("packages")["python"]["require"]
+    req_specs = set(x["spec"] for x in python_reqs)
+    assert req_specs == set(["@3.11:", "+ssl", "+tk"])
+
+
+def test_include_duplicate_source(tmpdir, mutable_config):
+    """Check precedence when include.yaml files have the same path."""
+    include_yaml = "debug.yaml"
+    include_list = {"include": [f"./{include_yaml}"]}
+
+    system_filename = mutable_config.get_config_filename("system", "include")
+    site_filename = mutable_config.get_config_filename("site", "include")
+
+    def write_configs(include_path, debug_data):
+        fs.mkdirp(os.path.dirname(include_path))
+        with open(include_path, "w", encoding="utf-8") as f:
+            syaml.dump_config(include_list, f)
+
+        debug_path = fs.join_path(os.path.dirname(include_path), include_yaml)
+        with open(debug_path, "w", encoding="utf-8") as f:
+            syaml.dump_config(debug_data, f)
+
+    system_config = {"config": {"debug": False}}
+    write_configs(system_filename, system_config)
+    spack.main.add_command_line_scopes(mutable_config, [os.path.dirname(system_filename)])
+
+    site_config = {"config": {"debug": True}}
+    write_configs(site_filename, site_config)
+    spack.main.add_command_line_scopes(mutable_config, [os.path.dirname(site_filename)])
+
+    # Ensure takes the last value of the option pushed onto the stack
+    assert mutable_config.get("config:debug") == site_config["config"]["debug"]
+
+
+def test_include_recurse_limit(tmpdir, mutable_config):
+    """Ensure hit the recursion limit."""
+    include_yaml = "include.yaml"
+    include_list = {"include": [f"./{include_yaml}"]}
+
+    include_path = str(tmpdir.join(include_yaml))
+    with open(include_path, "w", encoding="utf-8") as f:
+        syaml.dump_config(include_list, f)
+
+    with pytest.raises(spack.config.RecursiveIncludeError, match="recursion exceeded"):
+        spack.main.add_command_line_scopes(mutable_config, [os.path.dirname(include_path)])
+
+
+# TODO: Fix this once recursive includes are processed in the expected order.
+@pytest.mark.parametrize("child,expected", [("b", True), ("c", False)])
+def test_include_recurse_diamond(tmpdir, mutable_config, child, expected):
+    """Demonstrate include parent's value overrides that of child in diamond include.
+
+    Check that the value set by b or c overrides that set by d.
+    """
+    configs_root = tmpdir.join("configs")
+    configs_root.mkdir()
+
+    def write(path, contents):
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(contents)
+
+    def debug_contents(value):
+        return f"config:\n  debug: {value}\n"
+
+    def include_contents(paths):
+        indent = "\n  - "
+        values = indent.join([str(p) for p in paths])
+        return f"include:{indent}{values}"
+
+    a_yaml = tmpdir.join("a.yaml")
+    b_yaml = configs_root.join("b.yaml")
+    c_yaml = configs_root.join("c.yaml")
+    d_yaml = configs_root.join("d.yaml")
+    debug_yaml = configs_root.join("enable_debug.yaml")
+
+    write(debug_yaml, debug_contents("true"))
+
+    a_contents = f"""\
+include:
+- {b_yaml}
+- {c_yaml}
+"""
+    write(a_yaml, a_contents)
+    write(d_yaml, debug_contents("false"))
+
+    write(b_yaml, include_contents([debug_yaml, d_yaml] if child == "b" else [d_yaml]))
+    write(c_yaml, include_contents([debug_yaml, d_yaml] if child == "c" else [d_yaml]))
+
+    spack.main.add_command_line_scopes(mutable_config, [str(tmpdir)])
+
+    try:
+        assert mutable_config.get("config:debug") is expected
+    except AssertionError:
+        pytest.xfail("recursive includes are not processed in the expected order")

lib/spack/spack/test/multimethod.py

@@ -53,7 +53,7 @@ def test_no_version_match(pkg_name):
         # Constraints on compilers with a default
         ("%gcc", "has_a_default", "gcc"),
         ("%clang", "has_a_default", "clang"),
-        ("%apple-clang os=elcapitan", "has_a_default", "default"),
+        ("os=elcapitan %apple-clang", "has_a_default", "default"),
         # Constraints on dependencies
         ("^zmpi", "different_by_dep", "zmpi"),
         ("^mpich", "different_by_dep", "mpich"),
@@ -74,7 +74,7 @@ def test_multimethod_calls(
     with spack.config.override(
         "compilers", [compiler_factory(spec="apple-clang@9.1.0", operating_system="elcapitan")]
     ):
-        s = spack.concretize.concretize_one(pkg_name + constraint_str)
+        s = spack.concretize.concretize_one(f"{pkg_name} {constraint_str}")
     msg = f"Method {method_name} from {s} is giving a wrong result"
     assert getattr(s.package, method_name)() == expected_result, msg
 

lib/spack/spack/test/optional_deps.py

@@ -38,9 +38,9 @@
             {"optional-dep-test@1.1%intel": {"pkg-b": None, "pkg-c": None}},
         ),
         (
-            "optional-dep-test@1.1%intel@64.1.2+a",
+            "optional-dep-test@1.1+a%intel@64.1.2",
             {
-                "optional-dep-test@1.1%intel@64.1.2+a": {
+                "optional-dep-test@1.1+a%intel@64.1.2": {
                     "pkg-a": None,
                     "pkg-b": None,
                     "pkg-c": None,
@@ -49,8 +49,8 @@
             },
         ),
         (
-            "optional-dep-test@1.1%clang@36.5+a",
-            {"optional-dep-test@1.1%clang@36.5+a": {"pkg-b": None, "pkg-a": None, "pkg-e": None}},
+            "optional-dep-test@1.1+a%clang@36.5",
+            {"optional-dep-test@1.1+a%clang@36.5": {"pkg-b": None, "pkg-a": None, "pkg-e": None}},
         ),
         # Chained MPI
         (

lib/spack/spack/test/repo.py

@@ -319,3 +319,48 @@ def test_get_repo(self, mock_test_cache):
         # foo is not there, raise
         with pytest.raises(spack.repo.UnknownNamespaceError):
             repo.get_repo("foo")
+
+
+def test_parse_package_api_version():
+    """Test that we raise an error if a repository has a version that is not supported."""
+    # valid version
+    assert spack.repo._parse_package_api_version(
+        {"api": "v1.2"}, min_api=(1, 0), max_api=(2, 3)
+    ) == (1, 2)
+    # too new and too old
+    with pytest.raises(
+        spack.repo.BadRepoError,
+        match=r"Package API v2.4 is not supported .* \(must be between v1.0 and v2.3\)",
+    ):
+        spack.repo._parse_package_api_version({"api": "v2.4"}, min_api=(1, 0), max_api=(2, 3))
+    with pytest.raises(
+        spack.repo.BadRepoError,
+        match=r"Package API v0.9 is not supported .* \(must be between v1.0 and v2.3\)",
+    ):
+        spack.repo._parse_package_api_version({"api": "v0.9"}, min_api=(1, 0), max_api=(2, 3))
+    # default to v1.0 if not specified
+    assert spack.repo._parse_package_api_version({}, min_api=(1, 0), max_api=(2, 3)) == (1, 0)
+    # if v1.0 support is dropped we should also raise
+    with pytest.raises(
+        spack.repo.BadRepoError,
+        match=r"Package API v1.0 is not supported .* \(must be between v2.0 and v2.3\)",
+    ):
+        spack.repo._parse_package_api_version({}, min_api=(2, 0), max_api=(2, 3))
+    # finally test invalid input
+    with pytest.raises(spack.repo.BadRepoError, match="Invalid Package API version"):
+        spack.repo._parse_package_api_version({"api": "v2"}, min_api=(1, 0), max_api=(3, 3))
+    with pytest.raises(spack.repo.BadRepoError, match="Invalid Package API version"):
+        spack.repo._parse_package_api_version({"api": 2.0}, min_api=(1, 0), max_api=(3, 3))
+
+
+def test_repo_package_api_version(tmp_path: pathlib.Path):
+    """Test that we can specify the API version of a repository."""
+    (tmp_path / "example" / "packages").mkdir(parents=True)
+    (tmp_path / "example" / "repo.yaml").write_text(
+        """\
+repo:
+    namespace: example
+"""
+    )
+    cache = spack.util.file_cache.FileCache(str(tmp_path / "cache"))
+    assert spack.repo.Repo(str(tmp_path / "example"), cache=cache).package_api == (1, 0)

lib/spack/spack/test/rewiring.py

@@ -27,9 +27,7 @@ def check_spliced_spec_prefixes(spliced_spec):
         text_file_path = os.path.join(node.prefix, node.name)
         with open(text_file_path, "r", encoding="utf-8") as f:
             text = f.read()
-            print(text)
             for modded_spec in node.traverse(root=True, deptype=dt.ALL & ~dt.BUILD):
-                print(modded_spec)
                 assert modded_spec.prefix in text
 
 

lib/spack/spack/test/schema.py

@@ -17,7 +17,7 @@
 def validate_spec_schema():
     return {
         "type": "object",
-        "validate_spec": True,
+        "additionalKeysAreSpecs": True,
         "patternProperties": {r"\w[\w-]*": {"type": "string"}},
     }
 
@@ -34,7 +34,7 @@ def module_suffixes_schema():
                         "type": "object",
                         "properties": {
                             "suffixes": {
-                                "validate_spec": True,
+                                "additionalKeysAreSpecs": True,
                                 "patternProperties": {r"\w[\w-]*": {"type": "string"}},
                             }
                         },
@@ -84,6 +84,7 @@ def test_module_suffixes(module_suffixes_schema):
         "compilers",
         "config",
         "definitions",
+        "include",
         "env",
         "merged",
         "mirrors",

lib/spack/spack/test/spec_semantics.py

@@ -1751,7 +1751,6 @@ def test_package_hash_affects_dunder_and_dag_hash(mock_packages, default_mock_co
 
     assert hash(a1) == hash(a2)
     assert a1.dag_hash() == a2.dag_hash()
-    assert a1.process_hash() == a2.process_hash()
 
     a1.clear_caches()
     a2.clear_caches()
@@ -1764,7 +1763,6 @@ def test_package_hash_affects_dunder_and_dag_hash(mock_packages, default_mock_co
 
     assert hash(a1) != hash(a2)
     assert a1.dag_hash() != a2.dag_hash()
-    assert a1.process_hash() != a2.process_hash()
 
 
 def test_intersects_and_satisfies_on_concretized_spec(default_mock_concretization):

lib/spack/spack/test/spec_syntax.py

@@ -21,6 +21,7 @@
     SpecParsingError,
     SpecTokenizationError,
     SpecTokens,
+    parse_one_or_raise,
 )
 from spack.tokenize import Token
 
@@ -159,13 +160,13 @@ def _specfile_for(spec_str, filename):
         ),
         # Version after compiler
         (
-            "foo %bar@1.0 @2.0",
+            "foo @2.0 %bar@1.0",
             [
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="foo"),
-                Token(SpecTokens.COMPILER_AND_VERSION, value="%bar@1.0"),
                 Token(SpecTokens.VERSION, value="@2.0"),
+                Token(SpecTokens.COMPILER_AND_VERSION, value="%bar@1.0"),
             ],
-            "foo@2.0%bar@1.0",
+            "foo@2.0 %bar@1.0",
         ),
         # Single dependency with version
         dependency_with_version("openmpi ^hwloc@1.2e6"),
@@ -174,54 +175,54 @@ def _specfile_for(spec_str, filename):
         dependency_with_version("openmpi ^hwloc@1.2e6:1.4b7-rc3"),
         # Complex specs with multiple constraints
         (
-            "mvapich_foo ^_openmpi@1.2:1.4,1.6%intel@12.1+debug~qt_4 ^stackwalker@8.1_1e",
+            "mvapich_foo ^_openmpi@1.2:1.4,1.6+debug~qt_4 %intel@12.1 ^stackwalker@8.1_1e",
             [
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich_foo"),
                 Token(SpecTokens.DEPENDENCY, value="^"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"),
                 Token(SpecTokens.VERSION, value="@1.2:1.4,1.6"),
-                Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"),
                 Token(SpecTokens.BOOL_VARIANT, value="+debug"),
                 Token(SpecTokens.BOOL_VARIANT, value="~qt_4"),
+                Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"),
                 Token(SpecTokens.DEPENDENCY, value="^"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"),
                 Token(SpecTokens.VERSION, value="@8.1_1e"),
             ],
-            "mvapich_foo ^_openmpi@1.2:1.4,1.6%intel@12.1+debug~qt_4 ^stackwalker@8.1_1e",
+            "mvapich_foo ^_openmpi@1.2:1.4,1.6+debug~qt_4 %intel@12.1 ^stackwalker@8.1_1e",
         ),
         (
-            "mvapich_foo ^_openmpi@1.2:1.4,1.6%intel@12.1~qt_4 debug=2 ^stackwalker@8.1_1e",
+            "mvapich_foo ^_openmpi@1.2:1.4,1.6~qt_4 debug=2 %intel@12.1 ^stackwalker@8.1_1e",
             [
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich_foo"),
                 Token(SpecTokens.DEPENDENCY, value="^"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"),
                 Token(SpecTokens.VERSION, value="@1.2:1.4,1.6"),
-                Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"),
                 Token(SpecTokens.BOOL_VARIANT, value="~qt_4"),
                 Token(SpecTokens.KEY_VALUE_PAIR, value="debug=2"),
+                Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"),
                 Token(SpecTokens.DEPENDENCY, value="^"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"),
                 Token(SpecTokens.VERSION, value="@8.1_1e"),
             ],
-            "mvapich_foo ^_openmpi@1.2:1.4,1.6%intel@12.1~qt_4 debug=2 ^stackwalker@8.1_1e",
+            "mvapich_foo ^_openmpi@1.2:1.4,1.6~qt_4 debug=2 %intel@12.1 ^stackwalker@8.1_1e",
         ),
         (
-            "mvapich_foo ^_openmpi@1.2:1.4,1.6%intel@12.1 cppflags=-O3 +debug~qt_4 "
+            "mvapich_foo ^_openmpi@1.2:1.4,1.6 cppflags=-O3 +debug~qt_4 %intel@12.1 "
             "^stackwalker@8.1_1e",
             [
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich_foo"),
                 Token(SpecTokens.DEPENDENCY, value="^"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"),
                 Token(SpecTokens.VERSION, value="@1.2:1.4,1.6"),
-                Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"),
                 Token(SpecTokens.KEY_VALUE_PAIR, value="cppflags=-O3"),
                 Token(SpecTokens.BOOL_VARIANT, value="+debug"),
                 Token(SpecTokens.BOOL_VARIANT, value="~qt_4"),
+                Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"),
                 Token(SpecTokens.DEPENDENCY, value="^"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"),
                 Token(SpecTokens.VERSION, value="@8.1_1e"),
             ],
-            "mvapich_foo ^_openmpi@1.2:1.4,1.6%intel@12.1 cppflags=-O3 +debug~qt_4 "
+            "mvapich_foo ^_openmpi@1.2:1.4,1.6 cppflags=-O3 +debug~qt_4 %intel@12.1 "
             "^stackwalker@8.1_1e",
         ),
         # Specs containing YAML or JSON in the package name
@@ -235,7 +236,7 @@ def _specfile_for(spec_str, filename):
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="boost"),
                 Token(SpecTokens.VERSION, value="@3.1.4"),
             ],
-            "yaml-cpp@0.1.8%intel@12.1 ^boost@3.1.4",
+            "yaml-cpp@0.1.8 %intel@12.1 ^boost@3.1.4",
         ),
         (
             r"builtin.yaml-cpp%gcc",
@@ -243,7 +244,7 @@ def _specfile_for(spec_str, filename):
                 Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value="builtin.yaml-cpp"),
                 Token(SpecTokens.COMPILER, value="%gcc"),
             ],
-            "yaml-cpp%gcc",
+            "yaml-cpp %gcc",
         ),
         (
             r"testrepo.yaml-cpp%gcc",
@@ -251,7 +252,7 @@ def _specfile_for(spec_str, filename):
                 Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value="testrepo.yaml-cpp"),
                 Token(SpecTokens.COMPILER, value="%gcc"),
             ],
-            "yaml-cpp%gcc",
+            "yaml-cpp %gcc",
         ),
         (
             r"builtin.yaml-cpp@0.1.8%gcc@7.2.0 ^boost@3.1.4",
@@ -263,7 +264,7 @@ def _specfile_for(spec_str, filename):
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="boost"),
                 Token(SpecTokens.VERSION, value="@3.1.4"),
             ],
-            "yaml-cpp@0.1.8%gcc@7.2.0 ^boost@3.1.4",
+            "yaml-cpp@0.1.8 %gcc@7.2.0 ^boost@3.1.4",
         ),
         (
             r"builtin.yaml-cpp ^testrepo.boost ^zlib",
@@ -485,12 +486,12 @@ def _specfile_for(spec_str, filename):
             "a@1:",
         ),
         (
-            "% intel @ 12.1:12.6 + debug",
+            "+ debug % intel @ 12.1:12.6",
             [
-                Token(SpecTokens.COMPILER_AND_VERSION, value="% intel @ 12.1:12.6"),
                 Token(SpecTokens.BOOL_VARIANT, value="+ debug"),
+                Token(SpecTokens.COMPILER_AND_VERSION, value="% intel @ 12.1:12.6"),
             ],
-            "%intel@12.1:12.6+debug",
+            "+debug %intel@12.1:12.6",
         ),
         (
             "@ 12.1:12.6 + debug - qt_4",
@@ -515,7 +516,7 @@ def _specfile_for(spec_str, filename):
                 Token(SpecTokens.VERSION, value="@:0.4"),
                 Token(SpecTokens.COMPILER, value="% nvhpc"),
             ],
-            "@:0.4%nvhpc",
+            "@:0.4 %nvhpc",
         ),
         (
             "^[virtuals=mpi] openmpi",
@@ -639,15 +640,15 @@ def test_parse_single_spec(spec_str, tokens, expected_roundtrip, mock_git_test_p
             ["mvapich cppflags=-O3", "emacs"],
         ),
         (
-            "mvapich emacs @1.1.1 %intel cflags=-O3",
+            "mvapich emacs @1.1.1 cflags=-O3 %intel",
             [
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="emacs"),
                 Token(SpecTokens.VERSION, value="@1.1.1"),
-                Token(SpecTokens.COMPILER, value="%intel"),
                 Token(SpecTokens.KEY_VALUE_PAIR, value="cflags=-O3"),
+                Token(SpecTokens.COMPILER, value="%intel"),
             ],
-            ["mvapich", "emacs @1.1.1 %intel cflags=-O3"],
+            ["mvapich", "emacs @1.1.1 cflags=-O3 %intel"],
         ),
         (
             'mvapich cflags="-O3 -fPIC" emacs^ncurses%intel',
@@ -877,9 +878,7 @@ def test_ambiguous_hash(mutable_database):
     x1 = spack.concretize.concretize_one("pkg-a")
     x2 = x1.copy()
     x1._hash = "xyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"
-    x1._process_hash = "xyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"
     x2._hash = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-    x2._process_hash = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 
     assert x1 != x2  # doesn't hold when only the dag hash is modified.
 
@@ -1231,7 +1230,7 @@ def test_compare_abstract_specs():
         "foo.foo@foo+foo",
         "foo.foo@foo+foo arch=foo-foo-foo",
         "foo.foo@foo+foo arch=foo-foo-foo %foo",
-        "foo.foo@foo+foo arch=foo-foo-foo %foo cflags=foo",
+        "foo.foo@foo+foo arch=foo-foo-foo cflags=foo %foo",
     ]
     specs = [SpecParser(s).next_spec() for s in constraints]
 
@@ -1285,3 +1284,19 @@ def test_git_ref_spec_equivalences(mock_packages, lhs_str, rhs_str, expected):
 def test_platform_is_none_if_not_present(spec_str):
     s = SpecParser(spec_str).next_spec()
     assert s.architecture.platform is None, s
+
+
+def test_parse_one_or_raise_error_message():
+    with pytest.raises(ValueError) as exc:
+        parse_one_or_raise("  x y   z")
+
+    msg = """\
+expected a single spec, but got more:
+  x y   z
+    ^\
+"""
+
+    assert str(exc.value) == msg
+
+    with pytest.raises(ValueError, match="expected a single spec, but got none"):
+        parse_one_or_raise("    ")

lib/spack/spack/test/spec_yaml.py

@@ -203,13 +203,6 @@ def test_ordered_read_not_required_for_consistent_dag_hash(
     spec = spec.copy(deps=ht.dag_hash.depflag)
 
     # specs and their hashes are equal to the original
-    assert (
-        spec.process_hash()
-        == from_yaml.process_hash()
-        == from_json.process_hash()
-        == from_yaml_rev.process_hash()
-        == from_json_rev.process_hash()
-    )
     assert (
         spec.dag_hash()
         == from_yaml.dag_hash()
@@ -427,6 +420,9 @@ def test_load_json_specfiles(specfile, expected_hash, reader_cls):
     openmpi_edges = s2.edges_to_dependencies(name="openmpi")
     assert len(openmpi_edges) == 1
 
+    # Check that virtuals have been reconstructed
+    assert "mpi" in openmpi_edges[0].virtuals
+
     # The virtuals attribute must be a tuple, when read from a
     # JSON or YAML file, not a list
     for edge in s2.traverse_edges():

lib/spack/spack/test/util/environment.py

@@ -149,11 +149,8 @@ def test_reverse_environment_modifications(working_env):
     os.environ.clear()
     os.environ.update(start_env)
 
-    print(os.environ)
     to_reverse.apply_modifications()
-    print(os.environ)
     reversal.apply_modifications()
-    print(os.environ)
 
     start_env.pop("UNSET")
     assert os.environ == start_env

lib/spack/spack/test/util/path.py

@@ -134,3 +134,18 @@ def test_path_debug_padded_filter(debug, monkeypatch):
     monkeypatch.setattr(tty, "_debug", debug)
     with spack.config.override("config:install_tree", {"padded_length": 128}):
         assert expected == sup.debug_padded_filter(string)
+
+
+@pytest.mark.parametrize(
+    "path,expected",
+    [
+        ("/home/spack/path/to/file.txt", "/home/spack/path/to/file.txt"),
+        ("file:///home/another/config.yaml", "/home/another/config.yaml"),
+        ("path/to.txt", os.path.join(os.environ["SPACK_ROOT"], "path", "to.txt")),
+        (r"C:\Files (x86)\Windows\10", r"C:\Files (x86)\Windows\10"),
+        (r"E:/spack stage", "E:\\spack stage"),
+    ],
+)
+def test_canonicalize_file(path, expected):
+    """Confirm canonicalize path handles local files and file URLs."""
+    assert sup.canonicalize_path(path) == os.path.normpath(expected)

lib/spack/spack/test/util/remote_file_cache.py

@@ -0,0 +1,106 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os.path
+import sys
+
+import pytest
+
+import llnl.util.tty as tty
+from llnl.util.filesystem import join_path
+
+import spack.config
+import spack.util.remote_file_cache as rfc_util
+
+github_url = "https://github.com/fake/fake/{0}/develop"
+gitlab_url = "https://gitlab.fake.io/user/repo/-/blob/config/defaults"
+
+
+@pytest.mark.parametrize(
+    "path,err",
+    [
+        ("ssh://git@github.com:spack/", "Unsupported URL scheme"),
+        ("bad:///this/is/a/file/url/include.yaml", "Invalid URL scheme"),
+    ],
+)
+def test_rfc_local_path_bad_scheme(path, err):
+    with pytest.raises(ValueError, match=err):
+        _ = rfc_util.local_path(path, "")
+
+
+@pytest.mark.parametrize(
+    "path,expected",
+    [
+        ("/a/b/c/d/e/config.py", "/a/b/c/d/e/config.py"),
+        ("file:///this/is/a/file/url/include.yaml", "/this/is/a/file/url/include.yaml"),
+        (
+            "relative/packages.txt",
+            os.path.join(os.environ["SPACK_ROOT"], "relative", "packages.txt"),
+        ),
+        (r"C:\Files (x86)\Windows\10", r"C:\Files (x86)\Windows\10"),
+        (r"D:/spack stage", "D:\\spack stage"),
+    ],
+)
+def test_rfc_local_file(path, expected):
+    assert rfc_util.local_path(path, "") == os.path.normpath(expected)
+
+
+def test_rfc_remote_local_path_no_dest():
+    path = f"{gitlab_url}/packages.yaml"
+    with pytest.raises(ValueError, match="Requires the destination argument"):
+        _ = rfc_util.local_path(path, "")
+
+
+compilers_sha256 = (
+    "381732677538143a8f900406c0654f2730e2919a11740bdeaf35757ab3e1ef3e"
+    if sys.platform == "win32"
+    else "e91148ed5a0da7844e9f3f9cfce0fa60cce509461886bc3b006ee9eb711f69df"
+)
+
+
+@pytest.mark.parametrize(
+    "url,sha256,err,msg",
+    [
+        (
+            f"{join_path(github_url.format('tree'), 'config.yaml')}",
+            "",
+            ValueError,
+            "Requires sha256",
+        ),
+        (f"{gitlab_url}/compilers.yaml", compilers_sha256, None, ""),
+        (f"{gitlab_url}/packages.yaml", "abcdef", ValueError, "does not match"),
+        (f"{github_url.format('blob')}/README.md", "", OSError, "No such"),
+        (github_url.format("tree"), "", OSError, "No such"),
+        ("", "", ValueError, "argument is required"),
+    ],
+)
+def test_rfc_remote_local_path(
+    tmpdir, mutable_empty_config, mock_fetch_url_text, url, sha256, err, msg
+):
+    def _has_content(filename):
+        # The first element of all configuration files for this test happen to
+        # be the basename of the file so this check leverages that feature. If
+        # that changes, then this check will need to change accordingly.
+        element = f"{os.path.splitext(os.path.basename(filename))[0]}:"
+        with open(filename, "r", encoding="utf-8") as fd:
+            for line in fd:
+                if element in line:
+                    return True
+        tty.debug(f"Expected {element} in '{filename}'")
+        return False
+
+    def _dest_dir():
+        return join_path(tmpdir.strpath, "cache")
+
+    if err is not None:
+        with spack.config.override("config:url_fetch_method", "curl"):
+            with pytest.raises(err, match=msg):
+                rfc_util.local_path(url, sha256, _dest_dir)
+    else:
+        with spack.config.override("config:url_fetch_method", "curl"):
+            path = rfc_util.local_path(url, sha256, _dest_dir)
+            assert os.path.exists(path)
+            # Ensure correct file is "fetched"
+            assert os.path.basename(path) == os.path.basename(url)
+            # Ensure contents of the file contains expected config element
+            assert _has_content(path)

lib/spack/spack/test/util/unparse/unparse.py

@@ -257,7 +257,6 @@ def test_core_lib_files():
                 names.append(os.path.join(test_dir, n))
 
     for filename in names:
-        print("Testing %s" % filename)
         source = read_pyfile(filename)
         check_ast_roundtrip(source)
 

lib/spack/spack/test/variant.py

@@ -685,22 +685,6 @@ def test_str(self) -> None:
         c["shared"] = BoolValuedVariant("shared", True)
         assert str(c) == "+shared feebar=foo foo=bar,baz foobar=fee"
 
-    def test_concrete(self, mock_packages, config) -> None:
-        spec = Spec("pkg-a")
-        assert not VariantMap(spec).concrete
-
-        # concrete if associated spec is concrete
-        spec = spack.concretize.concretize_one(spec)
-        assert VariantMap(spec).concrete
-
-        # concrete if all variants are present (even if spec not concrete)
-        spec._mark_concrete(False)
-        assert spec.variants.concrete
-
-        # remove a variant to test the condition
-        del spec.variants["foo"]
-        assert not spec.variants.concrete
-
 
 def test_disjoint_set_initialization_errors():
     # Constructing from non-disjoint sets should raise an exception

lib/spack/spack/test/verification.py

@@ -133,7 +133,7 @@ def test_check_prefix_manifest(tmpdir):
 
     spec = spack.spec.Spec("libelf")
     spec._mark_concrete()
-    spec.prefix = prefix
+    spec.set_prefix(prefix)
 
     results = spack.verify.check_spec_manifest(spec)
     assert results.has_errors()

lib/spack/spack/test/views.py

@@ -35,8 +35,8 @@ def test_view_with_spec_not_contributing_files(mock_packages, tmpdir):
 
     a = Spec("pkg-a")
     b = Spec("pkg-b")
-    a.prefix = os.path.join(tmpdir, "a")
-    b.prefix = os.path.join(tmpdir, "b")
+    a.set_prefix(os.path.join(tmpdir, "a"))
+    b.set_prefix(os.path.join(tmpdir, "b"))
     a._mark_concrete()
     b._mark_concrete()