Minor update in Care package

This reverts commit 620fbc1b01.

.github/workflows/ci.yaml

@@ -9,6 +9,7 @@ on:
     branches:
       - develop
       - releases/**
+  merge_group:
 
 concurrency:
   group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
@@ -25,13 +26,17 @@ jobs:
       packages: ${{ steps.filter.outputs.packages }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-        if: ${{ github.event_name == 'push' }}
+        if: ${{ github.event_name == 'push' || github.event_name == 'merge_group' }}
         with:
           fetch-depth: 0
             # For pull requests it's not necessary to checkout the code
       - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
         id: filter
         with:
+          # For merge group events, compare against the target branch (main)
+          base: ${{ github.event_name == 'merge_group' && github.event.merge_group.base_ref || '' }}
+          # For merge group events, use the merge group head ref
+          ref: ${{ github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.ref }}
           # See https://github.com/dorny/paths-filter/issues/56 for the syntax used below
           # Don't run if we only modified packages in the
           # built-in repository or documentation
@@ -76,10 +81,11 @@ jobs:
 
   prechecks:
     needs: [ changes ]
-    uses: ./.github/workflows/valid-style.yml
+    uses: ./.github/workflows/prechecks.yml
     secrets: inherit
     with:
       with_coverage: ${{ needs.changes.outputs.core }}
+      with_packages: ${{ needs.changes.outputs.packages }}
 
   import-check:
     needs: [ changes ]
@@ -93,7 +99,7 @@ jobs:
     - name: Success
       run: |
         if [ "${{ needs.prechecks.result }}" == "failure" ] || [ "${{ needs.prechecks.result }}" == "canceled" ]; then
-          echo "Unit tests failed." 
+          echo "Unit tests failed."
           exit 1
         else
           exit 0
@@ -101,6 +107,7 @@ jobs:
 
   coverage:
     needs: [ unit-tests, prechecks ]
+    if: ${{ needs.changes.outputs.core }}
     uses: ./.github/workflows/coverage.yml
     secrets: inherit
 
@@ -113,10 +120,10 @@ jobs:
     - name: Status summary
       run: |
         if [ "${{ needs.unit-tests.result }}" == "failure" ] || [ "${{ needs.unit-tests.result }}" == "canceled" ]; then
-          echo "Unit tests failed." 
+          echo "Unit tests failed."
           exit 1
         elif [ "${{ needs.bootstrap.result }}" == "failure" ] || [ "${{ needs.bootstrap.result }}" == "canceled" ]; then
-          echo "Bootstrap tests failed." 
+          echo "Bootstrap tests failed."
           exit 1
         else
           exit 0

.github/workflows/prechecks.yml

@@ -1,4 +1,4 @@
-name: style
+name: prechecks
 
 on:
   workflow_call:
@@ -6,6 +6,9 @@ on:
       with_coverage:
         required: true
         type: string
+      with_packages:
+        required: true
+        type: string
 
 concurrency:
   group: style-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
@@ -30,6 +33,7 @@ jobs:
       run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
     - name: vermin (Repositories)
       run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv var/spack/repos
+
   # Run style checks on the files that have been changed
   style:
     runs-on: ubuntu-latest
@@ -53,12 +57,25 @@ jobs:
     - name: Run style tests
       run: |
           share/spack/qa/run-style-tests
+
   audit:
     uses: ./.github/workflows/audit.yaml
     secrets: inherit
     with:
       with_coverage: ${{ inputs.with_coverage }}
       python_version: '3.13'
+
+  verify-checksums:
+    if: ${{ inputs.with_packages == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+      with:
+        fetch-depth: 2
+    - name: Verify Added Checksums
+      run: |
+        bin/spack ci verify-versions HEAD^1 HEAD
+
   # Check that spack can bootstrap the development environment on Python 3.6 - RHEL8
   bootstrap-dev-rhel8:
     runs-on: ubuntu-latest

lib/spack/docs/configuration.rst

@@ -14,6 +14,7 @@ case you want to skip directly to specific docs:
 * :ref:`compilers.yaml <compiler-config>`
 * :ref:`concretizer.yaml <concretizer-options>`
 * :ref:`config.yaml <config-yaml>`
+* :ref:`include.yaml <include-yaml>`
 * :ref:`mirrors.yaml <mirrors>`
 * :ref:`modules.yaml <modules>`
 * :ref:`packages.yaml <packages-config>`

lib/spack/docs/environments.rst

@@ -457,6 +457,13 @@ developed package in the environment are concretized to match the
 version (and other constraints) passed as the spec argument to the
 ``spack develop`` command.
 
+When working deep in the graph it is often desirable to have multiple specs marked
+as ``develop`` so you don't have to restage and/or do full rebuilds each time you
+call ``spack install``.  The ``--recursive`` flag can be used in these scenarios
+to ensure that all the dependents of the initial spec you provide are also marked
+as develop specs.  The ``--recursive`` flag requires a pre-concretized environment
+so the graph can be traversed from the supplied spec all the way to the root specs.
+
 For packages with ``git`` attributes, git branches, tags, and commits can
 also be used as valid concrete versions (see :ref:`version-specifier`).
 This means that for a package ``foo``, ``spack develop foo@git.main`` will clone
@@ -670,24 +677,45 @@ This configuration sets the default compiler for all packages to
 Included configurations
 ^^^^^^^^^^^^^^^^^^^^^^^
 
-Spack environments allow an ``include`` heading in their yaml
-schema. This heading pulls in external configuration files and applies
-them to the environment.
+Spack environments allow an ``include`` heading in their yaml schema.
+This heading pulls in external configuration files and applies them to
+the environment.
 
 .. code-block:: yaml
 
    spack:
      include:
-     - relative/path/to/config.yaml
+     - environment/relative/path/to/config.yaml
      - https://github.com/path/to/raw/config/compilers.yaml
      - /absolute/path/to/packages.yaml
+     - path: /path/to/$os/$target/environment
+       optional: true
+     - path: /path/to/os-specific/config-dir
+       when: os == "ventura"
+
+Included configuration files are required *unless* they are explicitly optional
+or the entry's condition evaluates to ``false``. Optional includes are specified
+with the ``optional`` clause and conditional with the ``when`` clause. (See
+:ref:`include-yaml` for more information on optional and conditional entries.)
+
+Files are listed using paths to individual files or directories containing them.
+Path entries may be absolute or relative to the environment or specified as 
+URLs. URLs to individual files need link to the **raw** form of the file's 
+contents (e.g., `GitHub
+<https://docs.github.com/en/repositories/working-with-files/using-files/viewing-and-understanding-files#viewing-or-copying-the-raw-file-content>`_ 
+or `GitLab
+<https://docs.gitlab.com/ee/api/repository_files.html#get-raw-file-from-repository>`_).
+Only the ``file``, ``ftp``, ``http`` and ``https`` protocols (or schemes) are
+supported. Spack-specific, environment and user path variables can be used.
+(See :ref:`config-file-variables` for more information.)
+
+.. warning::
+
+   Recursive includes are not currently processed in a breadth-first manner
+   so the value of a configuration option that is altered by multiple included
+   files may not be what you expect. This will be addressed in a future
+   update.
 
-Environments can include files or URLs. File paths can be relative or
-absolute. URLs include the path to the text for individual files or
-can be the path to a directory containing configuration files.
-Spack supports ``file``, ``http``, ``https`` and ``ftp`` protocols (or
-schemes). Spack-specific, environment and user path variables may be
-used in these paths. See :ref:`config-file-variables` for more information.
 
 ^^^^^^^^^^^^^^^^^^^^^^^^
 Configuration precedence

lib/spack/docs/frequently_asked_questions.rst

@@ -30,7 +30,7 @@ than always choosing the latest versions or default variants.
 
 .. note::
 
-    As a rule of thumb: requirements + constraints > reuse > preferences > defaults.
+    As a rule of thumb: requirements + constraints > strong preferences > reuse > preferences > defaults.
 
 The following set of criteria (from lowest to highest precedence) explain
 common cases where concretization output may seem surprising at first.
@@ -56,7 +56,19 @@ common cases where concretization output may seem surprising at first.
       concretizer:
         reuse: dependencies  # other options are 'true' and 'false'
 
-3. :ref:`Package requirements <package-requirements>` configured in ``packages.yaml``,
+3. :ref:`Strong preferences <package-strong-preferences>` configured in ``packages.yaml``
+   are higher priority than reuse, and can be used to strongly prefer a specific version
+   or variant, without erroring out if it's not possible. Strong preferences are specified
+   as follows:
+
+   .. code-block:: yaml
+
+      packages:
+        foo:
+          prefer:
+          - "@1.1: ~mpi"
+
+4. :ref:`Package requirements <package-requirements>` configured in ``packages.yaml``,
    and constraints from the command line as well as ``package.py`` files override all
    of the above. Requirements are specified as follows:
 
@@ -66,6 +78,8 @@ common cases where concretization output may seem surprising at first.
         foo:
           require:
           - "@1.2: +mpi"
+          conflicts:
+          - "@1.4"
 
 Requirements and constraints restrict the set of possible solutions, while reuse
 behavior and preferences influence what an optimal solution looks like.

lib/spack/docs/include_yaml.rst

@@ -0,0 +1,51 @@
+.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+
+   SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+.. _include-yaml:
+
+===============================
+Include Settings (include.yaml)
+===============================
+
+Spack allows you to include configuration files through ``include.yaml``.
+Using the ``include:`` heading results in pulling in external configuration
+information to be used by any Spack command.
+
+Included configuration files are required *unless* they are explicitly optional
+or the entry's condition evaluates to ``false``. Optional includes are specified
+with the ``optional`` clause and conditional with the ``when`` clause. For
+example,
+
+.. code-block:: yaml
+
+   include:
+   - /path/to/a/required/config.yaml
+   - path: /path/to/$os/$target/config
+     optional: true
+   - path: /path/to/os-specific/config-dir
+     when: os == "ventura"
+
+shows all three. The first entry, ``/path/to/a/required/config.yaml``,
+indicates that included ``config.yaml`` file is required (so must exist).
+Use of ``optional: true`` for ``/path/to/$os/$target/config`` means
+the path is only included if it exists. The condition ``os == "ventura"``
+in the ``when`` clause for ``/path/to/os-specific/config-dir`` means the
+path is only included when the operating system (``os``) is ``ventura``.
+
+The same conditions and variables in `Spec List References 
+<https://spack.readthedocs.io/en/latest/environments.html#spec-list-references>`_
+can be used for conditional activation in the ``when`` clauses.
+
+Included files can be specified by path or by their parent directory.
+Paths may be absolute, relative (to the configuration file including the path), 
+or specified as URLs. Only the ``file``, ``ftp``, ``http`` and ``https`` protocols (or
+schemes) are supported. Spack-specific, environment and user path variables
+can be used. (See :ref:`config-file-variables` for more information.)
+
+.. warning::
+
+   Recursive includes are not currently processed in a breadth-first manner
+   so the value of a configuration option that is altered by multiple included
+   files may not be what you expect. This will be addressed in a future
+   update.

lib/spack/docs/index.rst

@@ -71,6 +71,7 @@ or refer to the full manual below.
 
    configuration
    config_yaml
+   include_yaml
    packages_yaml
    build_settings
    environments

lib/spack/docs/packages_yaml.rst

@@ -486,6 +486,8 @@ present. For instance with a configuration like:
 
 you will use ``mvapich2~cuda %gcc`` as an ``mpi`` provider.
 
+.. _package-strong-preferences:
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Conflicts and strong preferences
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

lib/spack/llnl/util/lang.py

@@ -11,6 +11,7 @@
 import re
 import sys
 import traceback
+import types
 import typing
 import warnings
 from datetime import datetime, timedelta
@@ -707,14 +708,24 @@ def __init__(self, wrapped_object):
 
 
 class Singleton:
-    """Simple wrapper for lazily initialized singleton objects."""
+    """Wrapper for lazily initialized singleton objects."""
 
-    def __init__(self, factory):
+    def __init__(self, factory: Callable[[], object]):
         """Create a new singleton to be inited with the factory function.
 
+        Most factories will simply create the object to be initialized and
+        return it.
+
+        In some cases, e.g. when bootstrapping some global state, the singleton
+        may need to be initialized incrementally. If the factory returns a generator
+        instead of a regular object, the singleton will assign each result yielded by
+        the generator to the singleton instance. This allows methods called by
+        the factory in later stages to refer back to the singleton.
+
         Args:
-            factory (function): function taking no arguments that
-                creates the singleton instance.
+            factory (function): function taking no arguments that creates the
+                singleton instance.
+
         """
         self.factory = factory
         self._instance = None
@@ -722,7 +733,16 @@ def __init__(self, factory):
     @property
     def instance(self):
         if self._instance is None:
-            self._instance = self.factory()
+            instance = self.factory()
+
+            if isinstance(instance, types.GeneratorType):
+                # if it's a generator, assign every value
+                for value in instance:
+                    self._instance = value
+            else:
+                # if not, just assign the result like a normal singleton
+                self._instance = instance
+
         return self._instance
 
     def __getattr__(self, name):

lib/spack/spack/build_environment.py

@@ -881,21 +881,6 @@ def get_rpath_deps(pkg: spack.package_base.PackageBase) -> List[spack.spec.Spec]
     return _get_rpath_deps_from_spec(pkg.spec, pkg.transitive_rpaths)
 
 
-def load_external_modules(pkg):
-    """Traverse a package's spec DAG and load any external modules.
-
-    Traverse a package's dependencies and load any external modules
-    associated with them.
-
-    Args:
-        pkg (spack.package_base.PackageBase): package to load deps for
-    """
-    for dep in list(pkg.spec.traverse()):
-        external_modules = dep.external_modules or []
-        for external_module in external_modules:
-            load_module(external_module)
-
-
 def setup_package(pkg, dirty, context: Context = Context.BUILD):
     """Execute all environment setup routines."""
     if context not in (Context.BUILD, Context.TEST):
@@ -946,7 +931,7 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD):
         for mod in pkg.compiler.modules:
             load_module(mod)
 
-    load_external_modules(pkg)
+    load_external_modules(setup_context)
 
     # Make sure nothing's strange about the Spack environment.
     validate(env_mods, tty.warn)
@@ -1235,6 +1220,21 @@ def _make_runnable(self, dep: spack.spec.Spec, env: EnvironmentModifications):
                 env.prepend_path("PATH", bin_dir)
 
 
+def load_external_modules(context: SetupContext) -> None:
+    """Traverse a package's spec DAG and load any external modules.
+
+    Traverse a package's dependencies and load any external modules
+    associated with them.
+
+    Args:
+        context: A populated SetupContext object
+    """
+    for spec, _ in context.external:
+        external_modules = spec.external_modules or []
+        for external_module in external_modules:
+            load_module(external_module)
+
+
 def _setup_pkg_and_run(
     serialized_pkg: "spack.subprocess_context.PackageInstallContext",
     function: Callable,

lib/spack/spack/build_systems/cached_cmake.py

@@ -2,9 +2,10 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import collections.abc
+import enum
 import os
 import re
-from typing import Tuple
+from typing import Optional, Tuple
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
@@ -13,6 +14,7 @@
 import spack.spec
 import spack.util.prefix
 from spack.directives import depends_on
+from spack.util.executable import which_string
 
 from .cmake import CMakeBuilder, CMakePackage
 
@@ -180,6 +182,64 @@ def initconfig_compiler_entries(self):
 
         return entries
 
+    class Scheduler(enum.Enum):
+        LSF = enum.auto()
+        SLURM = enum.auto()
+        FLUX = enum.auto()
+
+    def get_scheduler(self) -> Optional[Scheduler]:
+        spec = self.pkg.spec
+
+        # Check for Spectrum-mpi, which always uses LSF or LSF MPI variant
+        if spec.satisfies("^spectrum-mpi") or spec["mpi"].satisfies("schedulers=lsf"):
+            return self.Scheduler.LSF
+
+        # Check for Slurm MPI variants
+        slurm_checks = ["+slurm", "schedulers=slurm", "process_managers=slurm"]
+        if any(spec["mpi"].satisfies(variant) for variant in slurm_checks):
+            return self.Scheduler.SLURM
+
+        # TODO improve this when MPI implementations support flux
+        # Do this check last to avoid using a flux wrapper present next to Slurm/ LSF schedulers
+        if which_string("flux") is not None:
+            return self.Scheduler.FLUX
+
+        return None
+
+    def get_mpi_exec(self) -> Optional[str]:
+        spec = self.pkg.spec
+        scheduler = self.get_scheduler()
+
+        if scheduler == self.Scheduler.LSF:
+            return which_string("lrun")
+
+        elif scheduler == self.Scheduler.SLURM:
+            if spec["mpi"].external:
+                return which_string("srun")
+            else:
+                return os.path.join(spec["slurm"].prefix.bin, "srun")
+
+        elif scheduler == self.Scheduler.FLUX:
+            flux = which_string("flux")
+            return f"{flux};run" if flux else None
+
+        elif hasattr(spec["mpi"].package, "mpiexec"):
+            return spec["mpi"].package.mpiexec
+
+        else:
+            mpiexec = os.path.join(spec["mpi"].prefix.bin, "mpirun")
+            if not os.path.exists(mpiexec):
+                mpiexec = os.path.join(spec["mpi"].prefix.bin, "mpiexec")
+            return mpiexec
+
+    def get_mpi_exec_num_proc(self) -> str:
+        scheduler = self.get_scheduler()
+
+        if scheduler in [self.Scheduler.FLUX, self.Scheduler.LSF, self.Scheduler.SLURM]:
+            return "-n"
+        else:
+            return "-np"
+
     def initconfig_mpi_entries(self):
         spec = self.pkg.spec
 
@@ -199,27 +259,10 @@ def initconfig_mpi_entries(self):
         if hasattr(spec["mpi"], "mpifc"):
             entries.append(cmake_cache_path("MPI_Fortran_COMPILER", spec["mpi"].mpifc))
 
-        # Check for slurm
-        using_slurm = False
-        slurm_checks = ["+slurm", "schedulers=slurm", "process_managers=slurm"]
-        if any(spec["mpi"].satisfies(variant) for variant in slurm_checks):
-            using_slurm = True
-
         # Determine MPIEXEC
-        if using_slurm:
-            if spec["mpi"].external:
-                # Heuristic until we have dependents on externals
-                mpiexec = "/usr/bin/srun"
-            else:
-                mpiexec = os.path.join(spec["slurm"].prefix.bin, "srun")
-        elif hasattr(spec["mpi"].package, "mpiexec"):
-            mpiexec = spec["mpi"].package.mpiexec
-        else:
-            mpiexec = os.path.join(spec["mpi"].prefix.bin, "mpirun")
-            if not os.path.exists(mpiexec):
-                mpiexec = os.path.join(spec["mpi"].prefix.bin, "mpiexec")
+        mpiexec = self.get_mpi_exec()
 
-        if not os.path.exists(mpiexec):
+        if mpiexec is None or not os.path.exists(mpiexec.split(";")[0]):
             msg = "Unable to determine MPIEXEC, %s tests may fail" % self.pkg.name
             entries.append("# {0}\n".format(msg))
             tty.warn(msg)
@@ -232,10 +275,7 @@ def initconfig_mpi_entries(self):
                 entries.append(cmake_cache_path("MPIEXEC", mpiexec))
 
         # Determine MPIEXEC_NUMPROC_FLAG
-        if using_slurm:
-            entries.append(cmake_cache_string("MPIEXEC_NUMPROC_FLAG", "-n"))
-        else:
-            entries.append(cmake_cache_string("MPIEXEC_NUMPROC_FLAG", "-np"))
+        entries.append(cmake_cache_string("MPIEXEC_NUMPROC_FLAG", self.get_mpi_exec_num_proc()))
 
         return entries
 
@@ -278,23 +318,18 @@ def initconfig_hardware_entries(self):
             entries.append("# ROCm")
             entries.append("#------------------{0}\n".format("-" * 30))
 
-            # Explicitly setting HIP_ROOT_DIR may be a patch that is no longer necessary
-            entries.append(cmake_cache_path("HIP_ROOT_DIR", "{0}".format(spec["hip"].prefix)))
-            llvm_bin = spec["llvm-amdgpu"].prefix.bin
-            llvm_prefix = spec["llvm-amdgpu"].prefix
-            # Some ROCm systems seem to point to /<path>/rocm-<ver>/ and
-            # others point to /<path>/rocm-<ver>/llvm
-            if os.path.basename(os.path.normpath(llvm_prefix)) != "llvm":
-                llvm_bin = os.path.join(llvm_prefix, "llvm/bin/")
-            entries.append(
-                cmake_cache_filepath("CMAKE_HIP_COMPILER", os.path.join(llvm_bin, "clang++"))
-            )
+            rocm_root = os.path.dirname(spec["llvm-amdgpu"].prefix)
+            entries.append(cmake_cache_path("ROCM_PATH", rocm_root))
+
             archs = self.spec.variants["amdgpu_target"].value
             if archs[0] != "none":
                 arch_str = ";".join(archs)
                 entries.append(cmake_cache_string("CMAKE_HIP_ARCHITECTURES", arch_str))
-                entries.append(cmake_cache_string("AMDGPU_TARGETS", arch_str))
-                entries.append(cmake_cache_string("GPU_TARGETS", arch_str))
+
+            llvm_bin = spec["llvm-amdgpu"].prefix.bin
+            entries.append(
+                cmake_cache_filepath("CMAKE_HIP_COMPILER", os.path.join(llvm_bin, "amdclang++"))
+            )
 
             if spec.satisfies("%gcc"):
                 entries.append(
@@ -303,6 +338,15 @@ def initconfig_hardware_entries(self):
                     )
                 )
 
+            # Extra definitions that might be required in other cases
+            if not spec.satisfies("^blt"):
+                entries.append(cmake_cache_path("HIP_ROOT_DIR", "{0}".format(spec["hip"].prefix)))
+
+                if archs[0] != "none":
+                    arch_str = ";".join(archs)
+                    entries.append(cmake_cache_string("AMDGPU_TARGETS", arch_str))
+                    entries.append(cmake_cache_string("GPU_TARGETS", arch_str))
+
         return entries
 
     def std_initconfig_entries(self):

lib/spack/spack/ci/__init__.py

@@ -6,6 +6,7 @@
 import codecs
 import json
 import os
+import pathlib
 import re
 import shutil
 import stat
@@ -13,7 +14,7 @@
 import tempfile
 import zipfile
 from collections import namedtuple
-from typing import Callable, Dict, List, Set
+from typing import Callable, Dict, List, Set, Union
 from urllib.request import Request
 
 import llnl.path
@@ -23,7 +24,6 @@
 
 import spack
 import spack.binary_distribution as bindist
-import spack.builder
 import spack.concretize
 import spack.config as cfg
 import spack.environment as ev
@@ -33,6 +33,7 @@
 import spack.paths
 import spack.repo
 import spack.spec
+import spack.store
 import spack.util.git
 import spack.util.gpg as gpg_util
 import spack.util.spack_yaml as syaml
@@ -41,6 +42,7 @@
 from spack import traverse
 from spack.error import SpackError
 from spack.reporters.cdash import SPACK_CDASH_TIMEOUT
+from spack.version import GitVersion, StandardVersion
 
 from .common import (
     IS_WINDOWS,
@@ -79,6 +81,45 @@ def get_change_revisions():
     return None, None
 
 
+def get_added_versions(
+    checksums_version_dict: Dict[str, Union[StandardVersion, GitVersion]],
+    path: str,
+    from_ref: str = "HEAD~1",
+    to_ref: str = "HEAD",
+) -> List[Union[StandardVersion, GitVersion]]:
+    """Get a list of the versions added between `from_ref` and `to_ref`.
+    Args:
+       checksums_version_dict (Dict): all package versions keyed by known checksums.
+       path (str): path to the package.py
+       from_ref (str): oldest git ref, defaults to `HEAD~1`
+       to_ref (str): newer git ref, defaults to `HEAD`
+    Returns: list of versions added between refs
+    """
+    git_exe = spack.util.git.git(required=True)
+
+    # Gather git diff
+    diff_lines = git_exe("diff", from_ref, to_ref, "--", path, output=str).split("\n")
+
+    # Store added and removed versions
+    # Removed versions are tracked here to determine when versions are moved in a file
+    # and show up as both added and removed in a git diff.
+    added_checksums = set()
+    removed_checksums = set()
+
+    # Scrape diff for modified versions and prune added versions if they show up
+    # as also removed (which means they've actually just moved in the file and
+    # we shouldn't need to rechecksum them)
+    for checksum in checksums_version_dict.keys():
+        for line in diff_lines:
+            if checksum in line:
+                if line.startswith("+"):
+                    added_checksums.add(checksum)
+                if line.startswith("-"):
+                    removed_checksums.add(checksum)
+
+    return [checksums_version_dict[c] for c in added_checksums - removed_checksums]
+
+
 def get_stack_changed(env_path, rev1="HEAD^", rev2="HEAD"):
     """Given an environment manifest path and two revisions to compare, return
     whether or not the stack was changed.  Returns True if the environment
@@ -224,7 +265,7 @@ def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision:
 
 def _format_pruning_message(spec: spack.spec.Spec, prune: bool, reasons: List[str]) -> str:
     reason_msg = ", ".join(reasons)
-    spec_fmt = "{name}{@version}{%compiler}{/hash:7}"
+    spec_fmt = "{name}{@version}{/hash:7}{%compiler}"
 
     if not prune:
         status = colorize("@*g{[x]}  ")
@@ -581,22 +622,25 @@ def copy_stage_logs_to_artifacts(job_spec: spack.spec.Spec, job_log_dir: str) ->
     tty.debug(f"job spec: {job_spec}")
 
     try:
-        pkg_cls = spack.repo.PATH.get_pkg_class(job_spec.name)
-        job_pkg = pkg_cls(job_spec)
-        tty.debug(f"job package: {job_pkg}")
-    except AssertionError:
-        msg = f"Cannot copy stage logs: job spec ({job_spec}) must be concrete"
-        tty.error(msg)
+        package_metadata_root = pathlib.Path(spack.store.STORE.layout.metadata_path(job_spec))
+    except spack.error.SpackError as e:
+        tty.error(f"Cannot copy logs: {str(e)}")
         return
 
-    stage_dir = job_pkg.stage.path
-    tty.debug(f"stage dir: {stage_dir}")
-    for file in [
-        job_pkg.log_path,
-        job_pkg.env_mods_path,
-        *spack.builder.create(job_pkg).archive_files,
-    ]:
-        copy_files_to_artifacts(file, job_log_dir)
+    # Get the package's archived files
+    archive_files = []
+    archive_root = package_metadata_root / "archived-files"
+    if archive_root.is_dir():
+        archive_files = [f for f in archive_root.rglob("*") if f.is_file()]
+    else:
+        msg = "Cannot copy package archived files: archived-files must be a directory"
+        tty.warn(msg)
+
+    build_log_zipped = package_metadata_root / "spack-build-out.txt.gz"
+    build_env_mods = package_metadata_root / "spack-build-env.txt"
+
+    for f in [build_log_zipped, build_env_mods, *archive_files]:
+        copy_files_to_artifacts(str(f), job_log_dir)
 
 
 def copy_test_logs_to_artifacts(test_stage, job_test_dir):

lib/spack/spack/cmd/__init__.py

@@ -330,7 +330,7 @@ def ensure_single_spec_or_die(spec, matching_specs):
     if len(matching_specs) <= 1:
         return
 
-    format_string = "{name}{@version}{%compiler.name}{@compiler.version}{ arch=architecture}"
+    format_string = "{name}{@version}{ arch=architecture} {%compiler.name}{@compiler.version}"
     args = ["%s matches multiple packages." % spec, "Matching packages:"]
     args += [
         colorize("  @K{%s} " % s.dag_hash(7)) + s.cformat(format_string) for s in matching_specs
@@ -471,12 +471,11 @@ def get_arg(name, default=None):
         nfmt = "{fullname}" if namespaces else "{name}"
         ffmt = ""
         if full_compiler or flags:
-            ffmt += "{%compiler.name}"
+            ffmt += "{compiler_flags} {%compiler.name}"
             if full_compiler:
                 ffmt += "{@compiler.version}"
-            ffmt += " {compiler_flags}"
         vfmt = "{variants}" if variants else ""
-        format_string = nfmt + "{@version}" + ffmt + vfmt
+        format_string = nfmt + "{@version}" + vfmt + ffmt
 
     def fmt(s, depth=0):
         """Formatter function for all output specs"""

lib/spack/spack/cmd/ci.py

@@ -4,12 +4,15 @@
 
 import json
 import os
+import re
 import shutil
+import sys
+from typing import Dict
 from urllib.parse import urlparse, urlunparse
 
 import llnl.util.filesystem as fs
-import llnl.util.tty as tty
 import llnl.util.tty.color as clr
+from llnl.util import tty
 
 import spack.binary_distribution as bindist
 import spack.ci as spack_ci
@@ -18,12 +21,22 @@
 import spack.cmd.common.arguments
 import spack.config as cfg
 import spack.environment as ev
+import spack.error
+import spack.fetch_strategy
 import spack.hash_types as ht
 import spack.mirrors.mirror
+import spack.package_base
+import spack.paths
+import spack.repo
+import spack.spec
+import spack.stage
+import spack.util.executable
+import spack.util.git
 import spack.util.gpg as gpg_util
 import spack.util.timer as timer
 import spack.util.url as url_util
 import spack.util.web as web_util
+import spack.version
 
 description = "manage continuous integration pipelines"
 section = "build"
@@ -32,6 +45,7 @@
 SPACK_COMMAND = "spack"
 INSTALL_FAIL_CODE = 1
 FAILED_CREATE_BUILDCACHE_CODE = 100
+BUILTIN = re.compile(r"var\/spack\/repos\/builtin\/packages\/([^\/]+)\/package\.py")
 
 
 def deindent(desc):
@@ -191,6 +205,16 @@ def setup_parser(subparser):
 
     reproduce.set_defaults(func=ci_reproduce)
 
+    # Verify checksums inside of ci workflows
+    verify_versions = subparsers.add_parser(
+        "verify-versions",
+        description=deindent(ci_verify_versions.__doc__),
+        help=spack.cmd.first_line(ci_verify_versions.__doc__),
+    )
+    verify_versions.add_argument("from_ref", help="git ref from which start looking at changes")
+    verify_versions.add_argument("to_ref", help="git ref to end looking at changes")
+    verify_versions.set_defaults(func=ci_verify_versions)
+
 
 def ci_generate(args):
     """generate jobs file from a CI-aware spack file
@@ -427,7 +451,7 @@ def ci_rebuild(args):
 
     # Arguments when installing the root from sources
     deps_install_args = install_args + ["--only=dependencies"]
-    root_install_args = install_args + ["--keep-stage", "--only=package"]
+    root_install_args = install_args + ["--only=package"]
 
     if cdash_handler:
         # Add additional arguments to `spack install` for CDash reporting.
@@ -464,8 +488,7 @@ def ci_rebuild(args):
                 job_spec.to_dict(hash=ht.dag_hash),
             )
 
-    # We generated the "spack install ..." command to "--keep-stage", copy
-    # any logs from the staging directory to artifacts now
+    # Copy logs and archived files from the install metadata (.spack) directory to artifacts now
     spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir)
 
     # If the installation succeeded and we're running stand-alone tests for
@@ -660,6 +683,159 @@ def _gitlab_artifacts_url(url: str) -> str:
     return urlunparse(parsed._replace(path="/".join(parts), fragment="", query=""))
 
 
+def validate_standard_versions(
+    pkg: spack.package_base.PackageBase, versions: spack.version.VersionList
+) -> bool:
+    """Get and test the checksum of a package version based on a tarball.
+    Args:
+      pkg spack.package_base.PackageBase: Spack package for which to validate a version checksum
+      versions spack.version.VersionList: list of package versions to validate
+    Returns: bool: result of the validation. True is valid and false is failed.
+    """
+    url_dict: Dict[spack.version.StandardVersion, str] = {}
+
+    for version in versions:
+        url = pkg.find_valid_url_for_version(version)
+        url_dict[version] = url
+
+    version_hashes = spack.stage.get_checksums_for_versions(
+        url_dict, pkg.name, fetch_options=pkg.fetch_options
+    )
+
+    valid_checksums = True
+    for version, sha in version_hashes.items():
+        if sha != pkg.versions[version]["sha256"]:
+            tty.error(
+                f"Invalid checksum found {pkg.name}@{version}\n"
+                f"    [package.py] {pkg.versions[version]['sha256']}\n"
+                f"    [Downloaded] {sha}"
+            )
+            valid_checksums = False
+            continue
+
+        tty.info(f"Validated {pkg.name}@{version} --> {sha}")
+
+    return valid_checksums
+
+
+def validate_git_versions(
+    pkg: spack.package_base.PackageBase, versions: spack.version.VersionList
+) -> bool:
+    """Get and test the commit and tag of a package version based on a git repository.
+    Args:
+      pkg spack.package_base.PackageBase: Spack package for which to validate a version
+      versions spack.version.VersionList: list of package versions to validate
+    Returns: bool: result of the validation. True is valid and false is failed.
+    """
+    valid_commit = True
+    for version in versions:
+        fetcher = spack.fetch_strategy.for_package_version(pkg, version)
+        with spack.stage.Stage(fetcher) as stage:
+            known_commit = pkg.versions[version]["commit"]
+            try:
+                stage.fetch()
+            except spack.error.FetchError:
+                tty.error(
+                    f"Invalid commit for {pkg.name}@{version}\n"
+                    f"    {known_commit} could not be checked out in the git repository."
+                )
+                valid_commit = False
+                continue
+
+            # Test if the specified tag matches the commit in the package.py
+            # We retrieve the commit associated with a tag and compare it to the
+            # commit that is located in the package.py file.
+            if "tag" in pkg.versions[version]:
+                tag = pkg.versions[version]["tag"]
+                try:
+                    with fs.working_dir(stage.source_path):
+                        found_commit = fetcher.git(
+                            "rev-list", "-n", "1", tag, output=str, error=str
+                        ).strip()
+                except spack.util.executable.ProcessError:
+                    tty.error(
+                        f"Invalid tag for {pkg.name}@{version}\n"
+                        f"    {tag} could not be found in the git repository."
+                    )
+                    valid_commit = False
+                    continue
+
+                if found_commit != known_commit:
+                    tty.error(
+                        f"Mismatched tag <-> commit found for {pkg.name}@{version}\n"
+                        f"    [package.py] {known_commit}\n"
+                        f"    [Downloaded] {found_commit}"
+                    )
+                    valid_commit = False
+                    continue
+
+            # If we have downloaded the repository, found the commit, and compared
+            # the tag (if specified) we can conclude that the version is pointing
+            # at what we would expect.
+            tty.info(f"Validated {pkg.name}@{version} --> {known_commit}")
+
+    return valid_commit
+
+
+def ci_verify_versions(args):
+    """validate version checksum & commits between git refs
+    This command takes a from_ref and to_ref arguments and
+    then parses the git diff between the two to determine which packages
+    have been modified verifies the new checksums inside of them.
+    """
+    with fs.working_dir(spack.paths.prefix):
+        # We use HEAD^1 explicitly on the merge commit created by
+        # GitHub Actions. However HEAD~1 is a safer default for the helper function.
+        files = spack.util.git.get_modified_files(from_ref=args.from_ref, to_ref=args.to_ref)
+
+    # Get a list of package names from the modified files.
+    pkgs = [(m.group(1), p) for p in files for m in [BUILTIN.search(p)] if m]
+
+    failed_version = False
+    for pkg_name, path in pkgs:
+        spec = spack.spec.Spec(pkg_name)
+        pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)
+
+        # Skip checking manual download packages and trust the maintainers
+        if pkg.manual_download:
+            tty.warn(f"Skipping manual download package: {pkg_name}")
+            continue
+
+        # Store versions checksums / commits for future loop
+        checksums_version_dict = {}
+        commits_version_dict = {}
+        for version in pkg.versions:
+            # If the package version defines a sha256 we'll use that as the high entropy
+            # string to detect which versions have been added between from_ref and to_ref
+            if "sha256" in pkg.versions[version]:
+                checksums_version_dict[pkg.versions[version]["sha256"]] = version
+
+            # If a package version instead defines a commit we'll use that as a
+            # high entropy string to detect new versions.
+            elif "commit" in pkg.versions[version]:
+                commits_version_dict[pkg.versions[version]["commit"]] = version
+
+            # TODO: enforce every version have a commit or a sha256 defined if not
+            # an infinite version (there are a lot of package's where this doesn't work yet.)
+
+        with fs.working_dir(spack.paths.prefix):
+            added_checksums = spack_ci.get_added_versions(
+                checksums_version_dict, path, from_ref=args.from_ref, to_ref=args.to_ref
+            )
+            added_commits = spack_ci.get_added_versions(
+                commits_version_dict, path, from_ref=args.from_ref, to_ref=args.to_ref
+            )
+
+        if added_checksums:
+            failed_version = not validate_standard_versions(pkg, added_checksums) or failed_version
+
+        if added_commits:
+            failed_version = not validate_git_versions(pkg, added_commits) or failed_version
+
+    if failed_version:
+        sys.exit(1)
+
+
 def ci(parser, args):
     if args.func:
         return args.func(args)

lib/spack/spack/cmd/config.py

@@ -350,9 +350,12 @@ def _config_change(config_path, match_spec_str=None):
                 if spack.config.get(key_path, scope=scope):
                     ideal_scope_to_modify = scope
                     break
+            # If we find our key in a specific scope, that's the one we want
+            # to modify. Otherwise we use the default write scope.
+            write_scope = ideal_scope_to_modify or spack.config.default_modify_scope()
 
             update_path = f"{key_path}:[{str(spec)}]"
-            spack.config.add(update_path, scope=ideal_scope_to_modify)
+            spack.config.add(update_path, scope=write_scope)
     else:
         raise ValueError("'config change' can currently only change 'require' sections")
 

lib/spack/spack/cmd/dependencies.py

@@ -55,7 +55,7 @@ def dependencies(parser, args):
         env = ev.active_environment()
         spec = spack.cmd.disambiguate_spec(specs[0], env)
 
-        format_string = "{name}{@version}{%compiler}{/hash:7}"
+        format_string = "{name}{@version}{/hash:7}{%compiler}"
         if sys.stdout.isatty():
             tty.msg("Dependencies of %s" % spec.format(format_string, color=True))
         deps = spack.store.STORE.db.installed_relatives(

lib/spack/spack/cmd/dependents.py

@@ -93,7 +93,7 @@ def dependents(parser, args):
         env = ev.active_environment()
         spec = spack.cmd.disambiguate_spec(specs[0], env)
 
-        format_string = "{name}{@version}{%compiler}{/hash:7}"
+        format_string = "{name}{@version}{/hash:7}{%compiler}"
         if sys.stdout.isatty():
             tty.msg("Dependents of %s" % spec.cformat(format_string))
         deps = spack.store.STORE.db.installed_relatives(spec, "parents", args.transitive)

lib/spack/spack/cmd/develop.py

@@ -3,11 +3,13 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
 import shutil
+from typing import Optional
 
 import llnl.util.tty as tty
 
 import spack.cmd
 import spack.config
+import spack.environment
 import spack.fetch_strategy
 import spack.repo
 import spack.spec
@@ -31,37 +33,33 @@ def setup_parser(subparser):
         "--no-clone",
         action="store_false",
         dest="clone",
-        default=None,
         help="do not clone, the package already exists at the source path",
     )
     clone_group.add_argument(
         "--clone",
         action="store_true",
         dest="clone",
-        default=None,
-        help="clone the package even if the path already exists",
+        default=True,
+        help=(
+            "(default) clone the package unless the path already exists, "
+            "use --force to overwrite"
+        ),
     )
 
     subparser.add_argument(
         "-f", "--force", help="remove any files or directories that block cloning source code"
     )
 
+    subparser.add_argument(
+        "-r",
+        "--recursive",
+        action="store_true",
+        help="traverse nodes of the graph to mark everything up to the root as a develop spec",
+    )
+
     arguments.add_common_arguments(subparser, ["spec"])
 
 
-def _update_config(spec, path):
-    find_fn = lambda section: spec.name in section
-
-    entry = {"spec": str(spec)}
-    if path != spec.name:
-        entry["path"] = path
-
-    def change_fn(section):
-        section[spec.name] = entry
-
-    spack.config.change_or_add("develop", find_fn, change_fn)
-
-
 def _retrieve_develop_source(spec: spack.spec.Spec, abspath: str) -> None:
     # "steal" the source code via staging API. We ask for a stage
     # to be created, then copy it afterwards somewhere else. It would be
@@ -83,44 +81,43 @@ def _retrieve_develop_source(spec: spack.spec.Spec, abspath: str) -> None:
     package.stage.steal_source(abspath)
 
 
-def develop(parser, args):
-    # Note: we could put develop specs in any scope, but I assume
-    # users would only ever want to do this for either (a) an active
-    # env or (b) a specified config file (e.g. that is included by
-    # an environment)
-    # TODO: when https://github.com/spack/spack/pull/35307 is merged,
-    # an active env is not required if a scope is specified
-    env = spack.cmd.require_active_env(cmd_name="develop")
-    if not args.spec:
-        if args.clone is False:
-            raise SpackError("No spec provided to spack develop command")
+def assure_concrete_spec(env: spack.environment.Environment, spec: spack.spec.Spec):
+    version = spec.versions.concrete_range_as_version
+    if not version:
+        # first check environment for a matching concrete spec
+        matching_specs = env.all_matching_specs(spec)
+        if matching_specs:
+            version = matching_specs[0].version
+            test_spec = spack.spec.Spec(f"{spec}@{version}")
+            for m_spec in matching_specs:
+                if not m_spec.satisfies(test_spec):
+                    raise SpackError(
+                        f"{spec.name}: has multiple concrete instances in the graph that can't be"
+                        " satisified by a single develop spec. To use `spack develop` ensure one"
+                        " of the following:"
+                        f"\n a) {spec.name} nodes can satisfy the same develop spec (minimally "
+                        "this means they all share the same version)"
+                        f"\n b) Provide a concrete develop spec ({spec.name}@[version]) to clearly"
+                        " indicate what should be developed"
+                    )
+        else:
+            # look up the maximum version so infintiy versions are preferred for develop
+            version = max(spec.package_class.versions.keys())
+            tty.msg(f"Defaulting to highest version: {spec.name}@{version}")
+    spec.versions = spack.version.VersionList([version])
 
-        # download all dev specs
-        for name, entry in env.dev_specs.items():
-            path = entry.get("path", name)
-            abspath = spack.util.path.canonicalize_path(path, default_wd=env.path)
 
-            if os.path.exists(abspath):
-                msg = "Skipping developer download of %s" % entry["spec"]
-                msg += " because its path already exists."
-                tty.msg(msg)
-                continue
+def setup_src_code(spec: spack.spec.Spec, src_path: str, clone: bool = True, force: bool = False):
+    """
+    Handle checking, cloning or overwriting source code
+    """
+    assert spec.versions
 
-            # Both old syntax `spack develop pkg@x` and new syntax `spack develop pkg@=x`
-            # are currently supported.
-            spec = spack.spec.parse_with_version_concrete(entry["spec"])
-            _retrieve_develop_source(spec, abspath)
+    if clone:
+        _clone(spec, src_path, force)
 
-        if not env.dev_specs:
-            tty.warn("No develop specs to download")
-
-        return
-
-    specs = spack.cmd.parse_specs(args.spec)
-    if len(specs) > 1:
-        raise SpackError("spack develop requires at most one named spec")
-
-    spec = specs[0]
+    if not clone and not os.path.exists(src_path):
+        raise SpackError(f"Provided path {src_path} does not exist")
 
     version = spec.versions.concrete_range_as_version
     if not version:
@@ -129,40 +126,114 @@ def develop(parser, args):
         tty.msg(f"Defaulting to highest version: {spec.name}@{version}")
     spec.versions = spack.version.VersionList([version])
 
-    # If user does not specify --path, we choose to create a directory in the
-    # active environment's directory, named after the spec
-    path = args.path or spec.name
-    if not os.path.isabs(path):
-        abspath = spack.util.path.canonicalize_path(path, default_wd=env.path)
-    else:
-        abspath = path
 
-    # clone default: only if the path doesn't exist
-    clone = args.clone
-    if clone is None:
-        clone = not os.path.exists(abspath)
+def _update_config(spec, path):
+    find_fn = lambda section: spec.name in section
 
-    if not clone and not os.path.exists(abspath):
-        raise SpackError("Provided path %s does not exist" % abspath)
+    entry = {"spec": str(spec)}
+    if path and path != spec.name:
+        entry["path"] = path
 
-    if clone:
-        if os.path.exists(abspath):
-            if args.force:
-                shutil.rmtree(abspath)
-            else:
-                msg = "Path %s already exists and cannot be cloned to." % abspath
-                msg += " Use `spack develop -f` to overwrite."
-                raise SpackError(msg)
+    def change_fn(section):
+        section[spec.name] = entry
 
-        _retrieve_develop_source(spec, abspath)
+    spack.config.change_or_add("develop", find_fn, change_fn)
+
+
+def update_env(
+    env: spack.environment.Environment,
+    spec: spack.spec.Spec,
+    specified_path: Optional[str] = None,
+    build_dir: Optional[str] = None,
+):
+    """
+    Update the spack.yaml file with additions or changes from a develop call
+    """
+    tty.debug(f"Updating develop config for {env.name} transactionally")
+
+    if not specified_path:
+        dev_entry = env.dev_specs.get(spec.name)
+        if dev_entry:
+            specified_path = dev_entry.get("path", None)
 
-    tty.debug("Updating develop config for {0} transactionally".format(env.name))
     with env.write_transaction():
-        if args.build_directory is not None:
+        if build_dir is not None:
             spack.config.add(
-                "packages:{}:package_attributes:build_directory:{}".format(
-                    spec.name, args.build_directory
-                ),
+                f"packages:{spec.name}:package_attributes:build_directory:{build_dir}",
                 env.scope_name,
             )
-        _update_config(spec, path)
+        # add develop spec and update path
+        _update_config(spec, specified_path)
+
+
+def _clone(spec: spack.spec.Spec, abspath: str, force: bool = False):
+    if os.path.exists(abspath):
+        if force:
+            shutil.rmtree(abspath)
+        else:
+            msg = f"Skipping developer download of {spec.name}"
+            msg += f" because its path {abspath} already exists."
+            tty.msg(msg)
+            return
+
+    # cloning can take a while and it's nice to get a message for the longer clones
+    tty.msg(f"Cloning source code for {spec}")
+    _retrieve_develop_source(spec, abspath)
+
+
+def _abs_code_path(
+    env: spack.environment.Environment, spec: spack.spec.Spec, path: Optional[str] = None
+):
+    src_path = path if path else spec.name
+    return spack.util.path.canonicalize_path(src_path, default_wd=env.path)
+
+
+def _dev_spec_generator(args, env):
+    """
+    Generator function to loop over all the develop specs based on how the command is called
+    If no specs are supplied then loop over the develop specs listed in the environment.
+    """
+    if not args.spec:
+        if args.clone is False:
+            raise SpackError("No spec provided to spack develop command")
+
+        for name, entry in env.dev_specs.items():
+            path = entry.get("path", name)
+            abspath = spack.util.path.canonicalize_path(path, default_wd=env.path)
+            # Both old syntax `spack develop pkg@x` and new syntax `spack develop pkg@=x`
+            # are currently supported.
+            spec = spack.spec.parse_with_version_concrete(entry["spec"])
+            yield spec, abspath
+    else:
+        specs = spack.cmd.parse_specs(args.spec)
+        if (args.path or args.build_directory) and len(specs) > 1:
+            raise SpackError(
+                "spack develop requires at most one named spec when using the --path or"
+                " --build-directory arguments"
+            )
+
+        for spec in specs:
+            if args.recursive:
+                concrete_specs = env.all_matching_specs(spec)
+                if not concrete_specs:
+                    tty.warn(
+                        f"{spec.name} has no matching concrete specs in the environment and "
+                        "will be skipped. `spack develop --recursive` requires a concretized"
+                        " environment"
+                    )
+                else:
+                    for s in concrete_specs:
+                        for node_spec in s.traverse(direction="parents", root=True):
+                            tty.debug(f"Recursive develop for {node_spec.name}")
+                            yield node_spec, _abs_code_path(env, node_spec, args.path)
+            else:
+                yield spec, _abs_code_path(env, spec, args.path)
+
+
+def develop(parser, args):
+    env = spack.cmd.require_active_env(cmd_name="develop")
+
+    for spec, abspath in _dev_spec_generator(args, env):
+        assure_concrete_spec(env, spec)
+        setup_src_code(spec, abspath, clone=args.clone, force=args.force)
+        update_env(env, spec, args.path, args.build_directory)

lib/spack/spack/cmd/help.py

@@ -73,7 +73,7 @@
       boxlib @B{dim=2}                  boxlib built for 2 dimensions
       libdwarf @g{%intel} ^libelf@g{%gcc}
           libdwarf, built with intel compiler, linked to libelf built with gcc
-      mvapich2 @g{%gcc} @B{fabrics=psm,mrail,sock}
+      mvapich2 @B{fabrics=psm,mrail,sock} @g{%gcc}
           mvapich2, built with gcc compiler, with support for multiple fabrics
 """
 

lib/spack/spack/cmd/modules/__init__.py

@@ -383,8 +383,10 @@ def modules_cmd(parser, args, module_type, callbacks=callbacks):
         query = " ".join(str(s) for s in args.constraint_specs)
         msg = f"the constraint '{query}' matches multiple packages:\n"
         for s in specs:
-            spec_fmt = "{hash:7} {name}{@version}{%compiler}"
-            spec_fmt += "{compiler_flags}{variants}{arch=architecture}"
+            spec_fmt = (
+                "{hash:7} {name}{@version}{compiler_flags}{variants}"
+                "{arch=architecture} {%compiler}"
+            )
             msg += "\t" + s.cformat(spec_fmt) + "\n"
         tty.die(msg, "In this context exactly *one* match is needed.")
 

lib/spack/spack/cmd/style.py

@@ -6,8 +6,9 @@
 import os
 import re
 import sys
+import warnings
 from itertools import islice, zip_longest
-from typing import Dict, List, Optional
+from typing import Callable, Dict, List, Optional
 
 import llnl.util.tty as tty
 import llnl.util.tty.color as color
@@ -16,6 +17,9 @@
 import spack.paths
 import spack.repo
 import spack.util.git
+import spack.util.spack_yaml
+from spack.spec_parser import SPEC_TOKENIZER, SpecTokens
+from spack.tokenize import Token
 from spack.util.executable import Executable, which
 
 description = "runs source code style checks on spack"
@@ -198,6 +202,13 @@ def setup_parser(subparser):
         action="append",
         help="specify tools to skip (choose from %s)" % ", ".join(tool_names),
     )
+    subparser.add_argument(
+        "--spec-strings",
+        action="store_true",
+        help="upgrade spec strings in Python, JSON and YAML files for compatibility with Spack "
+        "v1.0 and v0.x. Example: spack style --spec-strings $(git ls-files). Note: this flag "
+        "will be removed in Spack v1.0.",
+    )
 
     subparser.add_argument("files", nargs=argparse.REMAINDER, help="specific files to check")
 
@@ -507,7 +518,196 @@ def _bootstrap_dev_dependencies():
         spack.bootstrap.ensure_environment_dependencies()
 
 
+IS_PROBABLY_COMPILER = re.compile(r"%[a-zA-Z_][a-zA-Z0-9\-]")
+
+
+def _spec_str_reorder_compiler(idx: int, blocks: List[List[Token]]) -> None:
+    # only move the compiler to the back if it exists and is not already at the end
+    if not 0 <= idx < len(blocks) - 1:
+        return
+    # if there's only whitespace after the compiler, don't move it
+    if all(token.kind == SpecTokens.WS for block in blocks[idx + 1 :] for token in block):
+        return
+    # rotate left and always add at least one WS token between compiler and previous token
+    compiler_block = blocks.pop(idx)
+    if compiler_block[0].kind != SpecTokens.WS:
+        compiler_block.insert(0, Token(SpecTokens.WS, " "))
+    # delete the WS tokens from the new first block if it was at the very start, to prevent leading
+    # WS tokens.
+    while idx == 0 and blocks[0][0].kind == SpecTokens.WS:
+        blocks[0].pop(0)
+    blocks.append(compiler_block)
+
+
+def _spec_str_format(spec_str: str) -> Optional[str]:
+    """Given any string, try to parse as spec string, and rotate the compiler token to the end
+    of each spec instance. Returns the formatted string if it was changed, otherwise None."""
+    # We parse blocks of tokens that include leading whitespace, and move the compiler block to
+    # the end when we hit a dependency ^... or the end of a string.
+    # [@3.1][ +foo][ +bar][ %gcc@3.1][ +baz]
+    # [@3.1][ +foo][ +bar][ +baz][ %gcc@3.1]
+
+    current_block: List[Token] = []
+    blocks: List[List[Token]] = []
+    compiler_block_idx = -1
+    in_edge_attr = False
+
+    for token in SPEC_TOKENIZER.tokenize(spec_str):
+        if token.kind == SpecTokens.UNEXPECTED:
+            # parsing error, we cannot fix this string.
+            return None
+        elif token.kind in (SpecTokens.COMPILER, SpecTokens.COMPILER_AND_VERSION):
+            # multiple compilers are not supported in Spack v0.x, so early return
+            if compiler_block_idx != -1:
+                return None
+            current_block.append(token)
+            blocks.append(current_block)
+            current_block = []
+            compiler_block_idx = len(blocks) - 1
+        elif token.kind in (
+            SpecTokens.START_EDGE_PROPERTIES,
+            SpecTokens.DEPENDENCY,
+            SpecTokens.UNQUALIFIED_PACKAGE_NAME,
+            SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME,
+        ):
+            _spec_str_reorder_compiler(compiler_block_idx, blocks)
+            compiler_block_idx = -1
+            if token.kind == SpecTokens.START_EDGE_PROPERTIES:
+                in_edge_attr = True
+            current_block.append(token)
+            blocks.append(current_block)
+            current_block = []
+        elif token.kind == SpecTokens.END_EDGE_PROPERTIES:
+            in_edge_attr = False
+            current_block.append(token)
+            blocks.append(current_block)
+            current_block = []
+        elif in_edge_attr:
+            current_block.append(token)
+        elif token.kind in (
+            SpecTokens.VERSION_HASH_PAIR,
+            SpecTokens.GIT_VERSION,
+            SpecTokens.VERSION,
+            SpecTokens.PROPAGATED_BOOL_VARIANT,
+            SpecTokens.BOOL_VARIANT,
+            SpecTokens.PROPAGATED_KEY_VALUE_PAIR,
+            SpecTokens.KEY_VALUE_PAIR,
+            SpecTokens.DAG_HASH,
+        ):
+            current_block.append(token)
+            blocks.append(current_block)
+            current_block = []
+        elif token.kind == SpecTokens.WS:
+            current_block.append(token)
+        else:
+            raise ValueError(f"unexpected token {token}")
+
+    if current_block:
+        blocks.append(current_block)
+    _spec_str_reorder_compiler(compiler_block_idx, blocks)
+
+    new_spec_str = "".join(token.value for block in blocks for token in block)
+    return new_spec_str if spec_str != new_spec_str else None
+
+
+SpecStrHandler = Callable[[str, int, int, str, str], None]
+
+
+def _spec_str_default_handler(path: str, line: int, col: int, old: str, new: str):
+    """A SpecStrHandler that prints formatted spec strings and their locations."""
+    print(f"{path}:{line}:{col}: `{old}` -> `{new}`")
+
+
+def _spec_str_fix_handler(path: str, line: int, col: int, old: str, new: str):
+    """A SpecStrHandler that updates formatted spec strings in files."""
+    with open(path, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+    new_line = lines[line - 1].replace(old, new)
+    if new_line == lines[line - 1]:
+        tty.warn(f"{path}:{line}:{col}: could not apply fix: `{old}` -> `{new}`")
+        return
+    lines[line - 1] = new_line
+    print(f"{path}:{line}:{col}: fixed `{old}` -> `{new}`")
+    with open(path, "w", encoding="utf-8") as f:
+        f.writelines(lines)
+
+
+def _spec_str_ast(path: str, tree: ast.AST, handler: SpecStrHandler) -> None:
+    """Walk the AST of a Python file and apply handler to formatted spec strings."""
+    has_constant = sys.version_info >= (3, 8)
+    for node in ast.walk(tree):
+        if has_constant and isinstance(node, ast.Constant) and isinstance(node.value, str):
+            current_str = node.value
+        elif not has_constant and isinstance(node, ast.Str):
+            current_str = node.s
+        else:
+            continue
+        if not IS_PROBABLY_COMPILER.search(current_str):
+            continue
+        new = _spec_str_format(current_str)
+        if new is not None:
+            handler(path, node.lineno, node.col_offset, current_str, new)
+
+
+def _spec_str_json_and_yaml(path: str, data: dict, handler: SpecStrHandler) -> None:
+    """Walk a YAML or JSON data structure and apply handler to formatted spec strings."""
+    queue = [data]
+    seen = set()
+
+    while queue:
+        current = queue.pop(0)
+        if id(current) in seen:
+            continue
+        seen.add(id(current))
+        if isinstance(current, dict):
+            queue.extend(current.values())
+            queue.extend(current.keys())
+        elif isinstance(current, list):
+            queue.extend(current)
+        elif isinstance(current, str) and IS_PROBABLY_COMPILER.search(current):
+            new = _spec_str_format(current)
+            if new is not None:
+                mark = getattr(current, "_start_mark", None)
+                if mark:
+                    line, col = mark.line + 1, mark.column + 1
+                else:
+                    line, col = 0, 0
+                handler(path, line, col, current, new)
+
+
+def _check_spec_strings(
+    paths: List[str], handler: SpecStrHandler = _spec_str_default_handler
+) -> None:
+    """Open Python, JSON and YAML files, and format their string literals that look like spec
+    strings. A handler is called for each formatting, which can be used to print or apply fixes."""
+    for path in paths:
+        is_json_or_yaml = path.endswith(".json") or path.endswith(".yaml") or path.endswith(".yml")
+        is_python = path.endswith(".py")
+        if not is_json_or_yaml and not is_python:
+            continue
+
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                # skip files that are likely too large to be user code or config
+                if os.fstat(f.fileno()).st_size > 1024 * 1024:
+                    warnings.warn(f"skipping {path}: too large.")
+                    continue
+                if is_json_or_yaml:
+                    _spec_str_json_and_yaml(path, spack.util.spack_yaml.load_config(f), handler)
+                elif is_python:
+                    _spec_str_ast(path, ast.parse(f.read()), handler)
+        except (OSError, spack.util.spack_yaml.SpackYAMLError, SyntaxError, ValueError):
+            warnings.warn(f"skipping {path}")
+            continue
+
+
 def style(parser, args):
+    if args.spec_strings:
+        if not args.files:
+            tty.die("No files provided to check spec strings.")
+        handler = _spec_str_fix_handler if args.fix else _spec_str_default_handler
+        return _check_spec_strings(args.files, handler)
+
     # save initial working directory for relativizing paths later
     args.initial_working_dir = os.getcwd()
 

lib/spack/spack/cmd/unit_test.py

@@ -17,6 +17,7 @@
     pytest = None  # type: ignore
 
 import llnl.util.filesystem
+import llnl.util.tty as tty
 import llnl.util.tty.color as color
 from llnl.util.tty.colify import colify
 
@@ -236,6 +237,12 @@ def unit_test(parser, args, unknown_args):
         pytest_root = spack.extensions.load_extension(args.extension)
 
     if args.numprocesses is not None and args.numprocesses > 1:
+        try:
+            import xdist  # noqa: F401
+        except ImportError:
+            tty.error("parallel unit-test requires pytest-xdist module")
+            return 1
+
         pytest_args.extend(
             [
                 "--dist",

lib/spack/spack/config.py

@@ -32,9 +32,10 @@
 import copy
 import functools
 import os
+import os.path
 import re
 import sys
-from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, Generator, List, NamedTuple, Optional, Tuple, Union
 
 import jsonschema
 
@@ -42,7 +43,6 @@
 
 import spack.error
 import spack.paths
-import spack.platforms
 import spack.schema
 import spack.schema.bootstrap
 import spack.schema.cdash
@@ -54,17 +54,18 @@
 import spack.schema.develop
 import spack.schema.env
 import spack.schema.env_vars
+import spack.schema.include
+import spack.schema.merged
 import spack.schema.mirrors
 import spack.schema.modules
 import spack.schema.packages
 import spack.schema.repos
 import spack.schema.upstreams
 import spack.schema.view
-
-# Hacked yaml for configuration files preserves line numbers.
+import spack.util.remote_file_cache as rfc_util
 import spack.util.spack_yaml as syaml
-import spack.util.web as web_util
 from spack.util.cpus import cpus_available
+from spack.util.spack_yaml import get_mark_from_yaml_data
 
 from .enums import ConfigScopePriority
 
@@ -74,6 +75,7 @@
     "concretizer": spack.schema.concretizer.schema,
     "definitions": spack.schema.definitions.schema,
     "env_vars": spack.schema.env_vars.schema,
+    "include": spack.schema.include.schema,
     "view": spack.schema.view.schema,
     "develop": spack.schema.develop.schema,
     "mirrors": spack.schema.mirrors.schema,
@@ -121,6 +123,17 @@
 #: Type used for raw YAML configuration
 YamlConfigDict = Dict[str, Any]
 
+#: prefix for name of included configuration scopes
+INCLUDE_SCOPE_PREFIX = "include"
+
+#: safeguard for recursive includes -- maximum include depth
+MAX_RECURSIVE_INCLUDES = 100
+
+
+def _include_cache_location():
+    """Location to cache included configuration files."""
+    return os.path.join(spack.paths.user_cache_path, "includes")
+
 
 class ConfigScope:
     def __init__(self, name: str) -> None:
@@ -128,6 +141,25 @@ def __init__(self, name: str) -> None:
         self.writable = False
         self.sections = syaml.syaml_dict()
 
+        #: names of any included scopes
+        self._included_scopes: Optional[List["ConfigScope"]] = None
+
+    @property
+    def included_scopes(self) -> List["ConfigScope"]:
+        """Memoized list of included scopes, in the order they appear in this scope."""
+        if self._included_scopes is None:
+            self._included_scopes = []
+
+            includes = self.get_section("include")
+            if includes:
+                include_paths = [included_path(data) for data in includes["include"]]
+                for path in include_paths:
+                    included_scope = include_path_scope(path)
+                    if included_scope:
+                        self._included_scopes.append(included_scope)
+
+        return self._included_scopes
+
     def get_section_filename(self, section: str) -> str:
         raise NotImplementedError
 
@@ -433,7 +465,9 @@ def highest(self) -> ConfigScope:
         return next(self.scopes.reversed_values())  # type: ignore
 
     @_config_mutator
-    def push_scope(self, scope: ConfigScope, priority: Optional[int] = None) -> None:
+    def push_scope(
+        self, scope: ConfigScope, priority: Optional[int] = None, _depth: int = 0
+    ) -> None:
         """Adds a scope to the Configuration, at a given priority.
 
         If a priority is not given, it is assumed to be the current highest priority.
@@ -442,18 +476,44 @@ def push_scope(self, scope: ConfigScope, priority: Optional[int] = None) -> None
             scope: scope to be added
             priority: priority of the scope
         """
+        # TODO: As a follow on to #48784, change this to create a graph of the
+        # TODO: includes AND ensure properly sorted such that the order included
+        # TODO: at the highest level is reflected in the value of an option that
+        # TODO: is set in multiple included files.
+        # before pushing the scope itself, push any included scopes recursively, at same priority
+        for included_scope in reversed(scope.included_scopes):
+            if _depth + 1 > MAX_RECURSIVE_INCLUDES:  # make sure we're not recursing endlessly
+                mark = ""
+                if hasattr(included_scope, "path") and syaml.marked(included_scope.path):
+                    mark = included_scope.path._start_mark  # type: ignore
+                raise RecursiveIncludeError(
+                    f"Maximum include recursion exceeded in {included_scope.name}", str(mark)
+                )
+
+            # record this inclusion so that remove_scope() can use it
+            self.push_scope(included_scope, priority=priority, _depth=_depth + 1)
+
         tty.debug(f"[CONFIGURATION: PUSH SCOPE]: {str(scope)}, priority={priority}", level=2)
         self.scopes.add(scope.name, value=scope, priority=priority)
 
     @_config_mutator
     def remove_scope(self, scope_name: str) -> Optional[ConfigScope]:
         """Removes a scope by name, and returns it. If the scope does not exist, returns None."""
+
         try:
             scope = self.scopes.remove(scope_name)
-            tty.debug(f"[CONFIGURATION: POP SCOPE]: {str(scope)}", level=2)
+            tty.debug(f"[CONFIGURATION: REMOVE SCOPE]: {str(scope)}", level=2)
         except KeyError as e:
-            tty.debug(f"[CONFIGURATION: POP SCOPE]: {e}", level=2)
+            tty.debug(f"[CONFIGURATION: REMOVE SCOPE]: {e}", level=2)
             return None
+
+        # transitively remove included scopes
+        for included_scope in scope.included_scopes:
+            assert (
+                included_scope.name in self.scopes
+            ), f"Included scope '{included_scope.name}' was never added to configuration!"
+            self.remove_scope(included_scope.name)
+
         return scope
 
     @property
@@ -763,6 +823,8 @@ def _add_platform_scope(
     cfg: Configuration, name: str, path: str, priority: ConfigScopePriority, writable: bool = True
 ) -> None:
     """Add a platform-specific subdirectory for the current platform."""
+    import spack.platforms  # circular dependency
+
     platform = spack.platforms.host().name
     scope = DirectoryConfigScope(
         f"{name}/{platform}", os.path.join(path, platform), writable=writable
@@ -770,6 +832,75 @@ def _add_platform_scope(
     cfg.push_scope(scope, priority=priority)
 
 
+#: Class for the relevance of an optional path conditioned on a limited
+#: python code that evaluates to a boolean and or explicit specification
+#: as optional.
+class IncludePath(NamedTuple):
+    path: str
+    when: str
+    sha256: str
+    optional: bool
+
+
+def included_path(entry: Union[str, dict]) -> IncludePath:
+    """Convert the included path entry into an IncludePath.
+
+    Args:
+        entry: include configuration entry
+
+    Returns: converted entry, where an empty ``when`` means the path is
+        not conditionally included
+    """
+    if isinstance(entry, str):
+        return IncludePath(path=entry, sha256="", when="", optional=False)
+
+    path = entry["path"]
+    sha256 = entry.get("sha256", "")
+    when = entry.get("when", "")
+    optional = entry.get("optional", False)
+    return IncludePath(path=path, sha256=sha256, when=when, optional=optional)
+
+
+def include_path_scope(include: IncludePath) -> Optional[ConfigScope]:
+    """Instantiate an appropriate configuration scope for the given path.
+
+    Args:
+        include: optional include path
+
+    Returns: configuration scope
+
+    Raises:
+        ValueError: included path has an unsupported URL scheme, is required
+            but does not exist; configuration stage directory argument is missing
+        ConfigFileError: unable to access remote configuration file(s)
+    """
+    # circular dependencies
+    import spack.spec
+
+    if (not include.when) or spack.spec.eval_conditional(include.when):
+        config_path = rfc_util.local_path(include.path, include.sha256, _include_cache_location)
+        if not config_path:
+            raise ConfigFileError(f"Unable to fetch remote configuration from {include.path}")
+
+        if os.path.isdir(config_path):
+            # directories are treated as regular ConfigScopes
+            config_name = f"{INCLUDE_SCOPE_PREFIX}:{os.path.basename(config_path)}"
+            tty.debug(f"Creating DirectoryConfigScope {config_name} for '{config_path}'")
+            return DirectoryConfigScope(config_name, config_path)
+
+        if os.path.exists(config_path):
+            # files are assumed to be SingleFileScopes
+            config_name = f"{INCLUDE_SCOPE_PREFIX}:{config_path}"
+            tty.debug(f"Creating SingleFileScope {config_name} for '{config_path}'")
+            return SingleFileScope(config_name, config_path, spack.schema.merged.schema)
+
+        if not include.optional:
+            path = f" at ({config_path})" if config_path != include.path else ""
+            raise ValueError(f"Required path ({include.path}) does not exist{path}")
+
+    return None
+
+
 def config_paths_from_entry_points() -> List[Tuple[str, str]]:
     """Load configuration paths from entry points
 
@@ -795,7 +926,7 @@ def config_paths_from_entry_points() -> List[Tuple[str, str]]:
     return config_paths
 
 
-def create() -> Configuration:
+def create_incremental() -> Generator[Configuration, None, None]:
     """Singleton Configuration instance.
 
     This constructs one instance associated with this module and returns
@@ -839,11 +970,25 @@ def create() -> Configuration:
         # Each scope can have per-platform overrides in subdirectories
         _add_platform_scope(cfg, name, path, priority=ConfigScopePriority.CONFIG_FILES)
 
-    return cfg
+        # yield the config incrementally so that each config level's init code can get
+        # data from the one below. This can be tricky, but it enables us to have a
+        # single unified config system.
+        #
+        # TODO: think about whether we want to restrict what types of config can be used
+        #     at each level. e.g., we may want to just more forcibly disallow remote
+        #     config (which uses ssl and other config options) for some of the scopes,
+        #     to make the bootstrap issues more explicit, even if allowing config scope
+        #     init to reference lower scopes is more flexible.
+        yield cfg
+
+
+def create() -> Configuration:
+    """Create a configuration using create_incremental(), return the last yielded result."""
+    return list(create_incremental())[-1]
 
 
 #: This is the singleton configuration instance for Spack.
-CONFIG: Configuration = lang.Singleton(create)  # type: ignore
+CONFIG: Configuration = lang.Singleton(create_incremental)  # type: ignore
 
 
 def add_from_file(filename: str, scope: Optional[str] = None) -> None:
@@ -939,7 +1084,8 @@ def set(path: str, value: Any, scope: Optional[str] = None) -> None:
 
     Accepts the path syntax described in ``get()``.
     """
-    return CONFIG.set(path, value, scope)
+    result = CONFIG.set(path, value, scope)
+    return result
 
 
 def scopes() -> lang.PriorityOrderedMapping[str, ConfigScope]:
@@ -1462,120 +1608,6 @@ def create_from(*scopes_or_paths: Union[ScopeWithOptionalPriority, str]) -> Conf
     return result
 
 
-def raw_github_gitlab_url(url: str) -> str:
-    """Transform a github URL to the raw form to avoid undesirable html.
-
-    Args:
-        url: url to be converted to raw form
-
-    Returns:
-        Raw github/gitlab url or the original url
-    """
-    # Note we rely on GitHub to redirect the 'raw' URL returned here to the
-    # actual URL under https://raw.githubusercontent.com/ with '/blob'
-    # removed and or, '/blame' if needed.
-    if "github" in url or "gitlab" in url:
-        return url.replace("/blob/", "/raw/")
-
-    return url
-
-
-def collect_urls(base_url: str) -> list:
-    """Return a list of configuration URLs.
-
-    Arguments:
-        base_url: URL for a configuration (yaml) file or a directory
-            containing yaml file(s)
-
-    Returns:
-        List of configuration file(s) or empty list if none
-    """
-    if not base_url:
-        return []
-
-    extension = ".yaml"
-
-    if base_url.endswith(extension):
-        return [base_url]
-
-    # Collect configuration URLs if the base_url is a "directory".
-    _, links = web_util.spider(base_url, 0)
-    return [link for link in links if link.endswith(extension)]
-
-
-def fetch_remote_configs(url: str, dest_dir: str, skip_existing: bool = True) -> str:
-    """Retrieve configuration file(s) at the specified URL.
-
-    Arguments:
-        url: URL for a configuration (yaml) file or a directory containing
-            yaml file(s)
-        dest_dir: destination directory
-        skip_existing: Skip files that already exist in dest_dir if
-            ``True``; otherwise, replace those files
-
-    Returns:
-        Path to the corresponding file if URL is or contains a
-        single file and it is the only file in the destination directory or
-        the root (dest_dir) directory if multiple configuration files exist
-        or are retrieved.
-    """
-
-    def _fetch_file(url):
-        raw = raw_github_gitlab_url(url)
-        tty.debug(f"Reading config from url {raw}")
-        return web_util.fetch_url_text(raw, dest_dir=dest_dir)
-
-    if not url:
-        raise ConfigFileError("Cannot retrieve configuration without a URL")
-
-    # Return the local path to the cached configuration file OR to the
-    # directory containing the cached configuration files.
-    config_links = collect_urls(url)
-    existing_files = os.listdir(dest_dir) if os.path.isdir(dest_dir) else []
-
-    paths = []
-    for config_url in config_links:
-        basename = os.path.basename(config_url)
-        if skip_existing and basename in existing_files:
-            tty.warn(
-                f"Will not fetch configuration from {config_url} since a "
-                f"version already exists in {dest_dir}"
-            )
-            path = os.path.join(dest_dir, basename)
-        else:
-            path = _fetch_file(config_url)
-
-        if path:
-            paths.append(path)
-
-    if paths:
-        return dest_dir if len(paths) > 1 else paths[0]
-
-    raise ConfigFileError(f"Cannot retrieve configuration (yaml) from {url}")
-
-
-def get_mark_from_yaml_data(obj):
-    """Try to get ``spack.util.spack_yaml`` mark from YAML data.
-
-    We try the object, and if that fails we try its first member (if it's a container).
-
-    Returns:
-        mark if one is found, otherwise None.
-    """
-    # mark of object itelf
-    mark = getattr(obj, "_start_mark", None)
-    if mark:
-        return mark
-
-    # mark of first member if it is a container
-    if isinstance(obj, (list, dict)):
-        first_member = next(iter(obj), None)
-        if first_member:
-            mark = getattr(first_member, "_start_mark", None)
-
-    return mark
-
-
 def determine_number_of_jobs(
     *,
     parallel: bool = False,
@@ -1680,3 +1712,7 @@ def get_path(path, data):
 
         # give up and return None if nothing worked
         return None
+
+
+class RecursiveIncludeError(spack.error.SpackError):
+    """Too many levels of recursive includes."""

lib/spack/spack/database.py

@@ -1126,7 +1126,7 @@ def _add(
             installation_time:
                 Date and time of installation
             allow_missing: if True, don't warn when installation is not found on on disk
-                This is useful when installing specs without build deps.
+                This is useful when installing specs without build/test deps.
         """
         if not spec.concrete:
             raise NonConcreteSpecAddError("Specs added to DB must be concrete.")
@@ -1146,10 +1146,8 @@ def _add(
                 edge.spec,
                 explicit=False,
                 installation_time=installation_time,
-                # allow missing build-only deps. This prevents excessive warnings when a spec is
-                # installed, and its build dep is missing a build dep; there's no need to install
-                # the build dep's build dep first, and there's no need to warn about it missing.
-                allow_missing=allow_missing or edge.depflag == dt.BUILD,
+                # allow missing build / test only deps
+                allow_missing=allow_missing or edge.depflag & (dt.BUILD | dt.TEST) == edge.depflag,
             )
 
         # Make sure the directory layout agrees whether the spec is installed

lib/spack/spack/detection/path.py

@@ -7,6 +7,7 @@
 import collections
 import concurrent.futures
 import os
+import pathlib
 import re
 import sys
 import traceback
@@ -15,6 +16,7 @@
 
 import llnl.util.filesystem
 import llnl.util.lang
+import llnl.util.symlink
 import llnl.util.tty
 
 import spack.error
@@ -70,13 +72,21 @@ def dedupe_paths(paths: List[str]) -> List[str]:
     """Deduplicate paths based on inode and device number. In case the list contains first a
     symlink and then the directory it points to, the symlink is replaced with the directory path.
     This ensures that we pick for example ``/usr/bin`` over ``/bin`` if the latter is a symlink to
-    the former`."""
+    the former."""
     seen: Dict[Tuple[int, int], str] = {}
+
+    linked_parent_check = lambda x: any(
+        [llnl.util.symlink.islink(str(y)) for y in pathlib.Path(x).parents]
+    )
+
     for path in paths:
         identifier = file_identifier(path)
         if identifier not in seen:
             seen[identifier] = path
-        elif not os.path.islink(path):
+        # we also want to deprioritize paths if they contain a symlink in any parent
+        # (not just the basedir): e.g. oneapi has "latest/bin",
+        # where "latest" is a symlink to 2025.0"
+        elif not (llnl.util.symlink.islink(path) or linked_parent_check(path)):
             seen[identifier] = path
     return list(seen.values())
 

lib/spack/spack/directives.py

@@ -462,8 +462,7 @@ def _execute_extends(pkg):
         if dep_spec.name == "python" and not pkg.name == "python-venv":
             _depends_on(pkg, spack.spec.Spec("python-venv"), when=when, type=("build", "run"))
 
-        # TODO: the values of the extendees dictionary are not used. Remove in next refactor.
-        pkg.extendees[dep_spec.name] = (dep_spec, None)
+        pkg.extendees[dep_spec.name] = (dep_spec, when_spec)
 
     return _execute_extends
 

lib/spack/spack/environment/depfile.py

@@ -166,7 +166,7 @@ def __init__(
                 " ".join(self._install_target(s.safe_name()) for s in item.prereqs),
                 item.target.spec_hash(),
                 item.target.unsafe_format(
-                    "{name}{@version}{%compiler}{variants}{arch=architecture}"
+                    "{name}{@version}{variants}{ arch=architecture} {%compiler}"
                 ),
                 item.buildcache_flag,
             )

lib/spack/spack/environment/environment.py

@@ -10,8 +10,6 @@
 import re
 import shutil
 import stat
-import urllib.parse
-import urllib.request
 import warnings
 from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union
 
@@ -32,7 +30,6 @@
 import spack.paths
 import spack.repo
 import spack.schema.env
-import spack.schema.merged
 import spack.spec
 import spack.spec_list
 import spack.store
@@ -43,7 +40,6 @@
 import spack.util.path
 import spack.util.spack_json as sjson
 import spack.util.spack_yaml as syaml
-import spack.util.url
 from spack import traverse
 from spack.installer import PackageInstaller
 from spack.schema.env import TOP_LEVEL_KEY
@@ -577,13 +573,6 @@ def _write_yaml(data, str_or_file):
     syaml.dump_config(data, str_or_file, default_flow_style=False)
 
 
-def _eval_conditional(string):
-    """Evaluate conditional definitions using restricted variable scope."""
-    valid_variables = spack.spec.get_host_environment()
-    valid_variables.update({"re": re, "env": os.environ})
-    return eval(string, valid_variables)
-
-
 def _is_dev_spec_and_has_changed(spec):
     """Check if the passed spec is a dev build and whether it has changed since the
     last installation"""
@@ -1016,7 +1005,7 @@ def _process_definition(self, entry):
         """Process a single spec definition item."""
         when_string = entry.get("when")
         if when_string is not None:
-            when = _eval_conditional(when_string)
+            when = spack.spec.eval_conditional(when_string)
             assert len([x for x in entry if x != "when"]) == 1
         else:
             when = True
@@ -1139,11 +1128,6 @@ def user_specs(self):
 
     @property
     def dev_specs(self):
-        if not self._dev_specs:
-            self._dev_specs = self._read_dev_specs()
-        return self._dev_specs
-
-    def _read_dev_specs(self):
         dev_specs = {}
         dev_config = spack.config.get("develop", {})
         for name, entry in dev_config.items():
@@ -1561,9 +1545,6 @@ def _get_specs_to_concretize(
         return new_user_specs, kept_user_specs, specs_to_concretize
 
     def _concretize_together_where_possible(self, tests: bool = False) -> Sequence[SpecPair]:
-        # Avoid cyclic dependency
-        import spack.solver.asp
-
         # Exit early if the set of concretized specs is the set of user specs
         new_user_specs, _, specs_to_concretize = self._get_specs_to_concretize()
         if not new_user_specs:
@@ -2674,20 +2655,23 @@ def _ensure_env_dir():
         # error handling for bad manifests is handled on other code paths
         return
 
+    # TODO: make this recursive
     includes = manifest[TOP_LEVEL_KEY].get("include", [])
     for include in includes:
-        if os.path.isabs(include):
+        included_path = spack.config.included_path(include)
+        path = included_path.path
+        if os.path.isabs(path):
             continue
 
-        abspath = pathlib.Path(os.path.normpath(environment_dir / include))
+        abspath = pathlib.Path(os.path.normpath(environment_dir / path))
         common_path = pathlib.Path(os.path.commonpath([environment_dir, abspath]))
         if common_path != environment_dir:
-            tty.debug(f"Will not copy relative include from outside environment: {include}")
+            tty.debug(f"Will not copy relative include file from outside environment: {path}")
             continue
 
-        orig_abspath = os.path.normpath(envfile.parent / include)
+        orig_abspath = os.path.normpath(envfile.parent / path)
         if not os.path.exists(orig_abspath):
-            tty.warn(f"Included file does not exist; will not copy: '{include}'")
+            tty.warn(f"Included file does not exist; will not copy: '{path}'")
             continue
 
         fs.touchp(abspath)
@@ -2910,7 +2894,7 @@ def extract_name(_item):
                 continue
 
             condition_str = item.get("when", "True")
-            if not _eval_conditional(condition_str):
+            if not spack.spec.eval_conditional(condition_str):
                 continue
 
             yield idx, item
@@ -2971,127 +2955,20 @@ def __iter__(self):
     def __str__(self):
         return str(self.manifest_file)
 
-    @property
-    def included_config_scopes(self) -> List[spack.config.ConfigScope]:
-        """List of included configuration scopes from the manifest.
-
-        Scopes are listed in the YAML file in order from highest to
-        lowest precedence, so configuration from earlier scope will take
-        precedence over later ones.
-
-        This routine returns them in the order they should be pushed onto
-        the internal scope stack (so, in reverse, from lowest to highest).
-
-        Returns: Configuration scopes associated with the environment manifest
-
-        Raises:
-            SpackEnvironmentError: if the manifest includes a remote file but
-                no configuration stage directory has been identified
-        """
-        scopes: List[spack.config.ConfigScope] = []
-
-        # load config scopes added via 'include:', in reverse so that
-        # highest-precedence scopes are last.
-        includes = self[TOP_LEVEL_KEY].get("include", [])
-        missing = []
-        for i, config_path in enumerate(reversed(includes)):
-            # allow paths to contain spack config/environment variables, etc.
-            config_path = substitute_path_variables(config_path)
-            include_url = urllib.parse.urlparse(config_path)
-
-            # If scheme is not valid, config_path is not a url
-            # of a type Spack is generally aware
-            if spack.util.url.validate_scheme(include_url.scheme):
-                # Transform file:// URLs to direct includes.
-                if include_url.scheme == "file":
-                    config_path = urllib.request.url2pathname(include_url.path)
-
-                # Any other URL should be fetched.
-                elif include_url.scheme in ("http", "https", "ftp"):
-                    # Stage any remote configuration file(s)
-                    staged_configs = (
-                        os.listdir(self.config_stage_dir)
-                        if os.path.exists(self.config_stage_dir)
-                        else []
-                    )
-                    remote_path = urllib.request.url2pathname(include_url.path)
-                    basename = os.path.basename(remote_path)
-                    if basename in staged_configs:
-                        # Do NOT re-stage configuration files over existing
-                        # ones with the same name since there is a risk of
-                        # losing changes (e.g., from 'spack config update').
-                        tty.warn(
-                            "Will not re-stage configuration from {0} to avoid "
-                            "losing changes to the already staged file of the "
-                            "same name.".format(remote_path)
-                        )
-
-                        # Recognize the configuration stage directory
-                        # is flattened to ensure a single copy of each
-                        # configuration file.
-                        config_path = self.config_stage_dir
-                        if basename.endswith(".yaml"):
-                            config_path = os.path.join(config_path, basename)
-                    else:
-                        staged_path = spack.config.fetch_remote_configs(
-                            config_path, str(self.config_stage_dir), skip_existing=True
-                        )
-                        if not staged_path:
-                            raise SpackEnvironmentError(
-                                "Unable to fetch remote configuration {0}".format(config_path)
-                            )
-                        config_path = staged_path
-
-                elif include_url.scheme:
-                    raise ValueError(
-                        f"Unsupported URL scheme ({include_url.scheme}) for "
-                        f"environment include: {config_path}"
-                    )
-
-            # treat relative paths as relative to the environment
-            if not os.path.isabs(config_path):
-                config_path = os.path.join(self.manifest_dir, config_path)
-                config_path = os.path.normpath(os.path.realpath(config_path))
-
-            if os.path.isdir(config_path):
-                # directories are treated as regular ConfigScopes
-                config_name = f"env:{self.name}:{os.path.basename(config_path)}"
-                tty.debug(f"Creating DirectoryConfigScope {config_name} for '{config_path}'")
-                scopes.append(spack.config.DirectoryConfigScope(config_name, config_path))
-            elif os.path.exists(config_path):
-                # files are assumed to be SingleFileScopes
-                config_name = f"env:{self.name}:{config_path}"
-                tty.debug(f"Creating SingleFileScope {config_name} for '{config_path}'")
-                scopes.append(
-                    spack.config.SingleFileScope(
-                        config_name, config_path, spack.schema.merged.schema
-                    )
-                )
-            else:
-                missing.append(config_path)
-                continue
-
-        if missing:
-            msg = "Detected {0} missing include path(s):".format(len(missing))
-            msg += "\n   {0}".format("\n   ".join(missing))
-            raise spack.config.ConfigFileError(msg)
-
-        return scopes
-
     @property
     def env_config_scopes(self) -> List[spack.config.ConfigScope]:
         """A list of all configuration scopes for the environment manifest. On the first call this
         instantiates all the scopes, on subsequent calls it returns the cached list."""
         if self._config_scopes is not None:
             return self._config_scopes
+
         scopes: List[spack.config.ConfigScope] = [
-            *self.included_config_scopes,
             spack.config.SingleFileScope(
                 self.scope_name,
                 str(self.manifest_file),
                 spack.schema.env.schema,
                 yaml_path=[TOP_LEVEL_KEY],
-            ),
+            )
         ]
         ensure_no_disallowed_env_config_mods(scopes)
         self._config_scopes = scopes

lib/spack/spack/environment/shell.py

@@ -8,6 +8,7 @@
 import llnl.util.tty as tty
 from llnl.util.tty.color import colorize
 
+import spack.config
 import spack.environment as ev
 import spack.repo
 import spack.schema.environment
@@ -158,7 +159,8 @@ def activate(
     # become PATH variables.
     #
 
-    env_vars_yaml = env.manifest.configuration.get("env_vars", None)
+    with env.manifest.use_config():
+        env_vars_yaml = spack.config.get("env_vars", None)
     if env_vars_yaml:
         env_mods.extend(spack.schema.environment.parse(env_vars_yaml))
 
@@ -195,7 +197,8 @@ def deactivate() -> EnvironmentModifications:
     if active is None:
         return env_mods
 
-    env_vars_yaml = active.manifest.configuration.get("env_vars", None)
+    with active.manifest.use_config():
+        env_vars_yaml = spack.config.get("env_vars", None)
     if env_vars_yaml:
         env_mods.extend(spack.schema.environment.parse(env_vars_yaml).reversed())
 

lib/spack/spack/filesystem_view.py

@@ -643,7 +643,7 @@ def print_status(self, *specs, **kwargs):
                 specs.sort()
 
                 abbreviated = [
-                    s.cformat("{name}{@version}{%compiler}{compiler_flags}{variants}")
+                    s.cformat("{name}{@version}{compiler_flags}{variants}{%compiler}")
                     for s in specs
                 ]
 

lib/spack/spack/graph.py

@@ -482,7 +482,7 @@ class SimpleDAG(DotGraphBuilder):
     """Simple DOT graph, with nodes colored uniformly and edges without properties"""
 
     def node_entry(self, node):
-        format_option = "{name}{@version}{%compiler}{/hash:7}"
+        format_option = "{name}{@version}{/hash:7}{%compiler}"
         return node.dag_hash(), f'[label="{node.format(format_option)}"]'
 
     def edge_entry(self, edge):
@@ -515,7 +515,7 @@ def visit(self, edge):
         super().visit(edge)
 
     def node_entry(self, node):
-        node_str = node.format("{name}{@version}{%compiler}{/hash:7}")
+        node_str = node.format("{name}{@version}{/hash:7}{%compiler}")
         options = f'[label="{node_str}", group="build_dependencies", fillcolor="coral"]'
         if node.dag_hash() in self.main_unified_space:
             options = f'[label="{node_str}", group="main_psid"]'

lib/spack/spack/hash_types.py

@@ -6,7 +6,7 @@
 import spack.deptypes as dt
 import spack.repo
 
-hashes = []
+HASHES = []
 
 
 class SpecHashDescriptor:
@@ -23,7 +23,7 @@ def __init__(self, depflag: dt.DepFlag, package_hash, name, override=None):
         self.depflag = depflag
         self.package_hash = package_hash
         self.name = name
-        hashes.append(self)
+        HASHES.append(self)
         # Allow spec hashes to have an alternate computation method
         self.override = override
 
@@ -43,13 +43,9 @@ def __repr__(self):
         )
 
 
-#: Spack's deployment hash. Includes all inputs that can affect how a package is built.
-dag_hash = SpecHashDescriptor(depflag=dt.BUILD | dt.LINK | dt.RUN, package_hash=True, name="hash")
-
-
-#: Hash descriptor used only to transfer a DAG, as is, across processes
-process_hash = SpecHashDescriptor(
-    depflag=dt.BUILD | dt.LINK | dt.RUN | dt.TEST, package_hash=True, name="process_hash"
+#: The DAG hash includes all inputs that can affect how a package is built.
+dag_hash = SpecHashDescriptor(
+    depflag=dt.BUILD | dt.LINK | dt.RUN | dt.TEST, package_hash=True, name="hash"
 )
 
 

lib/spack/spack/install_test.py

@@ -21,7 +21,6 @@
 from llnl.util.lang import nullcontext
 from llnl.util.tty.color import colorize
 
-import spack.build_environment
 import spack.config
 import spack.error
 import spack.package_base
@@ -398,7 +397,7 @@ def stand_alone_tests(self, kwargs):
         Args:
             kwargs (dict): arguments to be used by the test process
         """
-        import spack.build_environment
+        import spack.build_environment  # avoid circular dependency
 
         spack.build_environment.start_build_process(self.pkg, test_process, kwargs)
 
@@ -463,6 +462,8 @@ def write_tested_status(self):
 
 @contextlib.contextmanager
 def test_part(pkg: Pb, test_name: str, purpose: str, work_dir: str = ".", verbose: bool = False):
+    import spack.build_environment  # avoid circular dependency
+
     wdir = "." if work_dir is None else work_dir
     tester = pkg.tester
     assert test_name and test_name.startswith(

lib/spack/spack/modules/common.py

@@ -564,6 +564,12 @@ def __init__(self, configuration):
     def spec(self):
         return self.conf.spec
 
+    @tengine.context_property
+    def tags(self):
+        if not hasattr(self.spec.package, "tags"):
+            return []
+        return self.spec.package.tags
+
     @tengine.context_property
     def timestamp(self):
         return datetime.datetime.now()

lib/spack/spack/package_base.py

@@ -48,6 +48,7 @@
 import spack.store
 import spack.url
 import spack.util.environment
+import spack.util.executable
 import spack.util.path
 import spack.util.web
 import spack.variant
@@ -1287,12 +1288,13 @@ def extendee_spec(self):
         if not self.extendees:
             return None
 
-        deps = []
-
         # If the extendee is in the spec's deps already, return that.
-        for dep in self.spec.traverse(deptype=("link", "run")):
-            if dep.name in self.extendees:
-                deps.append(dep)
+        deps = [
+            dep
+            for dep in self.spec.dependencies(deptype=("link", "run"))
+            for d, when in self.extendees.values()
+            if dep.satisfies(d) and self.spec.satisfies(when)
+        ]
 
         if deps:
             assert len(deps) == 1
@@ -1369,6 +1371,14 @@ def prefix(self):
     def home(self):
         return self.prefix
 
+    @property
+    def command(self) -> spack.util.executable.Executable:
+        """Returns the main executable for this package."""
+        path = os.path.join(self.home.bin, self.spec.name)
+        if fsys.is_exe(path):
+            return spack.util.executable.Executable(path)
+        raise RuntimeError(f"Unable to locate {self.spec.name} command in {self.home.bin}")
+
     @property  # type: ignore[misc]
     @memoized
     def compiler(self):

lib/spack/spack/repo.py

@@ -1406,6 +1406,8 @@ def create_repo(root, namespace=None, subdir=packages_dir_name):
             config.write(f"  namespace: '{namespace}'\n")
             if subdir != packages_dir_name:
                 config.write(f"  subdirectory: '{subdir}'\n")
+            x, y = spack.package_api_version
+            config.write(f"  api: v{x}.{y}\n")
 
     except OSError as e:
         # try to clean up.

lib/spack/spack/schema/env.py

@@ -29,11 +29,7 @@
             # merged configuration scope schemas
             spack.schema.merged.properties,
             # extra environment schema properties
-            {
-                "include": {"type": "array", "default": [], "items": {"type": "string"}},
-                "specs": spec_list_schema,
-                "include_concrete": include_concrete,
-            },
+            {"specs": spec_list_schema, "include_concrete": include_concrete},
         ),
     }
 }

lib/spack/spack/schema/include.py

@@ -0,0 +1,41 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Schema for include.yaml configuration file.
+
+.. literalinclude:: _spack_root/lib/spack/spack/schema/include.py
+   :lines: 12-
+"""
+from typing import Any, Dict
+
+#: Properties for inclusion in other schemas
+properties: Dict[str, Any] = {
+    "include": {
+        "type": "array",
+        "default": [],
+        "additionalProperties": False,
+        "items": {
+            "anyOf": [
+                {
+                    "type": "object",
+                    "properties": {
+                        "when": {"type": "string"},
+                        "path": {"type": "string"},
+                        "sha256": {"type": "string"},
+                        "optional": {"type": "boolean"},
+                    },
+                    "required": ["path"],
+                    "additionalProperties": False,
+                },
+                {"type": "string"},
+            ]
+        },
+    }
+}
+
+#: Full schema with metadata
+schema = {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Spack include configuration file schema",
+    "properties": properties,
+}

lib/spack/spack/schema/merged.py

@@ -21,6 +21,7 @@
 import spack.schema.definitions
 import spack.schema.develop
 import spack.schema.env_vars
+import spack.schema.include
 import spack.schema.mirrors
 import spack.schema.modules
 import spack.schema.packages
@@ -40,6 +41,7 @@
     spack.schema.definitions.properties,
     spack.schema.develop.properties,
     spack.schema.env_vars.properties,
+    spack.schema.include.properties,
     spack.schema.mirrors.properties,
     spack.schema.modules.properties,
     spack.schema.packages.properties,
@@ -48,7 +50,6 @@
     spack.schema.view.properties,
 )
 
-
 #: Full schema with metadata
 schema = {
     "$schema": "http://json-schema.org/draft-07/schema#",

lib/spack/spack/solver/asp.py

@@ -50,7 +50,6 @@
 import spack.deptypes as dt
 import spack.environment as ev
 import spack.error
-import spack.hash_types as ht
 import spack.package_base
 import spack.package_prefs
 import spack.patch
@@ -567,7 +566,6 @@ def to_dict(self, test: bool = False) -> dict:
         serial_node_arg = (
             lambda node_dict: f"""{{"id": "{node_dict.id}", "pkg": "{node_dict.pkg}"}}"""
         )
-        spec_hash_type = ht.process_hash if test else ht.dag_hash
         ret = dict()
         ret["asp"] = self.asp
         ret["criteria"] = self.criteria
@@ -581,14 +579,14 @@ def to_dict(self, test: bool = False) -> dict:
             serial_answer = answer[:2]
             serial_answer_dict = {}
             for node, spec in answer[2].items():
-                serial_answer_dict[serial_node_arg(node)] = spec.to_dict(hash=spec_hash_type)
+                serial_answer_dict[serial_node_arg(node)] = spec.to_dict()
             serial_answer = serial_answer + (serial_answer_dict,)
             serial_answers.append(serial_answer)
         ret["answers"] = serial_answers
         ret["specs_by_input"] = {}
         input_specs = {} if not self.specs_by_input else self.specs_by_input
         for input, spec in input_specs.items():
-            ret["specs_by_input"][str(input)] = spec.to_dict(hash=spec_hash_type)
+            ret["specs_by_input"][str(input)] = spec.to_dict()
         return ret
 
     @staticmethod
@@ -648,10 +646,9 @@ class ConcretizationCache:
     """
 
     def __init__(self, root: Union[str, None] = None):
-        if not root:
-            root = spack.config.get(
-                "config:concretization_cache:url", spack.paths.default_conc_cache_path
-            )
+        root = root or spack.config.get(
+            "config:concretization_cache:url", spack.paths.default_conc_cache_path
+        )
         self.root = pathlib.Path(spack.util.path.canonicalize_path(root))
         self._fc = FileCache(self.root)
         self._cache_manifest = ".cache_manifest"
@@ -1212,7 +1209,7 @@ def solve(self, setup, specs, reuse=None, output=None, control=None, allow_depre
         full_path = lambda x: os.path.join(parent_dir, x)
         abs_control_files = [full_path(x) for x in control_files]
         for ctrl_file in abs_control_files:
-            with open(ctrl_file, "r+", encoding="utf-8") as f:
+            with open(ctrl_file, "r", encoding="utf-8") as f:
                 problem_repr += "\n" + f.read()
 
         result = None
@@ -2237,8 +2234,8 @@ def emit_facts_from_requirement_rules(self, rules: List[RequirementRule]):
                 spec.attach_git_version_lookup()
 
                 when_spec = spec
-                if virtual:
-                    when_spec = spack.spec.Spec(pkg_name)
+                if virtual and spec.name != pkg_name:
+                    when_spec = spack.spec.Spec(f"^[virtuals={pkg_name}] {spec.name}")
 
                 try:
                     context = ConditionContext()

lib/spack/spack/solver/concretize.lp

@@ -597,6 +597,13 @@ attr("virtual_on_edge", PackageNode, ProviderNode, Virtual)
 attr("virtual_on_incoming_edges", ProviderNode, Virtual)
   :- attr("virtual_on_edge", _, ProviderNode, Virtual).
 
+% This is needed to allow requirement on virtuals,
+% when a virtual root is requested
+attr("virtual_on_incoming_edges", ProviderNode, Virtual)
+  :- attr("virtual_root", node(min_dupe_id, Virtual)),
+     attr("root", ProviderNode),
+     provider(ProviderNode, node(min_dupe_id, Virtual)).
+
 % dependencies on virtuals also imply that the virtual is a virtual node
 1 { attr("virtual_node", node(0..X-1, Virtual)) : max_dupes(Virtual, X) }
   :- node_depends_on_virtual(PackageNode, Virtual).
@@ -953,12 +960,14 @@ error(100, "Cannot set variant '{0}' for package '{1}' because the variant condi
      build(node(ID, Package)).
 
 % at most one variant value for single-valued variants.
-error(100, "'{0}' required multiple values for single-valued variant '{1}'", Package, Variant)
+error(100, "'{0}' requires conflicting variant values 'Spec({1}={2})' and 'Spec({1}={3})'", Package, Variant, Value1, Value2)
   :- attr("node", node(ID, Package)),
      node_has_variant(node(ID, Package), Variant, _),
      variant_single_value(node(ID, Package), Variant),
-     build(node(ID, Package)),
-     2 { attr("variant_value", node(ID, Package), Variant, Value) }.
+     attr("variant_value", node(ID, Package), Variant, Value1),
+     attr("variant_value", node(ID, Package), Variant, Value2),
+     Value1 < Value2,
+     build(node(ID, Package)).
 
 error(100, "No valid value for variant '{1}' of package '{0}'", Package, Variant)
   :- attr("node", node(ID, Package)),
@@ -1682,11 +1691,10 @@ opt_criterion(45, "preferred providers (non-roots)").
 #minimize{ 0@245: #true }.
 #minimize{ 0@45: #true }.
 #minimize{
-    Weight@45+Priority,X,Provider,Virtual
-    : provider_weight(node(X, Provider), Virtual, Weight),
-      not attr("root", node(X, Provider)),
-      build_priority(node(X, Provider), Priority),
-      not runtime(Provider)
+    Weight@45+Priority,ProviderNode,Virtual
+    : provider_weight(ProviderNode, Virtual, Weight),
+      not attr("root", ProviderNode),
+      build_priority(ProviderNode, Priority)
 }.
 
 % Try to minimize the number of compiler mismatches in the DAG.

lib/spack/spack/solver/error_messages.lp

@@ -117,7 +117,7 @@ error(0, "Cannot find a valid provider for virtual {0}", Virtual, startcauses, C
      condition_holds(Cause, node(CID, TriggerPkg)).
 
 % At most one variant value for single-valued variants
-error(0, "'{0}' required multiple values for single-valued variant '{1}'\n    Requested 'Spec({1}={2})' and 'Spec({1}={3})'", Package, Variant, Value1, Value2, startcauses, Cause1, X, Cause2, X)
+error(0, "'{0}' requires conflicting variant values 'Spec({1}={2})' and 'Spec({1}={3})'", Package, Variant, Value1, Value2, startcauses, Cause1, X, Cause2, X)
   :- attr("node", node(X, Package)),
      node_has_variant(node(X, Package), Variant, VariantID),
      variant_single_value(node(X, Package), Variant),

lib/spack/spack/solver/requirements.py

@@ -10,7 +10,7 @@
 import spack.error
 import spack.package_base
 import spack.spec
-from spack.config import get_mark_from_yaml_data
+from spack.util.spack_yaml import get_mark_from_yaml_data
 
 
 class RequirementKind(enum.Enum):
@@ -66,18 +66,29 @@ def rules_from_package_py(self, pkg: spack.package_base.PackageBase) -> List[Req
         return rules
 
     def rules_from_virtual(self, virtual_str: str) -> List[RequirementRule]:
-        requirements = self.config.get("packages", {}).get(virtual_str, {}).get("require", [])
-        return self._rules_from_requirements(
-            virtual_str, requirements, kind=RequirementKind.VIRTUAL
-        )
+        kind, requests = self._raw_yaml_data(virtual_str, section="require", virtual=True)
+        result = self._rules_from_requirements(virtual_str, requests, kind=kind)
+
+        kind, requests = self._raw_yaml_data(virtual_str, section="prefer", virtual=True)
+        result.extend(self._rules_from_preferences(virtual_str, preferences=requests, kind=kind))
+
+        kind, requests = self._raw_yaml_data(virtual_str, section="conflict", virtual=True)
+        result.extend(self._rules_from_conflicts(virtual_str, conflicts=requests, kind=kind))
+
+        return result
 
     def rules_from_require(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:
-        kind, requirements = self._raw_yaml_data(pkg, section="require")
+        kind, requirements = self._raw_yaml_data(pkg.name, section="require")
         return self._rules_from_requirements(pkg.name, requirements, kind=kind)
 
     def rules_from_prefer(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:
+        kind, preferences = self._raw_yaml_data(pkg.name, section="prefer")
+        return self._rules_from_preferences(pkg.name, preferences=preferences, kind=kind)
+
+    def _rules_from_preferences(
+        self, pkg_name: str, *, preferences, kind: RequirementKind
+    ) -> List[RequirementRule]:
         result = []
-        kind, preferences = self._raw_yaml_data(pkg, section="prefer")
         for item in preferences:
             spec, condition, message = self._parse_prefer_conflict_item(item)
             result.append(
@@ -86,7 +97,7 @@ def rules_from_prefer(self, pkg: spack.package_base.PackageBase) -> List[Require
                 # require:
                 # - any_of: [spec_str, "@:"]
                 RequirementRule(
-                    pkg_name=pkg.name,
+                    pkg_name=pkg_name,
                     policy="any_of",
                     requirements=[spec, spack.spec.Spec("@:")],
                     kind=kind,
@@ -97,8 +108,13 @@ def rules_from_prefer(self, pkg: spack.package_base.PackageBase) -> List[Require
         return result
 
     def rules_from_conflict(self, pkg: spack.package_base.PackageBase) -> List[RequirementRule]:
+        kind, conflicts = self._raw_yaml_data(pkg.name, section="conflict")
+        return self._rules_from_conflicts(pkg.name, conflicts=conflicts, kind=kind)
+
+    def _rules_from_conflicts(
+        self, pkg_name: str, *, conflicts, kind: RequirementKind
+    ) -> List[RequirementRule]:
         result = []
-        kind, conflicts = self._raw_yaml_data(pkg, section="conflict")
         for item in conflicts:
             spec, condition, message = self._parse_prefer_conflict_item(item)
             result.append(
@@ -107,7 +123,7 @@ def rules_from_conflict(self, pkg: spack.package_base.PackageBase) -> List[Requi
                 # require:
                 # - one_of: [spec_str, "@:"]
                 RequirementRule(
-                    pkg_name=pkg.name,
+                    pkg_name=pkg_name,
                     policy="one_of",
                     requirements=[spec, spack.spec.Spec("@:")],
                     kind=kind,
@@ -129,10 +145,14 @@ def _parse_prefer_conflict_item(self, item):
             message = item.get("message")
         return spec, condition, message
 
-    def _raw_yaml_data(self, pkg: spack.package_base.PackageBase, *, section: str):
+    def _raw_yaml_data(self, pkg_name: str, *, section: str, virtual: bool = False):
         config = self.config.get("packages")
-        data = config.get(pkg.name, {}).get(section, [])
+        data = config.get(pkg_name, {}).get(section, [])
         kind = RequirementKind.PACKAGE
+
+        if virtual:
+            return RequirementKind.VIRTUAL, data
+
         if not data:
             data = config.get("all", {}).get(section, [])
             kind = RequirementKind.DEFAULT
@@ -165,7 +185,8 @@ def _rules_from_requirements(
 
                 # validate specs from YAML first, and fail with line numbers if parsing fails.
                 constraints = [
-                    parse_spec_from_yaml_string(constraint) for constraint in constraints
+                    parse_spec_from_yaml_string(constraint, named=kind == RequirementKind.VIRTUAL)
+                    for constraint in constraints
                 ]
                 when_str = requirement.get("when")
                 when = parse_spec_from_yaml_string(when_str) if when_str else spack.spec.Spec()
@@ -203,7 +224,7 @@ def reject_requirement_constraint(
                 s.validate_or_raise()
             except spack.error.SpackError as e:
                 tty.debug(
-                    f"[SETUP] Rejecting the default '{constraint}' requirement "
+                    f"[{__name__}] Rejecting the default '{constraint}' requirement "
                     f"on '{pkg_name}': {str(e)}",
                     level=2,
                 )
@@ -211,21 +232,37 @@ def reject_requirement_constraint(
         return False
 
 
-def parse_spec_from_yaml_string(string: str) -> spack.spec.Spec:
+def parse_spec_from_yaml_string(string: str, *, named: bool = False) -> spack.spec.Spec:
     """Parse a spec from YAML and add file/line info to errors, if it's available.
 
     Parse a ``Spec`` from the supplied string, but also intercept any syntax errors and
     add file/line information for debugging using file/line annotations from the string.
 
-    Arguments:
+    Args:
         string: a string representing a ``Spec`` from config YAML.
-
+        named: if True, the spec must have a name
     """
     try:
-        return spack.spec.Spec(string)
+        result = spack.spec.Spec(string)
     except spack.error.SpecSyntaxError as e:
         mark = get_mark_from_yaml_data(string)
         if mark:
             msg = f"{mark.name}:{mark.line + 1}: {str(e)}"
             raise spack.error.SpecSyntaxError(msg) from e
         raise e
+
+    if named is True and not result.name:
+        msg = f"expected a named spec, but got '{string}' instead"
+        mark = get_mark_from_yaml_data(string)
+
+        # Add a hint in case it's dependencies
+        deps = result.dependencies()
+        if len(deps) == 1:
+            msg = f"{msg}. Did you mean '{deps[0]}'?"
+
+        if mark:
+            msg = f"{mark.name}:{mark.line + 1}: {msg}"
+
+        raise spack.error.SpackError(msg)
+
+    return result

lib/spack/spack/spec.py

@@ -97,7 +97,6 @@
 import spack.spec_parser
 import spack.store
 import spack.traverse
-import spack.util.executable
 import spack.util.hash
 import spack.util.prefix
 import spack.util.spack_json as sjson
@@ -175,15 +174,17 @@
 #:     Spec(Spec("string").format()) == Spec("string)"
 DEFAULT_FORMAT = (
     "{name}{@versions}"
-    "{%compiler.name}{@compiler.versions}{compiler_flags}"
+    "{compiler_flags}"
     "{variants}{ namespace=namespace_if_anonymous}{ arch=architecture}{/abstract_hash}"
+    " {%compiler.name}{@compiler.versions}"
 )
 
 #: Display format, which eliminates extra `@=` in the output, for readability.
 DISPLAY_FORMAT = (
     "{name}{@version}"
-    "{%compiler.name}{@compiler.version}{compiler_flags}"
+    "{compiler_flags}"
     "{variants}{ namespace=namespace_if_anonymous}{ arch=architecture}{/abstract_hash}"
+    " {%compiler.name}{@compiler.version}"
 )
 
 #: Regular expression to pull spec contents out of clearsigned signature
@@ -1108,28 +1109,6 @@ def clear(self):
         self.edges.clear()
 
 
-def _command_default_handler(spec: "Spec"):
-    """Default handler when looking for the 'command' attribute.
-
-    Tries to search for ``spec.name`` in the ``spec.home.bin`` directory.
-
-    Parameters:
-        spec: spec that is being queried
-
-    Returns:
-        Executable: An executable of the command
-
-    Raises:
-        RuntimeError: If the command is not found
-    """
-    home = getattr(spec.package, "home")
-    path = os.path.join(home.bin, spec.name)
-
-    if fs.is_exe(path):
-        return spack.util.executable.Executable(path)
-    raise RuntimeError(f"Unable to locate {spec.name} command in {home.bin}")
-
-
 def _headers_default_handler(spec: "Spec"):
     """Default handler when looking for the 'headers' attribute.
 
@@ -1333,9 +1312,7 @@ class SpecBuildInterface(lang.ObjectWrapper):
     home = ForwardQueryToPackage("home", default_handler=None)
     headers = ForwardQueryToPackage("headers", default_handler=_headers_default_handler)
     libs = ForwardQueryToPackage("libs", default_handler=_libs_default_handler)
-    command = ForwardQueryToPackage(
-        "command", default_handler=_command_default_handler, _indirect=True
-    )
+    command = ForwardQueryToPackage("command", default_handler=None, _indirect=True)
 
     def __init__(
         self,
@@ -1513,7 +1490,7 @@ def __init__(self, spec_like=None, *, external_path=None, external_modules=None)
         self.abstract_hash = None
 
         # initial values for all spec hash types
-        for h in ht.hashes:
+        for h in ht.HASHES:
             setattr(self, h.attr, None)
 
         # cache for spec's prefix, computed lazily by prefix property
@@ -2106,16 +2083,18 @@ def long_spec(self):
     def short_spec(self):
         """Returns a version of the spec with the dependencies hashed
         instead of completely enumerated."""
-        spec_format = "{name}{@version}{%compiler.name}{@compiler.version}"
-        spec_format += "{variants}{ arch=architecture}{/hash:7}"
-        return self.format(spec_format)
+        return self.format(
+            "{name}{@version}{variants}{ arch=architecture}"
+            "{/hash:7}{%compiler.name}{@compiler.version}"
+        )
 
     @property
     def cshort_spec(self):
         """Returns an auto-colorized version of ``self.short_spec``."""
-        spec_format = "{name}{@version}{%compiler.name}{@compiler.version}"
-        spec_format += "{variants}{ arch=architecture}{/hash:7}"
-        return self.cformat(spec_format)
+        return self.cformat(
+            "{name}{@version}{variants}{ arch=architecture}"
+            "{/hash:7}{%compiler.name}{@compiler.version}"
+        )
 
     @property
     def prefix(self) -> spack.util.prefix.Prefix:
@@ -2194,30 +2173,16 @@ def package_hash(self):
     def dag_hash(self, length=None):
         """This is Spack's default hash, used to identify installations.
 
-        Same as the full hash (includes package hash and build/link/run deps).
-        Tells us when package files and any dependencies have changes.
-
         NOTE: Versions of Spack prior to 0.18 only included link and run deps.
+        NOTE: Versions of Spack prior to 1.0 only did not include test deps.
 
         """
         return self._cached_hash(ht.dag_hash, length)
 
-    def process_hash(self, length=None):
-        """Hash used to transfer specs among processes.
-
-        This hash includes build and test dependencies and is only used to
-        serialize a spec and pass it around among processes.
-        """
-        return self._cached_hash(ht.process_hash, length)
-
     def dag_hash_bit_prefix(self, bits):
         """Get the first <bits> bits of the DAG hash as an integer type."""
         return spack.util.hash.base32_prefix_bits(self.dag_hash(), bits)
 
-    def process_hash_bit_prefix(self, bits):
-        """Get the first <bits> bits of the DAG hash as an integer type."""
-        return spack.util.hash.base32_prefix_bits(self.process_hash(), bits)
-
     def _lookup_hash(self):
         """Lookup just one spec with an abstract hash, returning a spec from the the environment,
         store, or finally, binary caches."""
@@ -3584,11 +3549,11 @@ def _dup(self, other: "Spec", deps: Union[bool, dt.DepTypes, dt.DepFlag] = True)
 
         if self._concrete:
             self._dunder_hash = other._dunder_hash
-            for h in ht.hashes:
+            for h in ht.HASHES:
                 setattr(self, h.attr, getattr(other, h.attr, None))
         else:
             self._dunder_hash = None
-            for h in ht.hashes:
+            for h in ht.HASHES:
                 setattr(self, h.attr, None)
 
         return changed
@@ -3780,16 +3745,6 @@ def _cmp_iter(self):
         # serialized before the hash change and one after, are considered different.
         yield self.dag_hash() if self.concrete else None
 
-        # This needs to be in _cmp_iter so that no specs with different process hashes
-        # are considered the same by `__hash__` or `__eq__`.
-        #
-        # TODO: We should eventually unify the `_cmp_*` methods with `to_node_dict` so
-        # TODO: there aren't two sources of truth, but this needs some thought, since
-        # TODO: they exist for speed.  We should benchmark whether it's really worth
-        # TODO: having two types of hashing now that we use `json` instead of `yaml` for
-        # TODO: spec hashing.
-        yield self.process_hash() if self.concrete else None
-
         def deps():
             for dep in sorted(itertools.chain.from_iterable(self._dependencies.values())):
                 yield dep.spec.name
@@ -4443,7 +4398,7 @@ def clear_caches(self, ignore: Tuple[str, ...] = ()) -> None:
         """
         Clears all cached hashes in a Spec, while preserving other properties.
         """
-        for h in ht.hashes:
+        for h in ht.HASHES:
             if h.attr not in ignore:
                 if hasattr(self, h.attr):
                     setattr(self, h.attr, None)
@@ -4452,18 +4407,12 @@ def clear_caches(self, ignore: Tuple[str, ...] = ()) -> None:
                 setattr(self, attr, None)
 
     def __hash__(self):
-        # If the spec is concrete, we leverage the process hash and just use
-        # a 64-bit prefix of it. The process hash has the advantage that it's
-        # computed once per concrete spec, and it's saved -- so if we read
-        # concrete specs we don't need to recompute the whole hash. This is
-        # good for large, unchanging specs.
-        #
-        # We use the process hash instead of the DAG hash here because the DAG
-        # hash includes the package hash, which can cause infinite recursion,
-        # and which isn't defined unless the spec has a known package.
+        # If the spec is concrete, we leverage the dag hash and just use a 64-bit prefix of it.
+        # The dag hash has the advantage that it's computed once per concrete spec, and it's saved
+        # -- so if we read concrete specs we don't need to recompute the whole hash.
         if self.concrete:
             if not self._dunder_hash:
-                self._dunder_hash = self.process_hash_bit_prefix(64)
+                self._dunder_hash = self.dag_hash_bit_prefix(64)
             return self._dunder_hash
 
         # This is the normal hash for lazy_lexicographic_ordering. It's
@@ -4472,7 +4421,7 @@ def __hash__(self):
         return hash(lang.tuplify(self._cmp_iter))
 
     def __reduce__(self):
-        return Spec.from_dict, (self.to_dict(hash=ht.process_hash),)
+        return Spec.from_dict, (self.to_dict(hash=ht.dag_hash),)
 
     def attach_git_version_lookup(self):
         # Add a git lookup method for GitVersions
@@ -4794,7 +4743,7 @@ def from_node_dict(cls, node):
         spec = Spec()
 
         name, node = cls.name_and_data(node)
-        for h in ht.hashes:
+        for h in ht.HASHES:
             setattr(spec, h.attr, node.get(h.name, None))
 
         spec.name = name
@@ -4977,7 +4926,7 @@ def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):
         """
         for dep_name, elt in deps.items():
             if isinstance(elt, dict):
-                for h in ht.hashes:
+                for h in ht.HASHES:
                     if h.name in elt:
                         dep_hash, deptypes = elt[h.name], elt["type"]
                         hash_type = h.name
@@ -5020,7 +4969,7 @@ def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):
             dep_name = dep["name"]
             if isinstance(elt, dict):
                 # new format: elements of dependency spec are keyed.
-                for h in ht.hashes:
+                for h in ht.HASHES:
                     if h.name in elt:
                         dep_hash, deptypes, hash_type, virtuals = cls.extract_info_from_dep(elt, h)
                         break
@@ -5130,6 +5079,13 @@ def get_host_environment() -> Dict[str, Any]:
     }
 
 
+def eval_conditional(string):
+    """Evaluate conditional definitions using restricted variable scope."""
+    valid_variables = get_host_environment()
+    valid_variables.update({"re": re, "env": os.environ})
+    return eval(string, valid_variables)
+
+
 class SpecParseError(spack.error.SpecError):
     """Wrapper for ParseError for when we're parsing specs."""
 
@@ -5288,8 +5244,10 @@ def __init__(self, spec):
 
 class AmbiguousHashError(spack.error.SpecError):
     def __init__(self, msg, *specs):
-        spec_fmt = "{namespace}.{name}{@version}{%compiler}{compiler_flags}"
-        spec_fmt += "{variants}{ arch=architecture}{/hash:7}"
+        spec_fmt = (
+            "{namespace}.{name}{@version}{compiler_flags}{variants}"
+            "{ arch=architecture}{/hash:7}{%compiler}"
+        )
         specs_str = "\n  " + "\n  ".join(spec.format(spec_fmt) for spec in specs)
         super().__init__(msg + specs_str)
 

lib/spack/spack/spec_parser.py

@@ -60,13 +60,17 @@
 import pathlib
 import re
 import sys
-from typing import Iterator, List, Optional
+import traceback
+import warnings
+from typing import Iterator, List, Optional, Tuple
 
 from llnl.util.tty import color
 
 import spack.deptypes
 import spack.error
+import spack.paths
 import spack.spec
+import spack.util.spack_yaml
 import spack.version
 from spack.tokenize import Token, TokenBase, Tokenizer
 
@@ -204,6 +208,32 @@ def __init__(self, tokens: List[Token], text: str):
         super().__init__(message)
 
 
+def _warn_about_variant_after_compiler(literal_str: str, issues: List[str]):
+    """Issue a warning if variant or other token is preceded by a compiler token. The warning is
+    only issued if it's actionable: either we know the config file it originates from, or we have
+    call site that's not internal to Spack."""
+    ignore = [spack.paths.lib_path, spack.paths.bin_path]
+    mark = spack.util.spack_yaml.get_mark_from_yaml_data(literal_str)
+    issue_str = ", ".join(issues)
+    error = f"{issue_str} in `{literal_str}`"
+
+    # warning from config file
+    if mark:
+        warnings.warn(f"{mark.name}:{mark.line + 1}: {error}")
+        return
+
+    # warning from hopefully package.py
+    for frame in reversed(traceback.extract_stack()):
+        if frame.lineno and not any(frame.filename.startswith(path) for path in ignore):
+            warnings.warn_explicit(
+                error,
+                category=spack.error.SpackAPIWarning,
+                filename=frame.filename,
+                lineno=frame.lineno,
+            )
+            return
+
+
 class SpecParser:
     """Parse text into specs"""
 
@@ -242,26 +272,31 @@ def add_dependency(dep, **edge_properties):
                 raise SpecParsingError(str(e), self.ctx.current_token, self.literal_str) from e
 
         initial_spec = initial_spec or spack.spec.Spec()
-        root_spec = SpecNodeParser(self.ctx, self.literal_str).parse(initial_spec)
+        root_spec, parser_warnings = SpecNodeParser(self.ctx, self.literal_str).parse(initial_spec)
         while True:
             if self.ctx.accept(SpecTokens.START_EDGE_PROPERTIES):
                 edge_properties = EdgeAttributeParser(self.ctx, self.literal_str).parse()
                 edge_properties.setdefault("depflag", 0)
                 edge_properties.setdefault("virtuals", ())
-                dependency = self._parse_node(root_spec)
+                dependency, warnings = self._parse_node(root_spec)
+                parser_warnings.extend(warnings)
                 add_dependency(dependency, **edge_properties)
 
             elif self.ctx.accept(SpecTokens.DEPENDENCY):
-                dependency = self._parse_node(root_spec)
+                dependency, warnings = self._parse_node(root_spec)
+                parser_warnings.extend(warnings)
                 add_dependency(dependency, depflag=0, virtuals=())
 
             else:
                 break
 
+        if parser_warnings:
+            _warn_about_variant_after_compiler(self.literal_str, parser_warnings)
+
         return root_spec
 
     def _parse_node(self, root_spec):
-        dependency = SpecNodeParser(self.ctx, self.literal_str).parse()
+        dependency, parser_warnings = SpecNodeParser(self.ctx, self.literal_str).parse()
         if dependency is None:
             msg = (
                 "the dependency sigil and any optional edge attributes must be followed by a "
@@ -270,7 +305,7 @@ def _parse_node(self, root_spec):
             raise SpecParsingError(msg, self.ctx.current_token, self.literal_str)
         if root_spec.concrete:
             raise spack.spec.RedundantSpecError(root_spec, "^" + str(dependency))
-        return dependency
+        return dependency, parser_warnings
 
     def all_specs(self) -> List["spack.spec.Spec"]:
         """Return all the specs that remain to be parsed"""
@@ -290,7 +325,7 @@ def __init__(self, ctx, literal_str):
 
     def parse(
         self, initial_spec: Optional["spack.spec.Spec"] = None
-    ) -> Optional["spack.spec.Spec"]:
+    ) -> Tuple["spack.spec.Spec", List[str]]:
         """Parse a single spec node from a stream of tokens
 
         Args:
@@ -299,12 +334,15 @@ def parse(
         Return
             The object passed as argument
         """
-        if not self.ctx.next_token or self.ctx.expect(SpecTokens.DEPENDENCY):
-            return initial_spec
+        parser_warnings: List[str] = []
+        last_compiler = None
 
         if initial_spec is None:
             initial_spec = spack.spec.Spec()
 
+        if not self.ctx.next_token or self.ctx.expect(SpecTokens.DEPENDENCY):
+            return initial_spec, parser_warnings
+
         # If we start with a package name we have a named spec, we cannot
         # accept another package name afterwards in a node
         if self.ctx.accept(SpecTokens.UNQUALIFIED_PACKAGE_NAME):
@@ -318,7 +356,7 @@ def parse(
             initial_spec.namespace = namespace
 
         elif self.ctx.accept(SpecTokens.FILENAME):
-            return FileParser(self.ctx).parse(initial_spec)
+            return FileParser(self.ctx).parse(initial_spec), parser_warnings
 
         def raise_parsing_error(string: str, cause: Optional[Exception] = None):
             """Raise a spec parsing error with token context."""
@@ -331,6 +369,12 @@ def add_flag(name: str, value: str, propagate: bool):
             except Exception as e:
                 raise_parsing_error(str(e), e)
 
+        def warn_if_after_compiler(token: str):
+            """Register a warning for %compiler followed by +variant that will in the future apply
+            to the compiler instead of the current root."""
+            if last_compiler:
+                parser_warnings.append(f"`{token}` should go before `{last_compiler}`")
+
         while True:
             if self.ctx.accept(SpecTokens.COMPILER):
                 if self.has_compiler:
@@ -339,6 +383,7 @@ def add_flag(name: str, value: str, propagate: bool):
                 compiler_name = self.ctx.current_token.value[1:]
                 initial_spec.compiler = spack.spec.CompilerSpec(compiler_name.strip(), ":")
                 self.has_compiler = True
+                last_compiler = self.ctx.current_token.value
 
             elif self.ctx.accept(SpecTokens.COMPILER_AND_VERSION):
                 if self.has_compiler:
@@ -349,6 +394,7 @@ def add_flag(name: str, value: str, propagate: bool):
                     compiler_name.strip(), compiler_version
                 )
                 self.has_compiler = True
+                last_compiler = self.ctx.current_token.value
 
             elif (
                 self.ctx.accept(SpecTokens.VERSION_HASH_PAIR)
@@ -363,14 +409,17 @@ def add_flag(name: str, value: str, propagate: bool):
                 )
                 initial_spec.attach_git_version_lookup()
                 self.has_version = True
+                warn_if_after_compiler(self.ctx.current_token.value)
 
             elif self.ctx.accept(SpecTokens.BOOL_VARIANT):
                 variant_value = self.ctx.current_token.value[0] == "+"
                 add_flag(self.ctx.current_token.value[1:].strip(), variant_value, propagate=False)
+                warn_if_after_compiler(self.ctx.current_token.value)
 
             elif self.ctx.accept(SpecTokens.PROPAGATED_BOOL_VARIANT):
                 variant_value = self.ctx.current_token.value[0:2] == "++"
                 add_flag(self.ctx.current_token.value[2:].strip(), variant_value, propagate=True)
+                warn_if_after_compiler(self.ctx.current_token.value)
 
             elif self.ctx.accept(SpecTokens.KEY_VALUE_PAIR):
                 match = SPLIT_KVP.match(self.ctx.current_token.value)
@@ -378,6 +427,7 @@ def add_flag(name: str, value: str, propagate: bool):
 
                 name, _, value = match.groups()
                 add_flag(name, strip_quotes_and_unescape(value), propagate=False)
+                warn_if_after_compiler(self.ctx.current_token.value)
 
             elif self.ctx.accept(SpecTokens.PROPAGATED_KEY_VALUE_PAIR):
                 match = SPLIT_KVP.match(self.ctx.current_token.value)
@@ -385,17 +435,19 @@ def add_flag(name: str, value: str, propagate: bool):
 
                 name, _, value = match.groups()
                 add_flag(name, strip_quotes_and_unescape(value), propagate=True)
+                warn_if_after_compiler(self.ctx.current_token.value)
 
             elif self.ctx.expect(SpecTokens.DAG_HASH):
                 if initial_spec.abstract_hash:
                     break
                 self.ctx.accept(SpecTokens.DAG_HASH)
                 initial_spec.abstract_hash = self.ctx.current_token.value[1:]
+                warn_if_after_compiler(self.ctx.current_token.value)
 
             else:
                 break
 
-        return initial_spec
+        return initial_spec, parser_warnings
 
 
 class FileParser:
@@ -485,23 +537,18 @@ def parse_one_or_raise(
         text (str): text to be parsed
         initial_spec: buffer where to parse the spec. If None a new one will be created.
     """
-    stripped_text = text.strip()
-    parser = SpecParser(stripped_text)
+    parser = SpecParser(text)
     result = parser.next_spec(initial_spec)
-    last_token = parser.ctx.current_token
+    next_token = parser.ctx.next_token
 
-    if last_token is not None and last_token.end != len(stripped_text):
-        message = "a single spec was requested, but parsed more than one:"
-        message += f"\n{text}"
-        if last_token is not None:
-            underline = f"\n{' ' * last_token.end}{'^' * (len(text) - last_token.end)}"
-            message += color.colorize(f"@*r{{{underline}}}")
+    if next_token:
+        message = f"expected a single spec, but got more:\n{text}"
+        underline = f"\n{' ' * next_token.start}{'^' * len(next_token.value)}"
+        message += color.colorize(f"@*r{{{underline}}}")
         raise ValueError(message)
 
     if result is None:
-        message = "a single spec was requested, but none was parsed:"
-        message += f"\n{text}"
-        raise ValueError(message)
+        raise ValueError("expected a single spec, but got none")
 
     return result
 

lib/spack/spack/test/architecture.py

@@ -129,7 +129,7 @@ def test_satisfy_strict_constraint_when_not_concrete(architecture_tuple, constra
 )
 def test_concretize_target_ranges(root_target_range, dep_target_range, result, monkeypatch):
     spec = Spec(
-        f"pkg-a %gcc@10 foobar=bar target={root_target_range} ^pkg-b target={dep_target_range}"
+        f"pkg-a foobar=bar target={root_target_range} %gcc@10 ^pkg-b target={dep_target_range}"
     )
     with spack.concretize.disable_compiler_existence_check():
         spec = spack.concretize.concretize_one(spec)

lib/spack/spack/test/bindist.py

@@ -200,7 +200,11 @@ def dummy_prefix(tmpdir):
 @pytest.mark.requires_executables(*required_executables)
 @pytest.mark.maybeslow
 @pytest.mark.usefixtures(
-    "default_config", "cache_directory", "install_dir_default_layout", "temporary_mirror"
+    "default_config",
+    "cache_directory",
+    "install_dir_default_layout",
+    "temporary_mirror",
+    "mutable_mock_env_path",
 )
 def test_default_rpaths_create_install_default_layout(temporary_mirror_dir):
     """
@@ -272,7 +276,11 @@ def test_default_rpaths_install_nondefault_layout(temporary_mirror_dir):
 @pytest.mark.maybeslow
 @pytest.mark.nomockstage
 @pytest.mark.usefixtures(
-    "default_config", "cache_directory", "install_dir_default_layout", "temporary_mirror"
+    "default_config",
+    "cache_directory",
+    "install_dir_default_layout",
+    "temporary_mirror",
+    "mutable_mock_env_path",
 )
 def test_relative_rpaths_install_default_layout(temporary_mirror_dir):
     """
@@ -1132,7 +1140,7 @@ def test_get_valid_spec_file_no_json(tmp_path, filename):
 
 def test_download_tarball_with_unsupported_layout_fails(tmp_path, mutable_config, capsys):
     layout_version = bindist.CURRENT_BUILD_CACHE_LAYOUT_VERSION + 1
-    spec = Spec("gmake@4.4.1%gcc@13.1.0 arch=linux-ubuntu23.04-zen2")
+    spec = Spec("gmake@4.4.1 arch=linux-ubuntu23.04-zen2 %gcc@13.1.0")
     spec._mark_concrete()
     spec_dict = spec.to_dict()
     spec_dict["buildcache_layout_version"] = layout_version

lib/spack/spack/test/ci.py

@@ -18,6 +18,7 @@
 import spack.repo as repo
 import spack.util.git
 from spack.test.conftest import MockHTTPResponse
+from spack.version import Version
 
 pytestmark = [pytest.mark.usefixtures("mock_packages")]
 
@@ -30,6 +31,43 @@ def repro_dir(tmp_path):
         yield result
 
 
+def test_get_added_versions_new_checksum(mock_git_package_changes):
+    repo_path, filename, commits = mock_git_package_changes
+
+    checksum_versions = {
+        "3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04": Version("2.1.5"),
+        "a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a": Version("2.1.4"),
+        "6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200": Version("2.0.7"),
+        "86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8": Version("2.0.0"),
+    }
+
+    with fs.working_dir(str(repo_path)):
+        added_versions = ci.get_added_versions(
+            checksum_versions, filename, from_ref=commits[-1], to_ref=commits[-2]
+        )
+        assert len(added_versions) == 1
+        assert added_versions[0] == Version("2.1.5")
+
+
+def test_get_added_versions_new_commit(mock_git_package_changes):
+    repo_path, filename, commits = mock_git_package_changes
+
+    checksum_versions = {
+        "74253725f884e2424a0dd8ae3f69896d5377f325": Version("2.1.6"),
+        "3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04": Version("2.1.5"),
+        "a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a": Version("2.1.4"),
+        "6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200": Version("2.0.7"),
+        "86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8": Version("2.0.0"),
+    }
+
+    with fs.working_dir(str(repo_path)):
+        added_versions = ci.get_added_versions(
+            checksum_versions, filename, from_ref=commits[2], to_ref=commits[1]
+        )
+        assert len(added_versions) == 1
+        assert added_versions[0] == Version("2.1.6")
+
+
 def test_pipeline_dag(config, tmpdir):
     r"""Test creation, pruning, and traversal of PipelineDAG using the
     following package dependency graph:

lib/spack/spack/test/cmd/ci.py

@@ -22,7 +22,11 @@
 import spack.hash_types as ht
 import spack.main
 import spack.paths as spack_paths
+import spack.repo
+import spack.spec
+import spack.stage
 import spack.util.spack_yaml as syaml
+import spack.version
 from spack.ci import gitlab as gitlab_generator
 from spack.ci.common import PipelineDag, PipelineOptions, SpackCIConfig
 from spack.ci.generator_registry import generator
@@ -867,7 +871,7 @@ def test_push_to_build_cache(
             logs_dir = scratch / "logs_dir"
             logs_dir.mkdir()
             ci.copy_stage_logs_to_artifacts(concrete_spec, str(logs_dir))
-            assert "spack-build-out.txt" in os.listdir(logs_dir)
+            assert "spack-build-out.txt.gz" in os.listdir(logs_dir)
 
             dl_dir = scratch / "download_dir"
             buildcache_cmd("download", "--spec-file", json_path, "--path", str(dl_dir))
@@ -1841,3 +1845,216 @@ def test_ci_generate_alternate_target(
 
     assert pipeline_doc.startswith("unittestpipeline")
     assert "externaltest" in pipeline_doc
+
+
+@pytest.fixture
+def fetch_versions_match(monkeypatch):
+    """Fake successful checksums returned from downloaded tarballs."""
+
+    def get_checksums_for_versions(url_by_version, package_name, **kwargs):
+        pkg_cls = spack.repo.PATH.get_pkg_class(package_name)
+        return {v: pkg_cls.versions[v]["sha256"] for v in url_by_version}
+
+    monkeypatch.setattr(spack.stage, "get_checksums_for_versions", get_checksums_for_versions)
+
+
+@pytest.fixture
+def fetch_versions_invalid(monkeypatch):
+    """Fake successful checksums returned from downloaded tarballs."""
+
+    def get_checksums_for_versions(url_by_version, package_name, **kwargs):
+        return {
+            v: "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"
+            for v in url_by_version
+        }
+
+    monkeypatch.setattr(spack.stage, "get_checksums_for_versions", get_checksums_for_versions)
+
+
+@pytest.mark.parametrize("versions", [["2.1.4"], ["2.1.4", "2.1.5"]])
+def test_ci_validate_standard_versions_valid(capfd, mock_packages, fetch_versions_match, versions):
+    spec = spack.spec.Spec("diff-test")
+    pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)
+    version_list = [spack.version.Version(v) for v in versions]
+
+    assert spack.cmd.ci.validate_standard_versions(pkg, version_list)
+
+    out, err = capfd.readouterr()
+    for version in versions:
+        assert f"Validated diff-test@{version}" in out
+
+
+@pytest.mark.parametrize("versions", [["2.1.4"], ["2.1.4", "2.1.5"]])
+def test_ci_validate_standard_versions_invalid(
+    capfd, mock_packages, fetch_versions_invalid, versions
+):
+    spec = spack.spec.Spec("diff-test")
+    pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)
+    version_list = [spack.version.Version(v) for v in versions]
+
+    assert spack.cmd.ci.validate_standard_versions(pkg, version_list) is False
+
+    out, err = capfd.readouterr()
+    for version in versions:
+        assert f"Invalid checksum found diff-test@{version}" in err
+
+
+@pytest.mark.parametrize("versions", [[("1.0", -2)], [("1.1", -4), ("2.0", -6)]])
+def test_ci_validate_git_versions_valid(
+    capfd, monkeypatch, mock_packages, mock_git_version_info, versions
+):
+    spec = spack.spec.Spec("diff-test")
+    pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)
+    version_list = [spack.version.Version(v) for v, _ in versions]
+
+    repo_path, filename, commits = mock_git_version_info
+    version_commit_dict = {
+        spack.version.Version(v): {"tag": f"v{v}", "commit": commits[c]} for v, c in versions
+    }
+
+    pkg_class = spec.package_class
+
+    monkeypatch.setattr(pkg_class, "git", repo_path)
+    monkeypatch.setattr(pkg_class, "versions", version_commit_dict)
+
+    assert spack.cmd.ci.validate_git_versions(pkg, version_list)
+
+    out, err = capfd.readouterr()
+    for version in version_list:
+        assert f"Validated diff-test@{version}" in out
+
+
+@pytest.mark.parametrize("versions", [[("1.0", -3)], [("1.1", -5), ("2.0", -5)]])
+def test_ci_validate_git_versions_bad_tag(
+    capfd, monkeypatch, mock_packages, mock_git_version_info, versions
+):
+    spec = spack.spec.Spec("diff-test")
+    pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)
+    version_list = [spack.version.Version(v) for v, _ in versions]
+
+    repo_path, filename, commits = mock_git_version_info
+    version_commit_dict = {
+        spack.version.Version(v): {"tag": f"v{v}", "commit": commits[c]} for v, c in versions
+    }
+
+    pkg_class = spec.package_class
+
+    monkeypatch.setattr(pkg_class, "git", repo_path)
+    monkeypatch.setattr(pkg_class, "versions", version_commit_dict)
+
+    assert spack.cmd.ci.validate_git_versions(pkg, version_list) is False
+
+    out, err = capfd.readouterr()
+    for version in version_list:
+        assert f"Mismatched tag <-> commit found for diff-test@{version}" in err
+
+
+@pytest.mark.parametrize("versions", [[("1.0", -2)], [("1.1", -4), ("2.0", -6), ("3.0", -6)]])
+def test_ci_validate_git_versions_invalid(
+    capfd, monkeypatch, mock_packages, mock_git_version_info, versions
+):
+    spec = spack.spec.Spec("diff-test")
+    pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)
+    version_list = [spack.version.Version(v) for v, _ in versions]
+
+    repo_path, filename, commits = mock_git_version_info
+    version_commit_dict = {
+        spack.version.Version(v): {
+            "tag": f"v{v}",
+            "commit": "abcdefabcdefabcdefabcdefabcdefabcdefabc",
+        }
+        for v, c in versions
+    }
+
+    pkg_class = spec.package_class
+
+    monkeypatch.setattr(pkg_class, "git", repo_path)
+    monkeypatch.setattr(pkg_class, "versions", version_commit_dict)
+
+    assert spack.cmd.ci.validate_git_versions(pkg, version_list) is False
+
+    out, err = capfd.readouterr()
+    for version in version_list:
+        assert f"Invalid commit for diff-test@{version}" in err
+
+
+@pytest.fixture
+def verify_standard_versions_valid(monkeypatch):
+    def validate_standard_versions(pkg, versions):
+        for version in versions:
+            print(f"Validated {pkg.name}@{version}")
+        return True
+
+    monkeypatch.setattr(spack.cmd.ci, "validate_standard_versions", validate_standard_versions)
+
+
+@pytest.fixture
+def verify_git_versions_valid(monkeypatch):
+    def validate_git_versions(pkg, versions):
+        for version in versions:
+            print(f"Validated {pkg.name}@{version}")
+        return True
+
+    monkeypatch.setattr(spack.cmd.ci, "validate_git_versions", validate_git_versions)
+
+
+@pytest.fixture
+def verify_standard_versions_invalid(monkeypatch):
+    def validate_standard_versions(pkg, versions):
+        for version in versions:
+            print(f"Invalid checksum found {pkg.name}@{version}")
+        return False
+
+    monkeypatch.setattr(spack.cmd.ci, "validate_standard_versions", validate_standard_versions)
+
+
+@pytest.fixture
+def verify_git_versions_invalid(monkeypatch):
+    def validate_git_versions(pkg, versions):
+        for version in versions:
+            print(f"Invalid commit for {pkg.name}@{version}")
+        return False
+
+    monkeypatch.setattr(spack.cmd.ci, "validate_git_versions", validate_git_versions)
+
+
+def test_ci_verify_versions_valid(
+    monkeypatch,
+    mock_packages,
+    mock_git_package_changes,
+    verify_standard_versions_valid,
+    verify_git_versions_valid,
+):
+    repo_path, _, commits = mock_git_package_changes
+    monkeypatch.setattr(spack.paths, "prefix", repo_path)
+
+    out = ci_cmd("verify-versions", commits[-1], commits[-3])
+    assert "Validated diff-test@2.1.5" in out
+    assert "Validated diff-test@2.1.6" in out
+
+
+def test_ci_verify_versions_standard_invalid(
+    monkeypatch,
+    mock_packages,
+    mock_git_package_changes,
+    verify_standard_versions_invalid,
+    verify_git_versions_invalid,
+):
+    repo_path, _, commits = mock_git_package_changes
+
+    monkeypatch.setattr(spack.paths, "prefix", repo_path)
+
+    out = ci_cmd("verify-versions", commits[-1], commits[-3], fail_on_error=False)
+    assert "Invalid checksum found diff-test@2.1.5" in out
+    assert "Invalid commit for diff-test@2.1.6" in out
+
+
+def test_ci_verify_versions_manual_package(monkeypatch, mock_packages, mock_git_package_changes):
+    repo_path, _, commits = mock_git_package_changes
+    monkeypatch.setattr(spack.paths, "prefix", repo_path)
+
+    pkg_class = spack.spec.Spec("diff-test").package_class
+    monkeypatch.setattr(pkg_class, "manual_download", True)
+
+    out = ci_cmd("verify-versions", commits[-1], commits[-2])
+    assert "Skipping manual download package: diff-test" in out

lib/spack/spack/test/cmd/commands.py

@@ -5,6 +5,7 @@
 import filecmp
 import os
 import shutil
+import textwrap
 
 import pytest
 
@@ -259,15 +260,25 @@ def test_update_completion_arg(shell, tmpdir, monkeypatch):
 def test_updated_completion_scripts(shell, tmpdir):
     """Make sure our shell tab completion scripts remain up-to-date."""
 
-    msg = (
+    width = 72
+    lines = textwrap.wrap(
         "It looks like Spack's command-line interface has been modified. "
-        "Please update Spack's shell tab completion scripts by running:\n\n"
-        "    spack commands --update-completion\n\n"
-        "and adding the changed files to your pull request."
+        "If differences are more than your global 'include:' scopes, please "
+        "update Spack's shell tab completion scripts by running:",
+        width,
     )
+    lines.append("\n    spack commands --update-completion\n")
+    lines.extend(
+        textwrap.wrap(
+            "and adding the changed files (minus your global 'include:' scopes) "
+            "to your pull request.",
+            width,
+        )
+    )
+    msg = "\n".join(lines)
 
     header = os.path.join(spack.paths.share_path, shell, f"spack-completion.{shell}")
-    script = "spack-completion.{0}".format(shell)
+    script = f"spack-completion.{shell}"
     old_script = os.path.join(spack.paths.share_path, script)
     new_script = str(tmpdir.join(script))
 

lib/spack/spack/test/cmd/config.py

@@ -213,7 +213,7 @@ def test_config_add_update_dict(mutable_empty_config):
 
 def test_config_with_c_argument(mutable_empty_config):
     # I don't know how to add a spack argument to a Spack Command, so we test this way
-    config_file = "config:install_root:root:/path/to/config.yaml"
+    config_file = "config:install_tree:root:/path/to/config.yaml"
     parser = spack.main.make_argument_parser()
     args = parser.parse_args(["-c", config_file])
     assert config_file in args.config_vars
@@ -221,7 +221,7 @@ def test_config_with_c_argument(mutable_empty_config):
     # Add the path to the config
     config("add", args.config_vars[0], scope="command_line")
     output = config("get", "config")
-    assert "config:\n  install_root:\n    root: /path/to/config.yaml" in output
+    assert "config:\n  install_tree:\n    root: /path/to/config.yaml" in output
 
 
 def test_config_add_ordered_dict(mutable_empty_config):

lib/spack/spack/test/cmd/deprecate.py

@@ -15,6 +15,9 @@
 deprecate = SpackCommand("deprecate")
 find = SpackCommand("find")
 
+# Unit tests should not be affected by the user's managed environments
+pytestmark = pytest.mark.usefixtures("mutable_mock_env_path")
+
 
 def test_deprecate(mock_packages, mock_archive, mock_fetch, install_mockery):
     install("--fake", "libelf@0.8.13")

lib/spack/spack/test/cmd/develop.py

@@ -16,6 +16,7 @@
 import spack.stage
 import spack.util.git
 import spack.util.path
+from spack.error import SpackError
 from spack.main import SpackCommand
 
 add = SpackCommand("add")
@@ -159,6 +160,7 @@ def check_path(stage, dest):
             # Create path to allow develop to modify env
             fs.mkdirp(abspath)
             develop("--no-clone", "-p", path, "mpich@1.0")
+            self.check_develop(e, spack.spec.Spec("mpich@=1.0"), path)
 
             # Remove path to ensure develop with no args runs staging code
             os.rmdir(abspath)
@@ -218,6 +220,40 @@ def test_develop_full_git_repo(
         assert len(commits) > 1
 
 
+def test_recursive(mutable_mock_env_path, install_mockery, mock_fetch):
+    env("create", "test")
+
+    with ev.read("test") as e:
+        add("indirect-mpich@1.0")
+        e.concretize()
+        specs = e.all_specs()
+
+        assert len(specs) > 1
+        develop("--recursive", "mpich")
+
+        expected_dev_specs = ["mpich", "direct-mpich", "indirect-mpich"]
+        for spec in expected_dev_specs:
+            assert spec in e.dev_specs
+
+
+def test_develop_fails_with_multiple_concrete_versions(
+    mutable_mock_env_path, install_mockery, mock_fetch
+):
+    env("create", "test")
+
+    with ev.read("test") as e:
+        add("indirect-mpich@1.0")
+        add("indirect-mpich@0.9")
+        e.unify = False
+        e.concretize()
+
+        with pytest.raises(SpackError) as develop_error:
+            develop("indirect-mpich", fail_on_error=True)
+
+        error_str = "has multiple concrete instances in the graph"
+        assert error_str in str(develop_error.value)
+
+
 def test_concretize_dev_path_with_at_symbol_in_env(mutable_mock_env_path, tmpdir, mock_packages):
     spec_like = "develop-test@develop"
 

lib/spack/spack/test/cmd/env.py

@@ -1067,13 +1067,17 @@ def test_init_from_yaml_relative_includes(tmp_path):
         assert os.path.exists(os.path.join(e2.path, f))
 
 
+# TODO: Should we be supporting relative path rewrites when creating new env from existing?
+# TODO: If so, then this should confirm that the absolute include paths in the new env exist.
 def test_init_from_yaml_relative_includes_outside_env(tmp_path):
-    files = ["../outside_env_not_copied/repos.yaml"]
+    """Ensure relative includes to files outside the environment fail."""
+    files = ["../outside_env/repos.yaml"]
 
     manifest = f"""
 spack:
   specs: []
-  include: {files}
+  include:
+  - path: {files[0]}
 """
 
     # subdir to ensure parent of environment dir is not shared
@@ -1086,7 +1090,7 @@ def test_init_from_yaml_relative_includes_outside_env(tmp_path):
     for f in files:
         fs.touchp(e1_path / f)
 
-    with pytest.raises(spack.config.ConfigFileError, match="Detected 1 missing include"):
+    with pytest.raises(ValueError, match="does not exist"):
         _ = _env_create("test2", init_file=e1_manifest)
 
 
@@ -1186,14 +1190,14 @@ def test_env_with_config(environment_from_manifest):
 
 
 def test_with_config_bad_include_create(environment_from_manifest):
-    """Confirm missing include paths raise expected exception and error."""
-    with pytest.raises(spack.config.ConfigFileError, match="2 missing include path"):
+    """Confirm missing required include raises expected exception."""
+    err = "does not exist"
+    with pytest.raises(ValueError, match=err):
         environment_from_manifest(
             """
 spack:
   include:
   - /no/such/directory
-  - no/such/file.yaml
 """
         )
 
@@ -1203,34 +1207,25 @@ def test_with_config_bad_include_activate(environment_from_manifest, tmpdir):
     include1 = env_root / "include1.yaml"
     include1.touch()
 
-    abs_include_path = os.path.abspath(tmpdir.join("subdir").ensure("include2.yaml"))
-
     spack_yaml = env_root / ev.manifest_name
     spack_yaml.write_text(
-        f"""
+        """
 spack:
   include:
   - ./include1.yaml
-  - {abs_include_path}
 """
     )
 
     with ev.Environment(env_root) as e:
         e.concretize()
 
-    # we've created an environment with some included config files (which do
-    # in fact exist): now we remove them and check that we get a sensible
-    # error message
+    # We've created an environment with included config file (which does
+    # exist). Now we remove it and check that we get a sensible error.
 
-    os.remove(abs_include_path)
     os.remove(include1)
-    with pytest.raises(spack.config.ConfigFileError) as exc:
+    with pytest.raises(ValueError, match="does not exist"):
         ev.activate(ev.Environment(env_root))
 
-    err = exc.value.message
-    assert "missing include" in err
-    assert abs_include_path in err
-    assert "include1.yaml" in err
     assert ev.active_environment() is None
 
 
@@ -1338,8 +1333,10 @@ def test_config_change_existing(mutable_mock_env_path, tmp_path, mock_packages,
     included file scope.
     """
 
+    env_path = tmp_path / "test_config"
+    fs.mkdirp(env_path)
     included_file = "included-packages.yaml"
-    included_path = tmp_path / included_file
+    included_path = env_path / included_file
     with open(included_path, "w", encoding="utf-8") as f:
         f.write(
             """\
@@ -1355,7 +1352,7 @@ def test_config_change_existing(mutable_mock_env_path, tmp_path, mock_packages,
 """
         )
 
-    spack_yaml = tmp_path / ev.manifest_name
+    spack_yaml = env_path / ev.manifest_name
     spack_yaml.write_text(
         f"""\
 spack:
@@ -1369,7 +1366,8 @@ def test_config_change_existing(mutable_mock_env_path, tmp_path, mock_packages,
 """
     )
 
-    e = ev.Environment(tmp_path)
+    mutable_config.set("config:misc_cache", str(tmp_path / "cache"))
+    e = ev.Environment(env_path)
     with e:
         # List of requirements, flip a variant
         config("change", "packages:mpich:require:~debug")
@@ -1459,19 +1457,6 @@ def test_env_with_included_config_file_url(tmpdir, mutable_empty_config, package
     assert cfg["mpileaks"]["version"] == ["2.2"]
 
 
-def test_env_with_included_config_missing_file(tmpdir, mutable_empty_config):
-    """Test inclusion of a missing configuration file raises FetchError
-    noting missing file."""
-
-    spack_yaml = tmpdir.join("spack.yaml")
-    missing_file = tmpdir.join("packages.yaml")
-    with spack_yaml.open("w") as f:
-        f.write("spack:\n  include:\n    - {0}\n".format(missing_file.strpath))
-
-    with pytest.raises(spack.error.ConfigError, match="missing include path"):
-        ev.Environment(tmpdir.strpath)
-
-
 def test_env_with_included_config_scope(mutable_mock_env_path, packages_file):
     """Test inclusion of a package file from the environment's configuration
     stage directory. This test is intended to represent a case where a remote
@@ -1566,7 +1551,7 @@ def test_env_with_included_config_precedence(tmp_path):
 
 
 def test_env_with_included_configs_precedence(tmp_path):
-    """Test precendence of multiple included configuration files."""
+    """Test precedence of multiple included configuration files."""
     file1 = "high-config.yaml"
     file2 = "low-config.yaml"
 
@@ -1794,7 +1779,7 @@ def test_roots_display_with_variants():
     with ev.read("test"):
         out = find(output=str)
 
-    assert "boost +shared" in out
+    assert "boost+shared" in out
 
 
 def test_uninstall_keeps_in_env(mock_stage, mock_fetch, install_mockery):
@@ -3080,14 +3065,26 @@ def test_stack_view_activate_from_default(
 
 def test_envvar_set_in_activate(tmp_path, mock_packages, install_mockery):
     spack_yaml = tmp_path / "spack.yaml"
+    env_vars_yaml = tmp_path / "env_vars.yaml"
+
+    env_vars_yaml.write_text(
+        """
+env_vars:
+  set:
+    CONFIG_ENVAR_SET_IN_ENV_LOAD: "True"
+"""
+    )
+
     spack_yaml.write_text(
         """
 spack:
+  include:
+  - env_vars.yaml
   specs:
     - cmake%gcc
   env_vars:
     set:
-      ENVAR_SET_IN_ENV_LOAD: "True"
+      SPACK_ENVAR_SET_IN_ENV_LOAD: "True"
 """
     )
 
@@ -3098,12 +3095,16 @@ def test_envvar_set_in_activate(tmp_path, mock_packages, install_mockery):
     test_env = ev.read("test")
     output = env("activate", "--sh", "test")
 
-    assert "ENVAR_SET_IN_ENV_LOAD=True" in output
+    assert "SPACK_ENVAR_SET_IN_ENV_LOAD=True" in output
+    assert "CONFIG_ENVAR_SET_IN_ENV_LOAD=True" in output
 
     with test_env:
-        with spack.util.environment.set_env(ENVAR_SET_IN_ENV_LOAD="True"):
+        with spack.util.environment.set_env(
+            SPACK_ENVAR_SET_IN_ENV_LOAD="True", CONFIG_ENVAR_SET_IN_ENV_LOAD="True"
+        ):
             output = env("deactivate", "--sh")
-            assert "unset ENVAR_SET_IN_ENV_LOAD" in output
+            assert "unset SPACK_ENVAR_SET_IN_ENV_LOAD" in output
+            assert "unset CONFIG_ENVAR_SET_IN_ENV_LOAD" in output
 
 
 def test_stack_view_no_activate_without_default(
@@ -4277,21 +4278,31 @@ def test_unify_when_possible_works_around_conflicts():
     assert len([x for x in e.all_specs() if x.satisfies("mpich")]) == 1
 
 
+# Using mock_include_cache to ensure the "remote" file is cached in a temporary
+# location and not polluting the user cache.
 def test_env_include_packages_url(
-    tmpdir, mutable_empty_config, mock_spider_configs, mock_curl_configs
+    tmpdir, mutable_empty_config, mock_fetch_url_text, mock_curl_configs, mock_include_cache
 ):
     """Test inclusion of a (GitHub) URL."""
     develop_url = "https://github.com/fake/fake/blob/develop/"
     default_packages = develop_url + "etc/fake/defaults/packages.yaml"
+    sha256 = "a422e35b3a18869d0611a4137b37314131749ecdc070a7cd7183f488da81201a"
     spack_yaml = tmpdir.join("spack.yaml")
     with spack_yaml.open("w") as f:
-        f.write("spack:\n  include:\n    - {0}\n".format(default_packages))
-    assert os.path.isfile(spack_yaml.strpath)
+        f.write(
+            f"""\
+spack:
+  include:
+  - path: {default_packages}
+    sha256: {sha256}
+"""
+        )
 
     with spack.config.override("config:url_fetch_method", "curl"):
         env = ev.Environment(tmpdir.strpath)
         ev.activate(env)
 
+        # Make sure a setting from test/data/config/packages.yaml is present
         cfg = spack.config.get("packages")
         assert "mpich" in cfg["all"]["providers"]["mpi"]
 
@@ -4360,7 +4371,7 @@ def test_env_view_disabled(tmp_path, mutable_mock_env_path):
 
 
 @pytest.mark.parametrize("first", ["false", "true", "custom"])
-def test_env_include_mixed_views(tmp_path, mutable_mock_env_path, mutable_config, first):
+def test_env_include_mixed_views(tmp_path, mutable_config, mutable_mock_env_path, first):
     """Ensure including path and boolean views in different combinations result
     in the creation of only the first view if it is not disabled."""
     false_yaml = tmp_path / "false-view.yaml"

lib/spack/spack/test/cmd/find.py

@@ -320,7 +320,7 @@ def test_find_very_long(database, config):
 @pytest.mark.db
 def test_find_show_compiler(database, config):
     output = find("--no-groups", "--show-full-compiler", "mpileaks")
-    assert "mpileaks@2.3%gcc@10.2.1" in output
+    assert "mpileaks@2.3 %gcc@10.2.1" in output
 
 
 @pytest.mark.db
@@ -464,7 +464,7 @@ def test_environment_with_version_range_in_compiler_doesnt_fail(tmp_path):
 
     with test_environment:
         output = find()
-    assert "zlib%gcc@12.1.0" in output
+    assert "zlib %gcc@12.1.0" in output
 
 
 _pkga = (

lib/spack/spack/test/cmd/install.py

@@ -718,10 +718,11 @@ def test_install_deps_then_package(tmpdir, mock_fetch, install_mockery):
     assert os.path.exists(root.prefix)
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701")
 @pytest.mark.regression("12002")
 def test_install_only_dependencies_in_env(
-    tmpdir, mock_fetch, install_mockery, mutable_mock_env_path
+    tmpdir, mutable_mock_env_path, mock_fetch, install_mockery
 ):
     env("create", "test")
 
@@ -735,9 +736,10 @@ def test_install_only_dependencies_in_env(
         assert not os.path.exists(root.prefix)
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.regression("12002")
 def test_install_only_dependencies_of_all_in_env(
-    tmpdir, mock_fetch, install_mockery, mutable_mock_env_path
+    tmpdir, mutable_mock_env_path, mock_fetch, install_mockery
 ):
     env("create", "--without-view", "test")
 
@@ -757,7 +759,8 @@ def test_install_only_dependencies_of_all_in_env(
                 assert os.path.exists(dep.prefix)
 
 
-def test_install_no_add_in_env(tmpdir, mock_fetch, install_mockery, mutable_mock_env_path):
+# Unit tests should not be affected by the user's managed environments
+def test_install_no_add_in_env(tmpdir, mutable_mock_env_path, mock_fetch, install_mockery):
     # To test behavior of --add option, we create the following environment:
     #
     #     mpileaks
@@ -932,9 +935,10 @@ def test_install_fails_no_args_suggests_env_activation(tmpdir):
     assert "using the `spack.yaml` in this directory" in output
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701")
 def test_install_env_with_tests_all(
-    tmpdir, mock_packages, mock_fetch, install_mockery, mutable_mock_env_path
+    tmpdir, mutable_mock_env_path, mock_packages, mock_fetch, install_mockery
 ):
     env("create", "test")
     with ev.read("test"):
@@ -944,9 +948,10 @@ def test_install_env_with_tests_all(
         assert os.path.exists(test_dep.prefix)
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701")
 def test_install_env_with_tests_root(
-    tmpdir, mock_packages, mock_fetch, install_mockery, mutable_mock_env_path
+    tmpdir, mutable_mock_env_path, mock_packages, mock_fetch, install_mockery
 ):
     env("create", "test")
     with ev.read("test"):
@@ -956,9 +961,10 @@ def test_install_env_with_tests_root(
         assert not os.path.exists(test_dep.prefix)
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.not_on_windows("Environment views not supported on windows. Revisit after #34701")
 def test_install_empty_env(
-    tmpdir, mock_packages, mock_fetch, install_mockery, mutable_mock_env_path
+    tmpdir, mutable_mock_env_path, mock_packages, mock_fetch, install_mockery
 ):
     env_name = "empty"
     env("create", env_name)
@@ -994,9 +1000,17 @@ def test_installation_fail_tests(install_mockery, mock_fetch, name, method):
     assert "See test log for details" in output
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.not_on_windows("Buildcache not supported on windows")
 def test_install_use_buildcache(
-    capsys, mock_packages, mock_fetch, mock_archive, mock_binary_index, tmpdir, install_mockery
+    capsys,
+    mutable_mock_env_path,
+    mock_packages,
+    mock_fetch,
+    mock_archive,
+    mock_binary_index,
+    tmpdir,
+    install_mockery,
 ):
     """
     Make sure installing with use-buildcache behaves correctly.

lib/spack/spack/test/cmd/mark.py

@@ -12,6 +12,9 @@
 install = SpackCommand("install")
 uninstall = SpackCommand("uninstall")
 
+# Unit tests should not be affected by the user's managed environments
+pytestmark = pytest.mark.usefixtures("mutable_mock_env_path")
+
 
 @pytest.mark.db
 def test_mark_mode_required(mutable_database):

lib/spack/spack/test/cmd/mirror.py

@@ -38,8 +38,9 @@ def test_regression_8083(tmpdir, capfd, mock_packages, mock_fetch, config):
     assert "as it is an external spec" in output
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.regression("12345")
-def test_mirror_from_env(tmp_path, mock_packages, mock_fetch, mutable_mock_env_path):
+def test_mirror_from_env(mutable_mock_env_path, tmp_path, mock_packages, mock_fetch):
     mirror_dir = str(tmp_path / "mirror")
     env_name = "test"
 
@@ -342,8 +343,16 @@ def test_mirror_name_collision(mutable_config):
         mirror("add", "first", "1")
 
 
+# Unit tests should not be affected by the user's managed environments
 def test_mirror_destroy(
-    install_mockery, mock_packages, mock_fetch, mock_archive, mutable_config, monkeypatch, tmpdir
+    mutable_mock_env_path,
+    install_mockery,
+    mock_packages,
+    mock_fetch,
+    mock_archive,
+    mutable_config,
+    monkeypatch,
+    tmpdir,
 ):
     # Create a temp mirror directory for buildcache usage
     mirror_dir = tmpdir.join("mirror_dir")

lib/spack/spack/test/cmd/spec.py

@@ -13,7 +13,10 @@
 import spack.store
 from spack.main import SpackCommand, SpackCommandError
 
-pytestmark = pytest.mark.usefixtures("mutable_config", "mutable_mock_repo")
+# Unit tests should not be affected by the user's managed environments
+pytestmark = pytest.mark.usefixtures(
+    "mutable_mock_env_path", "mutable_config", "mutable_mock_repo"
+)
 
 spec = SpackCommand("spec")
 

lib/spack/spack/test/cmd/style.py

@@ -409,3 +409,108 @@ def test_case_sensitive_imports(tmp_path: pathlib.Path):
 def test_pkg_imports():
     assert spack.cmd.style._module_part(spack.paths.prefix, "spack.pkg.builtin.boost") is None
     assert spack.cmd.style._module_part(spack.paths.prefix, "spack.pkg") is None
+
+
+def test_spec_strings(tmp_path):
+    (tmp_path / "example.py").write_text(
+        """\
+def func(x):
+    print("dont fix %s me" % x, 3)
+    return x.satisfies("+foo %gcc +bar") and x.satisfies("%gcc +baz")
+"""
+    )
+    (tmp_path / "example.json").write_text(
+        """\
+{
+    "spec": [
+        "+foo %gcc +bar~nope   ^dep %clang +yup @3.2 target=x86_64 /abcdef ^another   %gcc   ",
+        "%gcc +baz"
+    ],
+    "%gcc x=y": 2
+}
+"""
+    )
+    (tmp_path / "example.yaml").write_text(
+        """\
+spec:
+  - "+foo   %gcc +bar"
+  - "%gcc +baz"
+  - "this is fine %clang"
+"%gcc x=y": 2
+"""
+    )
+
+    issues = set()
+
+    def collect_issues(path: str, line: int, col: int, old: str, new: str):
+        issues.add((path, line, col, old, new))
+
+    # check for issues with custom handler
+    spack.cmd.style._check_spec_strings(
+        [
+            str(tmp_path / "nonexistent.py"),
+            str(tmp_path / "example.py"),
+            str(tmp_path / "example.json"),
+            str(tmp_path / "example.yaml"),
+        ],
+        handler=collect_issues,
+    )
+
+    assert issues == {
+        (
+            str(tmp_path / "example.json"),
+            3,
+            9,
+            "+foo %gcc +bar~nope   ^dep %clang +yup @3.2 target=x86_64 /abcdef ^another   %gcc   ",
+            "+foo +bar~nope %gcc   ^dep +yup @3.2 target=x86_64 /abcdef %clang ^another   %gcc   ",
+        ),
+        (str(tmp_path / "example.json"), 4, 9, "%gcc +baz", "+baz %gcc"),
+        (str(tmp_path / "example.json"), 6, 5, "%gcc x=y", "x=y %gcc"),
+        (str(tmp_path / "example.py"), 3, 23, "+foo %gcc +bar", "+foo +bar %gcc"),
+        (str(tmp_path / "example.py"), 3, 57, "%gcc +baz", "+baz %gcc"),
+        (str(tmp_path / "example.yaml"), 2, 5, "+foo   %gcc +bar", "+foo +bar   %gcc"),
+        (str(tmp_path / "example.yaml"), 3, 5, "%gcc +baz", "+baz %gcc"),
+        (str(tmp_path / "example.yaml"), 5, 1, "%gcc x=y", "x=y %gcc"),
+    }
+
+    # fix the issues in the files
+    spack.cmd.style._check_spec_strings(
+        [
+            str(tmp_path / "nonexistent.py"),
+            str(tmp_path / "example.py"),
+            str(tmp_path / "example.json"),
+            str(tmp_path / "example.yaml"),
+        ],
+        handler=spack.cmd.style._spec_str_fix_handler,
+    )
+
+    assert (
+        (tmp_path / "example.json").read_text()
+        == """\
+{
+    "spec": [
+        "+foo +bar~nope %gcc   ^dep +yup @3.2 target=x86_64 /abcdef %clang ^another   %gcc   ",
+        "+baz %gcc"
+    ],
+    "x=y %gcc": 2
+}
+"""
+    )
+    assert (
+        (tmp_path / "example.py").read_text()
+        == """\
+def func(x):
+    print("dont fix %s me" % x, 3)
+    return x.satisfies("+foo +bar %gcc") and x.satisfies("+baz %gcc")
+"""
+    )
+    assert (
+        (tmp_path / "example.yaml").read_text()
+        == """\
+spec:
+  - "+foo +bar   %gcc"
+  - "+baz %gcc"
+  - "this is fine %clang"
+"x=y %gcc": 2
+"""
+    )

lib/spack/spack/test/cmd/uninstall.py

@@ -16,6 +16,9 @@
 uninstall = SpackCommand("uninstall")
 install = SpackCommand("install")
 
+# Unit tests should not be affected by the user's managed environments
+pytestmark = pytest.mark.usefixtures("mutable_mock_env_path")
+
 
 class MockArgs:
     def __init__(self, packages, all=False, force=False, dependents=False):
@@ -220,9 +223,7 @@ class TestUninstallFromEnv:
     find = SpackCommand("find")
 
     @pytest.fixture(scope="function")
-    def environment_setup(
-        self, mutable_mock_env_path, mock_packages, mutable_database, install_mockery
-    ):
+    def environment_setup(self, mock_packages, mutable_database, install_mockery):
         TestUninstallFromEnv.env("create", "e1")
         e1 = spack.environment.read("e1")
         with e1:

lib/spack/spack/test/cmd/versions.py

@@ -59,13 +59,6 @@ def mock_fetch_remote_versions(*args, **kwargs):
     assert v.strip(" \n\t") == "99.99.99\n  3.2.1"
 
 
-@pytest.mark.maybeslow
-def test_no_versions():
-    """Test a package for which no remote versions are available."""
-
-    versions("converge")
-
-
 @pytest.mark.maybeslow
 def test_no_unchecksummed_versions():
     """Test a package for which no unchecksummed versions are available."""

lib/spack/spack/test/concretization/compiler_runtimes.py

@@ -92,7 +92,7 @@ def test_external_nodes_do_not_have_runtimes(runtime_repo, mutable_config, tmp_p
         # Same as before, but tests that we can reuse from a more generic target
         pytest.param(
             "pkg-a%gcc@9.4.0",
-            "pkg-b%gcc@10.2.1 target=x86_64",
+            "pkg-b target=x86_64 %gcc@10.2.1",
             {"pkg-a": "gcc-runtime@9.4.0", "pkg-b": "gcc-runtime@9.4.0"},
             1,
             marks=pytest.mark.skipif(
@@ -101,7 +101,7 @@ def test_external_nodes_do_not_have_runtimes(runtime_repo, mutable_config, tmp_p
         ),
         pytest.param(
             "pkg-a%gcc@10.2.1",
-            "pkg-b%gcc@9.4.0 target=x86_64",
+            "pkg-b target=x86_64 %gcc@9.4.0",
             {
                 "pkg-a": "gcc-runtime@10.2.1 target=x86_64",
                 "pkg-b": "gcc-runtime@9.4.0 target=x86_64",

lib/spack/spack/test/concretization/core.py

@@ -121,7 +121,7 @@ def binary_compatibility(monkeypatch, request):
         "mpileaks ^mpi@1.2:2",
         # conflict not triggered
         "conflict",
-        "conflict%clang~foo",
+        "conflict~foo%clang",
         "conflict-parent%gcc",
     ]
 )
@@ -387,8 +387,8 @@ def test_different_compilers_get_different_flags(
         t = archspec.cpu.host().family
         client = spack.concretize.concretize_one(
             Spec(
-                f"cmake-client %gcc@11.1.0 platform=test os=redhat6 target={t}"
-                f" ^cmake %clang@12.2.0 platform=test os=redhat6 target={t}"
+                f"cmake-client platform=test os=redhat6 target={t} %gcc@11.1.0"
+                f" ^cmake platform=test os=redhat6 target={t} %clang@12.2.0"
             )
         )
         cmake = client["cmake"]
@@ -403,7 +403,7 @@ def test_spec_flags_maintain_order(self, mutable_config, gcc11_with_flags):
         for successive concretizations.
         """
         mutable_config.set("compilers", [gcc11_with_flags])
-        spec_str = "libelf %gcc@11.1.0 os=redhat6"
+        spec_str = "libelf os=redhat6 %gcc@11.1.0"
         for _ in range(3):
             s = spack.concretize.concretize_one(spec_str)
             assert all(
@@ -414,7 +414,7 @@ def test_compiler_flags_differ_identical_compilers(self, mutable_config, clang12
         mutable_config.set("compilers", [clang12_with_flags])
         # Correct arch to use test compiler that has flags
         t = archspec.cpu.host().family
-        spec = Spec(f"pkg-a %clang@12.2.0 platform=test os=redhat6 target={t}")
+        spec = Spec(f"pkg-a platform=test os=redhat6 target={t} %clang@12.2.0")
 
         # Get the compiler that matches the spec (
         compiler = spack.compilers.compiler_for_spec("clang@=12.2.0", spec.architecture)
@@ -488,13 +488,13 @@ def test_architecture_deep_inheritance(self, mock_targets, compiler_factory):
         # CNL compiler has no target attribute, and this is essential to make detection pass
         del cnl_compiler["compiler"]["target"]
         with spack.config.override("compilers", [cnl_compiler]):
-            spec_str = "mpileaks %gcc@4.5.0 os=CNL target=nocona ^dyninst os=CNL ^callpath os=CNL"
+            spec_str = "mpileaks os=CNL target=nocona %gcc@4.5.0 ^dyninst os=CNL ^callpath os=CNL"
             spec = spack.concretize.concretize_one(spec_str)
             for s in spec.traverse(root=False):
                 assert s.architecture.target == spec.architecture.target
 
     def test_compiler_flags_from_user_are_grouped(self):
-        spec = Spec('pkg-a%gcc cflags="-O -foo-flag foo-val" platform=test')
+        spec = Spec('pkg-a cflags="-O -foo-flag foo-val" platform=test %gcc')
         spec = spack.concretize.concretize_one(spec)
         cflags = spec.compiler_flags["cflags"]
         assert any(x == "-foo-flag foo-val" for x in cflags)
@@ -804,7 +804,7 @@ def test_external_and_virtual(self, mutable_config):
         assert spec["stuff"].compiler.satisfies("gcc")
 
     def test_compiler_child(self):
-        s = Spec("mpileaks%clang target=x86_64 ^dyninst%gcc")
+        s = Spec("mpileaks target=x86_64 %clang ^dyninst%gcc")
         s = spack.concretize.concretize_one(s)
         assert s["mpileaks"].satisfies("%clang")
         assert s["dyninst"].satisfies("%gcc")
@@ -826,7 +826,7 @@ def test_conflict_in_all_directives_true(self):
         with pytest.raises(spack.error.SpackError):
             s = spack.concretize.concretize_one(s)
 
-    @pytest.mark.parametrize("spec_str", ["conflict@10.0%clang+foo"])
+    @pytest.mark.parametrize("spec_str", ["conflict@10.0+foo%clang"])
     def test_no_conflict_in_external_specs(self, spec_str):
         # Modify the configuration to have the spec with conflict
         # registered as an external
@@ -940,9 +940,9 @@ def test_noversion_pkg(self, spec):
         "spec, best_achievable",
         [
             ("mpileaks%gcc@=4.4.7 ^dyninst@=10.2.1 target=x86_64:", "core2"),
-            ("mpileaks%gcc@=4.8 target=x86_64:", "haswell"),
-            ("mpileaks%gcc@=5.3.0 target=x86_64:", "broadwell"),
-            ("mpileaks%apple-clang@=5.1.0 target=x86_64:", "x86_64"),
+            ("mpileaks target=x86_64: %gcc@=4.8", "haswell"),
+            ("mpileaks target=x86_64: %gcc@=5.3.0", "broadwell"),
+            ("mpileaks target=x86_64: %apple-clang@=5.1.0", "x86_64"),
         ],
     )
     @pytest.mark.regression("13361", "20537")
@@ -1286,7 +1286,7 @@ def test_compiler_match_is_preferred_to_newer_version(self, compiler_factory):
         with spack.config.override(
             "compilers", [compiler_factory(spec="gcc@10.1.0", operating_system="redhat6")]
         ):
-            spec_str = "simple-inheritance+openblas %gcc@10.1.0 os=redhat6"
+            spec_str = "simple-inheritance+openblas os=redhat6 %gcc@10.1.0"
             s = spack.concretize.concretize_one(spec_str)
             assert "openblas@0.2.15" in s
             assert s["openblas"].satisfies("%gcc@10.1.0")
@@ -1319,7 +1319,7 @@ def test_custom_compiler_version(self, mutable_config, compiler_factory, monkeyp
             "compilers", [compiler_factory(spec="gcc@10foo", operating_system="redhat6")]
         )
         monkeypatch.setattr(spack.compiler.Compiler, "real_version", "10.2.1")
-        s = spack.concretize.concretize_one("pkg-a %gcc@10foo os=redhat6")
+        s = spack.concretize.concretize_one("pkg-a os=redhat6 %gcc@10foo")
         assert "%gcc@10foo" in s
 
     def test_all_patches_applied(self):
@@ -1531,8 +1531,8 @@ def test_external_with_non_default_variant_as_dependency(self):
             ("mpileaks", "os=debian6"),
             # To trigger the bug in 22871 we need to have the same compiler
             # spec available on both operating systems
-            ("mpileaks%gcc@10.2.1 platform=test os=debian6", "os=debian6"),
-            ("mpileaks%gcc@10.2.1 platform=test os=redhat6", "os=redhat6"),
+            ("mpileaks platform=test os=debian6 %gcc@10.2.1", "os=debian6"),
+            ("mpileaks platform=test os=redhat6 %gcc@10.2.1", "os=redhat6"),
         ],
     )
     def test_os_selection_when_multiple_choices_are_possible(
@@ -3013,7 +3013,7 @@ def test_virtuals_provided_together_but_only_one_required_in_dag(self):
 
 
 def test_reusable_externals_match(mock_packages, tmpdir):
-    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0")
     spec.external_path = tmpdir.strpath
     spec.external_modules = ["mpich/4.1"]
     spec._mark_concrete()
@@ -3031,7 +3031,7 @@ def test_reusable_externals_match(mock_packages, tmpdir):
 
 
 def test_reusable_externals_match_virtual(mock_packages, tmpdir):
-    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0")
     spec.external_path = tmpdir.strpath
     spec.external_modules = ["mpich/4.1"]
     spec._mark_concrete()
@@ -3049,7 +3049,7 @@ def test_reusable_externals_match_virtual(mock_packages, tmpdir):
 
 
 def test_reusable_externals_different_prefix(mock_packages, tmpdir):
-    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0")
     spec.external_path = "/other/path"
     spec.external_modules = ["mpich/4.1"]
     spec._mark_concrete()
@@ -3068,7 +3068,7 @@ def test_reusable_externals_different_prefix(mock_packages, tmpdir):
 
 @pytest.mark.parametrize("modules", [None, ["mpich/4.1", "libfabric/1.19"]])
 def test_reusable_externals_different_modules(mock_packages, tmpdir, modules):
-    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0")
     spec.external_path = tmpdir.strpath
     spec.external_modules = modules
     spec._mark_concrete()
@@ -3086,7 +3086,7 @@ def test_reusable_externals_different_modules(mock_packages, tmpdir, modules):
 
 
 def test_reusable_externals_different_spec(mock_packages, tmpdir):
-    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec = Spec("mpich@4.1~debug build_system=generic arch=linux-ubuntu23.04-zen2 %gcc@13.1.0")
     spec.external_path = tmpdir.strpath
     spec._mark_concrete()
     assert not spack.solver.asp._is_reusable(

lib/spack/spack/test/concretization/errors.py

@@ -29,8 +29,7 @@
 ]
 
 variant_error_messages = [
-    "'fftw' required multiple values for single-valued variant 'mpi'",
-    "    Requested '~mpi' and '+mpi'",
+    "'fftw' requires conflicting variant values '~mpi' and '+mpi'",
     "        required because quantum-espresso depends on fftw+mpi when +invino",
     "          required because quantum-espresso+invino ^fftw~mpi requested explicitly",
     "        required because quantum-espresso+invino ^fftw~mpi requested explicitly",

lib/spack/spack/test/concretization/flag_mixing.py

@@ -94,7 +94,7 @@ def test_mix_spec_and_compiler_cfg(concretize_scope, test_repo):
     conf_str = _compiler_cfg_one_entry_with_cflags("-Wall")
     update_concretize_scope(conf_str, "compilers")
 
-    s1 = spack.concretize.concretize_one('y %gcc@12.100.100 cflags="-O2"')
+    s1 = spack.concretize.concretize_one('y cflags="-O2" %gcc@12.100.100')
     assert s1.satisfies('cflags="-Wall -O2"')
 
 
@@ -182,7 +182,7 @@ def test_propagate_and_compiler_cfg(concretize_scope, test_repo):
     conf_str = _compiler_cfg_one_entry_with_cflags("-f2")
     update_concretize_scope(conf_str, "compilers")
 
-    root_spec = spack.concretize.concretize_one("v %gcc@12.100.100 cflags=='-f1'")
+    root_spec = spack.concretize.concretize_one("v cflags=='-f1' %gcc@12.100.100")
     assert root_spec["y"].satisfies("cflags='-f1 -f2'")
 
 
@@ -229,7 +229,7 @@ def test_dev_mix_flags(tmp_path, concretize_scope, mutable_mock_env_path, test_r
     env_content = f"""\
 spack:
   specs:
-  - y %gcc@12.100.100 cflags=='-fsanitize=address'
+  - y cflags=='-fsanitize=address' %gcc@12.100.100
   develop:
     y:
       spec: y cflags=='-fsanitize=address'

lib/spack/spack/test/concretization/requirements.py

@@ -359,10 +359,10 @@ def test_one_package_multiple_oneof_groups(concretize_scope, test_repo):
     update_packages_config(conf_str)
 
     s1 = spack.concretize.concretize_one("y@2.5")
-    assert s1.satisfies("%clang~shared")
+    assert s1.satisfies("~shared%clang")
 
     s2 = spack.concretize.concretize_one("y@2.4")
-    assert s2.satisfies("%gcc+shared")
+    assert s2.satisfies("+shared%gcc")
 
 
 @pytest.mark.regression("34241")
@@ -377,11 +377,14 @@ def test_require_cflags(concretize_scope, mock_packages):
 """
     update_packages_config(conf_str)
 
-    spec_mpich2 = spack.concretize.concretize_one("mpich2")
-    assert spec_mpich2.satisfies("cflags=-g")
+    mpich2 = spack.concretize.concretize_one("mpich2")
+    assert mpich2.satisfies("cflags=-g")
 
-    spec_mpi = spack.concretize.concretize_one("mpi")
-    assert spec_mpi.satisfies("mpich cflags=-O1")
+    mpileaks = spack.concretize.concretize_one("mpileaks")
+    assert mpileaks["mpi"].satisfies("mpich cflags=-O1")
+
+    mpi = spack.concretize.concretize_one("mpi")
+    assert mpi.satisfies("mpich cflags=-O1")
 
 
 def test_requirements_for_package_that_is_not_needed(concretize_scope, test_repo):
@@ -499,7 +502,7 @@ def test_default_requirements_with_all(spec_str, requirement_str, concretize_sco
     "requirements,expectations",
     [
         (("%gcc", "%clang"), ("%gcc", "%clang")),
-        (("%gcc~shared", "@1.0"), ("%gcc~shared", "@1.0+shared")),
+        (("~shared%gcc", "@1.0"), ("~shared%gcc", "@1.0+shared")),
     ],
 )
 def test_default_and_package_specific_requirements(
@@ -754,7 +757,7 @@ def test_skip_requirement_when_default_requirement_condition_cannot_be_met(
     update_packages_config(packages_yaml)
     s = spack.concretize.concretize_one("mpileaks")
 
-    assert s.satisfies("%clang+shared")
+    assert s.satisfies("+shared %clang")
     # Sanity checks that 'callpath' doesn't have the shared variant, but that didn't
     # cause failures during concretization.
     assert "shared" not in s["callpath"].variants
@@ -982,6 +985,52 @@ def test_requiring_package_on_multiple_virtuals(concretize_scope, mock_packages)
             ["%clang"],
             ["%gcc"],
         ),
+        # Test using preferences on virtuals
+        (
+            """
+            packages:
+              all:
+                providers:
+                  mpi: [mpich]
+              mpi:
+                prefer:
+                - zmpi
+        """,
+            "mpileaks",
+            ["^[virtuals=mpi] zmpi"],
+            ["^[virtuals=mpi] mpich"],
+        ),
+        (
+            """
+            packages:
+              all:
+                providers:
+                  mpi: [mpich]
+              mpi:
+                prefer:
+                - zmpi
+        """,
+            "mpileaks ^[virtuals=mpi] mpich",
+            ["^[virtuals=mpi] mpich"],
+            ["^[virtuals=mpi] zmpi"],
+        ),
+        # Tests that strong preferences can be overridden by requirements
+        (
+            """
+                packages:
+                  all:
+                    providers:
+                      mpi: [zmpi]
+                  mpi:
+                    require:
+                    - mpich
+                    prefer:
+                    - zmpi
+            """,
+            "mpileaks",
+            ["^[virtuals=mpi] mpich"],
+            ["^[virtuals=mpi] zmpi"],
+        ),
     ],
 )
 def test_strong_preferences_packages_yaml(
@@ -1032,6 +1081,16 @@ def test_strong_preferences_packages_yaml(
         """,
             "multivalue-variant@=2.3 %clang",
         ),
+        # Test using conflict on virtual
+        (
+            """
+        packages:
+          mpi:
+            conflict:
+            - mpich
+    """,
+            "mpileaks ^[virtuals=mpi] mpich",
+        ),
     ],
 )
 def test_conflict_packages_yaml(packages_yaml, spec_str, concretize_scope, mock_packages):
@@ -1125,3 +1184,69 @@ def test_strong_preferences_higher_priority_than_reuse(concretize_scope, mock_pa
         )
         ascent = result.specs[0]
     assert ascent["adios2"].dag_hash() == reused_spec.dag_hash(), ascent
+
+
+@pytest.mark.parametrize(
+    "packages_yaml,err_match",
+    [
+        (
+            """
+packages:
+  mpi:
+    require:
+    - "+bzip2"
+""",
+            "expected a named spec",
+        ),
+        (
+            """
+packages:
+  mpi:
+    require:
+    - one_of: ["+bzip2", openmpi]
+""",
+            "expected a named spec",
+        ),
+        (
+            """
+packages:
+  mpi:
+    require:
+    - "^mpich"
+""",
+            "Did you mean",
+        ),
+    ],
+)
+def test_anonymous_spec_cannot_be_used_in_virtual_requirements(
+    packages_yaml, err_match, concretize_scope, mock_packages
+):
+    """Tests that using anonymous specs in requirements for virtual packages raises an
+    appropriate error message.
+    """
+    update_packages_config(packages_yaml)
+    with pytest.raises(spack.error.SpackError, match=err_match):
+        spack.concretize.concretize_one("mpileaks")
+
+
+def test_virtual_requirement_respects_any_of(concretize_scope, mock_packages):
+    """Tests that "any of" requirements can be used with virtuals"""
+    conf_str = """\
+        packages:
+          mpi:
+            require:
+            - any_of: ["mpich2", "mpich"]
+        """
+    update_packages_config(conf_str)
+
+    s = spack.concretize.concretize_one("mpileaks")
+    assert s.satisfies("^[virtuals=mpi] mpich2")
+
+    s = spack.concretize.concretize_one("mpileaks ^mpich2")
+    assert s.satisfies("^[virtuals=mpi] mpich2")
+
+    s = spack.concretize.concretize_one("mpileaks ^mpich")
+    assert s.satisfies("^[virtuals=mpi] mpich")
+
+    with pytest.raises(spack.error.SpackError):
+        spack.concretize.concretize_one("mpileaks ^[virtuals=mpi] zmpi")

lib/spack/spack/test/config.py

@@ -11,8 +11,7 @@
 
 import pytest
 
-import llnl.util.tty as tty
-from llnl.util.filesystem import join_path, touch, touchp
+from llnl.util.filesystem import join_path, touch
 
 import spack
 import spack.config
@@ -26,6 +25,7 @@
 import spack.schema.compilers
 import spack.schema.config
 import spack.schema.env
+import spack.schema.include
 import spack.schema.mirrors
 import spack.schema.repos
 import spack.spec
@@ -51,22 +51,9 @@
 
 config_override_list = {"config": {"build_stage:": ["pathd", "pathe"]}}
 
-config_merge_dict = {"config": {"info": {"a": 3, "b": 4}}}
+config_merge_dict = {"config": {"aliases": {"ls": "find", "dev": "develop"}}}
 
-config_override_dict = {"config": {"info:": {"a": 7, "c": 9}}}
-
-
-@pytest.fixture()
-def write_config_file(tmpdir):
-    """Returns a function that writes a config file."""
-
-    def _write(config, data, scope):
-        config_yaml = tmpdir.join(scope, config + ".yaml")
-        config_yaml.ensure()
-        with config_yaml.open("w") as f:
-            syaml.dump_config(data, f)
-
-    return _write
+config_override_dict = {"config": {"aliases:": {"be": "build-env", "deps": "dependencies"}}}
 
 
 @pytest.fixture()
@@ -1037,6 +1024,16 @@ def test_bad_config_yaml(tmpdir):
         )
 
 
+def test_bad_include_yaml(tmpdir):
+    with pytest.raises(spack.config.ConfigFormatError, match="is not of type"):
+        check_schema(
+            spack.schema.include.schema,
+            """\
+include: $HOME/include.yaml
+""",
+        )
+
+
 def test_bad_mirrors_yaml(tmpdir):
     with pytest.raises(spack.config.ConfigFormatError):
         check_schema(
@@ -1101,9 +1098,9 @@ def test_internal_config_section_override(mock_low_high_config, write_config_fil
 
 def test_internal_config_dict_override(mock_low_high_config, write_config_file):
     write_config_file("config", config_merge_dict, "low")
-    wanted_dict = config_override_dict["config"]["info:"]
+    wanted_dict = config_override_dict["config"]["aliases:"]
     mock_low_high_config.push_scope(spack.config.InternalConfigScope("high", config_override_dict))
-    assert mock_low_high_config.get("config:info") == wanted_dict
+    assert mock_low_high_config.get("config:aliases") == wanted_dict
 
 
 def test_internal_config_list_override(mock_low_high_config, write_config_file):
@@ -1135,10 +1132,10 @@ def test_set_list_override(mock_low_high_config, write_config_file):
 
 def test_set_dict_override(mock_low_high_config, write_config_file):
     write_config_file("config", config_merge_dict, "low")
-    wanted_dict = config_override_dict["config"]["info:"]
-    with spack.config.override("config:info:", wanted_dict):
-        assert wanted_dict == mock_low_high_config.get("config:info")
-    assert config_merge_dict["config"]["info"] == mock_low_high_config.get("config:info")
+    wanted_dict = config_override_dict["config"]["aliases:"]
+    with spack.config.override("config:aliases:", wanted_dict):
+        assert wanted_dict == mock_low_high_config.get("config:aliases")
+    assert config_merge_dict["config"]["aliases"] == mock_low_high_config.get("config:aliases")
 
 
 def test_set_bad_path(config):
@@ -1224,7 +1221,7 @@ def test_user_config_path_is_default_when_env_var_is_empty(working_env):
 
 
 def test_default_install_tree(monkeypatch, default_config):
-    s = spack.spec.Spec("nonexistent@x.y.z %none@a.b.c arch=foo-bar-baz")
+    s = spack.spec.Spec("nonexistent@x.y.z arch=foo-bar-baz %none@a.b.c")
     monkeypatch.setattr(s, "dag_hash", lambda length: "abc123")
     _, _, projections = spack.store.parse_install_tree(spack.config.get("config"))
     assert s.format(projections["all"]) == "foo-bar-baz/none-a.b.c/nonexistent-x.y.z-abc123"
@@ -1264,134 +1261,6 @@ def test_user_cache_path_is_default_when_env_var_is_empty(working_env):
     assert os.path.expanduser("~%s.spack" % os.sep) == spack.paths._get_user_cache_path()
 
 
-github_url = "https://github.com/fake/fake/{0}/develop"
-gitlab_url = "https://gitlab.fake.io/user/repo/-/blob/config/defaults"
-
-
-@pytest.mark.parametrize(
-    "url,isfile",
-    [
-        (github_url.format("tree"), False),
-        ("{0}/README.md".format(github_url.format("blob")), True),
-        ("{0}/etc/fake/defaults/packages.yaml".format(github_url.format("blob")), True),
-        (gitlab_url, False),
-        (None, False),
-    ],
-)
-def test_config_collect_urls(mutable_empty_config, mock_spider_configs, url, isfile):
-    with spack.config.override("config:url_fetch_method", "curl"):
-        urls = spack.config.collect_urls(url)
-        if url:
-            if isfile:
-                expected = 1 if url.endswith(".yaml") else 0
-                assert len(urls) == expected
-            else:
-                # Expect multiple configuration files for a "directory"
-                assert len(urls) > 1
-        else:
-            assert not urls
-
-
-@pytest.mark.parametrize(
-    "url,isfile,fail",
-    [
-        (github_url.format("tree"), False, False),
-        (gitlab_url, False, False),
-        ("{0}/README.md".format(github_url.format("blob")), True, True),
-        ("{0}/compilers.yaml".format(gitlab_url), True, False),
-        (None, False, True),
-    ],
-)
-def test_config_fetch_remote_configs(
-    tmpdir, mutable_empty_config, mock_collect_urls, mock_curl_configs, url, isfile, fail
-):
-    def _has_content(filename):
-        # The first element of all configuration files for this test happen to
-        # be the basename of the file so this check leverages that feature. If
-        # that changes, then this check will need to change accordingly.
-        element = "{0}:".format(os.path.splitext(os.path.basename(filename))[0])
-        with open(filename, "r", encoding="utf-8") as fd:
-            for line in fd:
-                if element in line:
-                    return True
-        tty.debug("Expected {0} in '{1}'".format(element, filename))
-        return False
-
-    dest_dir = join_path(tmpdir.strpath, "defaults")
-    if fail:
-        msg = "Cannot retrieve configuration"
-        with spack.config.override("config:url_fetch_method", "curl"):
-            with pytest.raises(spack.config.ConfigFileError, match=msg):
-                spack.config.fetch_remote_configs(url, dest_dir)
-    else:
-        with spack.config.override("config:url_fetch_method", "curl"):
-            path = spack.config.fetch_remote_configs(url, dest_dir)
-            assert os.path.exists(path)
-            if isfile:
-                # Ensure correct file is "fetched"
-                assert os.path.basename(path) == os.path.basename(url)
-                # Ensure contents of the file has expected config element
-                assert _has_content(path)
-            else:
-                for filename in os.listdir(path):
-                    assert _has_content(join_path(path, filename))
-
-
-@pytest.fixture(scope="function")
-def mock_collect_urls(mock_config_data, monkeypatch):
-    """Mock the collection of URLs to avoid mocking spider."""
-
-    _, config_files = mock_config_data
-
-    def _collect(base_url):
-        if not base_url:
-            return []
-
-        ext = os.path.splitext(base_url)[1]
-        if ext:
-            return [base_url] if ext == ".yaml" else []
-
-        return [join_path(base_url, f) for f in config_files]
-
-    monkeypatch.setattr(spack.config, "collect_urls", _collect)
-
-    yield
-
-
-@pytest.mark.parametrize(
-    "url,skip",
-    [(github_url.format("tree"), True), ("{0}/compilers.yaml".format(gitlab_url), True)],
-)
-def test_config_fetch_remote_configs_skip(
-    tmpdir, mutable_empty_config, mock_collect_urls, mock_curl_configs, url, skip
-):
-    """Ensure skip fetching remote config file if it already exists when
-    required and not skipping if replacing it."""
-
-    def check_contents(filename, expected):
-        with open(filename, "r", encoding="utf-8") as fd:
-            lines = fd.readlines()
-            if expected:
-                assert lines[0] == "compilers:"
-            else:
-                assert not lines
-
-    dest_dir = join_path(tmpdir.strpath, "defaults")
-    filename = "compilers.yaml"
-
-    # Create a stage directory with an empty configuration file
-    path = join_path(dest_dir, filename)
-    touchp(path)
-
-    # Do NOT replace the existing cached configuration file if skipping
-    expected = None if skip else "compilers:"
-
-    with spack.config.override("config:url_fetch_method", "curl"):
-        path = spack.config.fetch_remote_configs(url, dest_dir, skip)
-        result_filename = path if path.endswith(".yaml") else join_path(path, filename)
-        check_contents(result_filename, expected)
-
-
 def test_config_file_dir_failure(tmpdir, mutable_empty_config):
     with pytest.raises(spack.config.ConfigFileError, match="not a file"):
         spack.config.read_config_file(tmpdir.strpath)

lib/spack/spack/test/conftest.py

@@ -30,7 +30,15 @@
 import llnl.util.lang
 import llnl.util.lock
 import llnl.util.tty as tty
-from llnl.util.filesystem import copy_tree, mkdirp, remove_linked_tree, touchp, working_dir
+from llnl.util.filesystem import (
+    copy,
+    copy_tree,
+    join_path,
+    mkdirp,
+    remove_linked_tree,
+    touchp,
+    working_dir,
+)
 
 import spack.binary_distribution
 import spack.bootstrap.core
@@ -65,6 +73,7 @@
 from spack.installer import PackageInstaller
 from spack.main import SpackCommand
 from spack.util.pattern import Bunch
+from spack.util.remote_file_cache import raw_github_gitlab_url
 
 from ..enums import ConfigScopePriority
 
@@ -152,7 +161,7 @@ def mock_git_version_info(git, tmpdir, override_git_repos_cache_path):
     version tags on multiple branches, and version order is not equal to time
     order or topological order.
     """
-    repo_path = str(tmpdir.mkdir("git_repo"))
+    repo_path = str(tmpdir.mkdir("git_version_info_repo"))
     filename = "file.txt"
 
     def commit(message):
@@ -233,6 +242,84 @@ def latest_commit():
     yield repo_path, filename, commits
 
 
+@pytest.fixture
+def mock_git_package_changes(git, tmpdir, override_git_repos_cache_path):
+    """Create a mock git repo with known structure of package edits
+
+    The structure of commits in this repo is as follows::
+
+       o diff-test: modification to make manual download package
+       |
+       o diff-test: add v1.2 (from a git ref)
+       |
+       o diff-test: add v1.1 (from source tarball)
+       |
+       o diff-test: new package (testing multiple added versions)
+
+    The repo consists of a single package.py file for DiffTest.
+
+    Important attributes of the repo for test coverage are: multiple package
+    versions are added with some coming from a tarball and some from git refs.
+    """
+    repo_path = str(tmpdir.mkdir("git_package_changes_repo"))
+    filename = "var/spack/repos/builtin/packages/diff-test/package.py"
+
+    def commit(message):
+        global commit_counter
+        git(
+            "commit",
+            "--no-gpg-sign",
+            "--date",
+            "2020-01-%02d 12:0:00 +0300" % commit_counter,
+            "-am",
+            message,
+        )
+        commit_counter += 1
+
+    with working_dir(repo_path):
+        git("init")
+
+        git("config", "user.name", "Spack")
+        git("config", "user.email", "spack@spack.io")
+
+        commits = []
+
+        def latest_commit():
+            return git("rev-list", "-n1", "HEAD", output=str, error=str).strip()
+
+        os.makedirs(os.path.dirname(filename))
+
+        # add pkg-a as a new package to the repository
+        shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-0.txt", filename)
+        git("add", filename)
+        commit("diff-test: new package")
+        commits.append(latest_commit())
+
+        # add v2.1.5 to pkg-a
+        shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-1.txt", filename)
+        git("add", filename)
+        commit("diff-test: add v2.1.5")
+        commits.append(latest_commit())
+
+        # add v2.1.6 to pkg-a
+        shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-2.txt", filename)
+        git("add", filename)
+        commit("diff-test: add v2.1.6")
+        commits.append(latest_commit())
+
+        # convert pkg-a to a manual download package
+        shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-3.txt", filename)
+        git("add", filename)
+        commit("diff-test: modification to make manual download package")
+        commits.append(latest_commit())
+
+        # The commits are ordered with the last commit first in the list
+        commits = list(reversed(commits))
+
+    # Return the git directory to install, the filename used, and the commits
+    yield repo_path, filename, commits
+
+
 @pytest.fixture(autouse=True)
 def clear_recorded_monkeypatches():
     yield
@@ -1691,7 +1778,7 @@ def installation_dir_with_headers(tmpdir_factory):
 ##########
 
 
-@pytest.fixture(params=["conflict%clang+foo", "conflict-parent@0.9^conflict~foo"])
+@pytest.fixture(params=["conflict+foo%clang", "conflict-parent@0.9^conflict~foo"])
 def conflict_spec(request):
     """Specs which violate constraints specified with the "conflicts"
     directive in the "conflict" package.
@@ -1906,35 +1993,21 @@ def __call__(self, *args, **kwargs):
 
 
 @pytest.fixture(scope="function")
-def mock_spider_configs(mock_config_data, monkeypatch):
-    """
-    Mock retrieval of configuration file URLs from the web by grabbing
-    them from the test data configuration directory.
-    """
-    config_data_dir, config_files = mock_config_data
+def mock_fetch_url_text(tmpdir, mock_config_data, monkeypatch):
+    """Mock spack.util.web.fetch_url_text."""
 
-    def _spider(*args, **kwargs):
-        root_urls = args[0]
-        if not root_urls:
-            return [], set()
+    stage_dir, config_files = mock_config_data
 
-        root_urls = [root_urls] if isinstance(root_urls, str) else root_urls
+    def _fetch_text_file(url, dest_dir):
+        raw_url = raw_github_gitlab_url(url)
+        mkdirp(dest_dir)
+        basename = os.path.basename(raw_url)
+        src = join_path(stage_dir, basename)
+        dest = join_path(dest_dir, basename)
+        copy(src, dest)
+        return dest
 
-        # Any URL with an extension will be treated like a file; otherwise,
-        # it is considered a directory/folder and we'll grab all available
-        # files.
-        urls = []
-        for url in root_urls:
-            if os.path.splitext(url)[1]:
-                urls.append(url)
-            else:
-                urls.extend([os.path.join(url, f) for f in config_files])
-
-        return [], set(urls)
-
-    monkeypatch.setattr(spack.util.web, "spider", _spider)
-
-    yield
+    monkeypatch.setattr(spack.util.web, "fetch_url_text", _fetch_text_file)
 
 
 @pytest.fixture(scope="function")
@@ -2197,3 +2270,27 @@ def info(self):
 @pytest.fixture()
 def mock_runtimes(config, mock_packages):
     return mock_packages.packages_with_tags("runtime")
+
+
+@pytest.fixture()
+def write_config_file(tmpdir):
+    """Returns a function that writes a config file."""
+
+    def _write(config, data, scope):
+        config_yaml = tmpdir.join(scope, config + ".yaml")
+        config_yaml.ensure()
+        with config_yaml.open("w") as f:
+            syaml.dump_config(data, f)
+        return config_yaml
+
+    return _write
+
+
+def _include_cache_root():
+    return join_path(str(tempfile.mkdtemp()), "user_cache", "includes")
+
+
+@pytest.fixture()
+def mock_include_cache(monkeypatch):
+    """Override the include cache directory so tests don't pollute user cache."""
+    monkeypatch.setattr(spack.config, "_include_cache_location", _include_cache_root)

lib/spack/spack/test/data/conftest/diff-test/package-0.txt

@@ -0,0 +1,19 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class DiffTest(AutotoolsPackage):
+    """zlib replacement with optimizations for next generation systems."""
+
+    homepage = "https://github.com/zlib-ng/zlib-ng"
+    url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz"
+    git = "https://github.com/zlib-ng/zlib-ng.git"
+
+    license("Zlib")
+
+    version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a")
+    version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8")
+    version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200")

lib/spack/spack/test/data/conftest/diff-test/package-1.txt

@@ -0,0 +1,20 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class DiffTest(AutotoolsPackage):
+    """zlib replacement with optimizations for next generation systems."""
+
+    homepage = "https://github.com/zlib-ng/zlib-ng"
+    url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz"
+    git = "https://github.com/zlib-ng/zlib-ng.git"
+
+    license("Zlib")
+
+    version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04")
+    version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a")
+    version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200")
+    version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8")

lib/spack/spack/test/data/conftest/diff-test/package-2.txt

@@ -0,0 +1,21 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class DiffTest(AutotoolsPackage):
+    """zlib replacement with optimizations for next generation systems."""
+
+    homepage = "https://github.com/zlib-ng/zlib-ng"
+    url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz"
+    git = "https://github.com/zlib-ng/zlib-ng.git"
+
+    license("Zlib")
+
+    version("2.1.6", tag="2.1.6", commit="74253725f884e2424a0dd8ae3f69896d5377f325")
+    version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04")
+    version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a")
+    version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200")
+    version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8")

lib/spack/spack/test/data/conftest/diff-test/package-3.txt

@@ -0,0 +1,23 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class DiffTest(AutotoolsPackage):
+    """zlib replacement with optimizations for next generation systems."""
+
+    homepage = "https://github.com/zlib-ng/zlib-ng"
+    url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz"
+    git = "https://github.com/zlib-ng/zlib-ng.git"
+
+    license("Zlib")
+
+    manual_download = True
+
+    version("2.1.6", tag="2.1.6", commit="74253725f884e2424a0dd8ae3f69896d5377f325")
+    version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04")
+    version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a")
+    version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200")
+    version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8")

lib/spack/spack/test/directives.py

@@ -67,6 +67,20 @@ def test_extends_spec(config, mock_packages):
     assert extender.package.extends(extendee)
 
 
+@pytest.mark.regression("48024")
+def test_conditionally_extends_transitive_dep(config, mock_packages):
+    spec = spack.spec.Spec("conditionally-extends-transitive-dep").concretized()
+
+    assert not spec.package.extendee_spec
+
+
+@pytest.mark.regression("48025")
+def test_conditionally_extends_direct_dep(config, mock_packages):
+    spec = spack.spec.Spec("conditionally-extends-direct-dep").concretized()
+
+    assert not spec.package.extendee_spec
+
+
 @pytest.mark.regression("34368")
 def test_error_on_anonymous_dependency(config, mock_packages):
     pkg = spack.repo.PATH.get_pkg_class("pkg-a")

lib/spack/spack/test/env.py

@@ -12,6 +12,7 @@
 
 import spack.config
 import spack.environment as ev
+import spack.platforms
 import spack.solver.asp
 import spack.spec
 from spack.environment.environment import (
@@ -921,3 +922,50 @@ def test_environment_from_name_or_dir(mock_packages, mutable_mock_env_path, tmp_
 
     with pytest.raises(ev.SpackEnvironmentError, match="no such environment"):
         _ = ev.environment_from_name_or_dir("fake-env")
+
+
+def test_env_include_configs(mutable_mock_env_path, mock_packages):
+    """check config and package values using new include schema"""
+    env_path = mutable_mock_env_path
+    env_path.mkdir()
+
+    this_os = spack.platforms.host().default_os
+    config_root = env_path / this_os
+    config_root.mkdir()
+    config_path = str(config_root / "config.yaml")
+    with open(config_path, "w", encoding="utf-8") as f:
+        f.write(
+            """\
+config:
+  verify_ssl: False
+"""
+        )
+
+    packages_path = str(env_path / "packages.yaml")
+    with open(packages_path, "w", encoding="utf-8") as f:
+        f.write(
+            """\
+packages:
+  python:
+    require:
+    - spec: "@3.11:"
+"""
+        )
+
+    spack_yaml = env_path / ev.manifest_name
+    spack_yaml.write_text(
+        f"""\
+spack:
+  include:
+  - path: {config_path}
+    optional: true
+  - path: {packages_path}
+"""
+    )
+
+    e = ev.Environment(env_path)
+    with e.manifest.use_config():
+        assert not spack.config.get("config:verify_ssl")
+        python_reqs = spack.config.get("packages")["python"]["require"]
+        req_specs = set(x["spec"] for x in python_reqs)
+        assert req_specs == set(["@3.11:"])

lib/spack/spack/test/installer.py

@@ -680,13 +680,19 @@ def test_install_spliced_build_spec_installed(install_mockery, capfd, mock_fetch
         assert node.build_spec.installed
 
 
+# Unit tests should not be affected by the user's managed environments
 @pytest.mark.not_on_windows("lacking windows support for binary installs")
 @pytest.mark.parametrize("transitive", [True, False])
 @pytest.mark.parametrize(
     "root_str", ["splice-t^splice-h~foo", "splice-h~foo", "splice-vt^splice-a"]
 )
 def test_install_splice_root_from_binary(
-    install_mockery, mock_fetch, mutable_temporary_mirror, transitive, root_str
+    mutable_mock_env_path,
+    install_mockery,
+    mock_fetch,
+    mutable_temporary_mirror,
+    transitive,
+    root_str,
 ):
     """Test installing a spliced spec with the root available in binary cache"""
     # Test splicing and rewiring a spec with the same name, different hash.

lib/spack/spack/test/main.py

@@ -3,6 +3,9 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 
+import os
+import os.path
+
 import pytest
 
 import llnl.util.filesystem as fs
@@ -13,8 +16,10 @@
 import spack.error
 import spack.main
 import spack.paths
+import spack.platforms
 import spack.util.executable as exe
 import spack.util.git
+import spack.util.spack_yaml as syaml
 
 pytestmark = pytest.mark.not_on_windows(
     "Test functionality supported but tests are failing on Win"
@@ -167,3 +172,163 @@ def test_add_command_line_scope_env(tmp_path, mutable_mock_env_path):
     assert config.get("config:install_tree:root") == "/tmp/first"
 
     assert ev.active_environment() is None  # shouldn't cause an environment to be activated
+
+
+def test_include_cfg(mock_low_high_config, write_config_file, tmpdir):
+    cfg1_path = str(tmpdir.join("include1.yaml"))
+    with open(cfg1_path, "w", encoding="utf-8") as f:
+        f.write(
+            """\
+config:
+  verify_ssl: False
+  dirty: True
+packages:
+  python:
+    require:
+    - spec: "@3.11:"
+"""
+        )
+
+    def python_cfg(_spec):
+        return f"""\
+packages:
+  python:
+    require:
+    - spec: {_spec}
+"""
+
+    def write_python_cfg(_spec, _cfg_name):
+        cfg_path = str(tmpdir.join(_cfg_name))
+        with open(cfg_path, "w", encoding="utf-8") as f:
+            f.write(python_cfg(_spec))
+        return cfg_path
+
+    # This config will not be included
+    cfg2_path = write_python_cfg("+shared", "include2.yaml")
+
+    # The config will point to this using substitutable variables,
+    # namely $os; we expect that Spack resolves these variables
+    # into the actual path of the config
+    this_os = spack.platforms.host().default_os
+    cfg3_expanded_path = os.path.join(str(tmpdir), f"{this_os}", "include3.yaml")
+    fs.mkdirp(os.path.dirname(cfg3_expanded_path))
+    with open(cfg3_expanded_path, "w", encoding="utf-8") as f:
+        f.write(python_cfg("+ssl"))
+    cfg3_abstract_path = os.path.join(str(tmpdir), "$os", "include3.yaml")
+
+    # This will be included unconditionally
+    cfg4_path = write_python_cfg("+tk", "include4.yaml")
+
+    # This config will not exist, and the config will explicitly
+    # allow this
+    cfg5_path = os.path.join(str(tmpdir), "non-existent.yaml")
+
+    include_entries = [
+        {"path": f"{cfg1_path}", "when": f'os == "{this_os}"'},
+        {"path": f"{cfg2_path}", "when": "False"},
+        {"path": cfg3_abstract_path},
+        cfg4_path,
+        {"path": cfg5_path, "optional": True},
+    ]
+    include_cfg = {"include": include_entries}
+    filename = write_config_file("include", include_cfg, "low")
+
+    assert not spack.config.get("config:dirty")
+
+    spack.main.add_command_line_scopes(mock_low_high_config, [os.path.dirname(filename)])
+
+    assert spack.config.get("config:dirty")
+    python_reqs = spack.config.get("packages")["python"]["require"]
+    req_specs = set(x["spec"] for x in python_reqs)
+    assert req_specs == set(["@3.11:", "+ssl", "+tk"])
+
+
+def test_include_duplicate_source(tmpdir, mutable_config):
+    """Check precedence when include.yaml files have the same path."""
+    include_yaml = "debug.yaml"
+    include_list = {"include": [f"./{include_yaml}"]}
+
+    system_filename = mutable_config.get_config_filename("system", "include")
+    site_filename = mutable_config.get_config_filename("site", "include")
+
+    def write_configs(include_path, debug_data):
+        fs.mkdirp(os.path.dirname(include_path))
+        with open(include_path, "w", encoding="utf-8") as f:
+            syaml.dump_config(include_list, f)
+
+        debug_path = fs.join_path(os.path.dirname(include_path), include_yaml)
+        with open(debug_path, "w", encoding="utf-8") as f:
+            syaml.dump_config(debug_data, f)
+
+    system_config = {"config": {"debug": False}}
+    write_configs(system_filename, system_config)
+    spack.main.add_command_line_scopes(mutable_config, [os.path.dirname(system_filename)])
+
+    site_config = {"config": {"debug": True}}
+    write_configs(site_filename, site_config)
+    spack.main.add_command_line_scopes(mutable_config, [os.path.dirname(site_filename)])
+
+    # Ensure takes the last value of the option pushed onto the stack
+    assert mutable_config.get("config:debug") == site_config["config"]["debug"]
+
+
+def test_include_recurse_limit(tmpdir, mutable_config):
+    """Ensure hit the recursion limit."""
+    include_yaml = "include.yaml"
+    include_list = {"include": [f"./{include_yaml}"]}
+
+    include_path = str(tmpdir.join(include_yaml))
+    with open(include_path, "w", encoding="utf-8") as f:
+        syaml.dump_config(include_list, f)
+
+    with pytest.raises(spack.config.RecursiveIncludeError, match="recursion exceeded"):
+        spack.main.add_command_line_scopes(mutable_config, [os.path.dirname(include_path)])
+
+
+# TODO: Fix this once recursive includes are processed in the expected order.
+@pytest.mark.parametrize("child,expected", [("b", True), ("c", False)])
+def test_include_recurse_diamond(tmpdir, mutable_config, child, expected):
+    """Demonstrate include parent's value overrides that of child in diamond include.
+
+    Check that the value set by b or c overrides that set by d.
+    """
+    configs_root = tmpdir.join("configs")
+    configs_root.mkdir()
+
+    def write(path, contents):
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(contents)
+
+    def debug_contents(value):
+        return f"config:\n  debug: {value}\n"
+
+    def include_contents(paths):
+        indent = "\n  - "
+        values = indent.join([str(p) for p in paths])
+        return f"include:{indent}{values}"
+
+    a_yaml = tmpdir.join("a.yaml")
+    b_yaml = configs_root.join("b.yaml")
+    c_yaml = configs_root.join("c.yaml")
+    d_yaml = configs_root.join("d.yaml")
+    debug_yaml = configs_root.join("enable_debug.yaml")
+
+    write(debug_yaml, debug_contents("true"))
+
+    a_contents = f"""\
+include:
+- {b_yaml}
+- {c_yaml}
+"""
+    write(a_yaml, a_contents)
+    write(d_yaml, debug_contents("false"))
+
+    write(b_yaml, include_contents([debug_yaml, d_yaml] if child == "b" else [d_yaml]))
+    write(c_yaml, include_contents([debug_yaml, d_yaml] if child == "c" else [d_yaml]))
+
+    spack.main.add_command_line_scopes(mutable_config, [str(tmpdir)])
+
+    try:
+        assert mutable_config.get("config:debug") is expected
+    except AssertionError:
+        pytest.xfail("recursive includes are not processed in the expected order")

lib/spack/spack/test/multimethod.py

@@ -53,7 +53,7 @@ def test_no_version_match(pkg_name):
         # Constraints on compilers with a default
         ("%gcc", "has_a_default", "gcc"),
         ("%clang", "has_a_default", "clang"),
-        ("%apple-clang os=elcapitan", "has_a_default", "default"),
+        ("os=elcapitan %apple-clang", "has_a_default", "default"),
         # Constraints on dependencies
         ("^zmpi", "different_by_dep", "zmpi"),
         ("^mpich", "different_by_dep", "mpich"),
@@ -74,7 +74,7 @@ def test_multimethod_calls(
     with spack.config.override(
         "compilers", [compiler_factory(spec="apple-clang@9.1.0", operating_system="elcapitan")]
     ):
-        s = spack.concretize.concretize_one(pkg_name + constraint_str)
+        s = spack.concretize.concretize_one(f"{pkg_name} {constraint_str}")
     msg = f"Method {method_name} from {s} is giving a wrong result"
     assert getattr(s.package, method_name)() == expected_result, msg
 

lib/spack/spack/test/optional_deps.py

@@ -38,9 +38,9 @@
             {"optional-dep-test@1.1%intel": {"pkg-b": None, "pkg-c": None}},
         ),
         (
-            "optional-dep-test@1.1%intel@64.1.2+a",
+            "optional-dep-test@1.1+a%intel@64.1.2",
             {
-                "optional-dep-test@1.1%intel@64.1.2+a": {
+                "optional-dep-test@1.1+a%intel@64.1.2": {
                     "pkg-a": None,
                     "pkg-b": None,
                     "pkg-c": None,
@@ -49,8 +49,8 @@
             },
         ),
         (
-            "optional-dep-test@1.1%clang@36.5+a",
-            {"optional-dep-test@1.1%clang@36.5+a": {"pkg-b": None, "pkg-a": None, "pkg-e": None}},
+            "optional-dep-test@1.1+a%clang@36.5",
+            {"optional-dep-test@1.1+a%clang@36.5": {"pkg-b": None, "pkg-a": None, "pkg-e": None}},
         ),
         # Chained MPI
         (

lib/spack/spack/test/schema.py

@@ -84,6 +84,7 @@ def test_module_suffixes(module_suffixes_schema):
         "compilers",
         "config",
         "definitions",
+        "include",
         "env",
         "merged",
         "mirrors",

lib/spack/spack/test/spec_semantics.py

@@ -1751,7 +1751,6 @@ def test_package_hash_affects_dunder_and_dag_hash(mock_packages, default_mock_co
 
     assert hash(a1) == hash(a2)
     assert a1.dag_hash() == a2.dag_hash()
-    assert a1.process_hash() == a2.process_hash()
 
     a1.clear_caches()
     a2.clear_caches()
@@ -1764,7 +1763,6 @@ def test_package_hash_affects_dunder_and_dag_hash(mock_packages, default_mock_co
 
     assert hash(a1) != hash(a2)
     assert a1.dag_hash() != a2.dag_hash()
-    assert a1.process_hash() != a2.process_hash()
 
 
 def test_intersects_and_satisfies_on_concretized_spec(default_mock_concretization):

lib/spack/spack/test/spec_syntax.py

@@ -21,6 +21,7 @@
     SpecParsingError,
     SpecTokenizationError,
     SpecTokens,
+    parse_one_or_raise,
 )
 from spack.tokenize import Token
 
@@ -159,13 +160,13 @@ def _specfile_for(spec_str, filename):
         ),
         # Version after compiler
         (
-            "foo %bar@1.0 @2.0",
+            "foo @2.0 %bar@1.0",
             [
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="foo"),
-                Token(SpecTokens.COMPILER_AND_VERSION, value="%bar@1.0"),
                 Token(SpecTokens.VERSION, value="@2.0"),
+                Token(SpecTokens.COMPILER_AND_VERSION, value="%bar@1.0"),
             ],
-            "foo@2.0%bar@1.0",
+            "foo@2.0 %bar@1.0",
         ),
         # Single dependency with version
         dependency_with_version("openmpi ^hwloc@1.2e6"),
@@ -174,54 +175,54 @@ def _specfile_for(spec_str, filename):
         dependency_with_version("openmpi ^hwloc@1.2e6:1.4b7-rc3"),
         # Complex specs with multiple constraints
         (
-            "mvapich_foo ^_openmpi@1.2:1.4,1.6%intel@12.1+debug~qt_4 ^stackwalker@8.1_1e",
+            "mvapich_foo ^_openmpi@1.2:1.4,1.6+debug~qt_4 %intel@12.1 ^stackwalker@8.1_1e",
             [
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich_foo"),
                 Token(SpecTokens.DEPENDENCY, value="^"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"),
                 Token(SpecTokens.VERSION, value="@1.2:1.4,1.6"),
-                Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"),
                 Token(SpecTokens.BOOL_VARIANT, value="+debug"),
                 Token(SpecTokens.BOOL_VARIANT, value="~qt_4"),
+                Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"),
                 Token(SpecTokens.DEPENDENCY, value="^"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"),
                 Token(SpecTokens.VERSION, value="@8.1_1e"),
             ],
-            "mvapich_foo ^_openmpi@1.2:1.4,1.6%intel@12.1+debug~qt_4 ^stackwalker@8.1_1e",
+            "mvapich_foo ^_openmpi@1.2:1.4,1.6+debug~qt_4 %intel@12.1 ^stackwalker@8.1_1e",
         ),
         (
-            "mvapich_foo ^_openmpi@1.2:1.4,1.6%intel@12.1~qt_4 debug=2 ^stackwalker@8.1_1e",
+            "mvapich_foo ^_openmpi@1.2:1.4,1.6~qt_4 debug=2 %intel@12.1 ^stackwalker@8.1_1e",
             [
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich_foo"),
                 Token(SpecTokens.DEPENDENCY, value="^"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"),
                 Token(SpecTokens.VERSION, value="@1.2:1.4,1.6"),
-                Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"),
                 Token(SpecTokens.BOOL_VARIANT, value="~qt_4"),
                 Token(SpecTokens.KEY_VALUE_PAIR, value="debug=2"),
+                Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"),
                 Token(SpecTokens.DEPENDENCY, value="^"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"),
                 Token(SpecTokens.VERSION, value="@8.1_1e"),
             ],
-            "mvapich_foo ^_openmpi@1.2:1.4,1.6%intel@12.1~qt_4 debug=2 ^stackwalker@8.1_1e",
+            "mvapich_foo ^_openmpi@1.2:1.4,1.6~qt_4 debug=2 %intel@12.1 ^stackwalker@8.1_1e",
         ),
         (
-            "mvapich_foo ^_openmpi@1.2:1.4,1.6%intel@12.1 cppflags=-O3 +debug~qt_4 "
+            "mvapich_foo ^_openmpi@1.2:1.4,1.6 cppflags=-O3 +debug~qt_4 %intel@12.1 "
             "^stackwalker@8.1_1e",
             [
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich_foo"),
                 Token(SpecTokens.DEPENDENCY, value="^"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="_openmpi"),
                 Token(SpecTokens.VERSION, value="@1.2:1.4,1.6"),
-                Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"),
                 Token(SpecTokens.KEY_VALUE_PAIR, value="cppflags=-O3"),
                 Token(SpecTokens.BOOL_VARIANT, value="+debug"),
                 Token(SpecTokens.BOOL_VARIANT, value="~qt_4"),
+                Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"),
                 Token(SpecTokens.DEPENDENCY, value="^"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"),
                 Token(SpecTokens.VERSION, value="@8.1_1e"),
             ],
-            "mvapich_foo ^_openmpi@1.2:1.4,1.6%intel@12.1 cppflags=-O3 +debug~qt_4 "
+            "mvapich_foo ^_openmpi@1.2:1.4,1.6 cppflags=-O3 +debug~qt_4 %intel@12.1 "
             "^stackwalker@8.1_1e",
         ),
         # Specs containing YAML or JSON in the package name
@@ -235,7 +236,7 @@ def _specfile_for(spec_str, filename):
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="boost"),
                 Token(SpecTokens.VERSION, value="@3.1.4"),
             ],
-            "yaml-cpp@0.1.8%intel@12.1 ^boost@3.1.4",
+            "yaml-cpp@0.1.8 %intel@12.1 ^boost@3.1.4",
         ),
         (
             r"builtin.yaml-cpp%gcc",
@@ -243,7 +244,7 @@ def _specfile_for(spec_str, filename):
                 Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value="builtin.yaml-cpp"),
                 Token(SpecTokens.COMPILER, value="%gcc"),
             ],
-            "yaml-cpp%gcc",
+            "yaml-cpp %gcc",
         ),
         (
             r"testrepo.yaml-cpp%gcc",
@@ -251,7 +252,7 @@ def _specfile_for(spec_str, filename):
                 Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value="testrepo.yaml-cpp"),
                 Token(SpecTokens.COMPILER, value="%gcc"),
             ],
-            "yaml-cpp%gcc",
+            "yaml-cpp %gcc",
         ),
         (
             r"builtin.yaml-cpp@0.1.8%gcc@7.2.0 ^boost@3.1.4",
@@ -263,7 +264,7 @@ def _specfile_for(spec_str, filename):
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="boost"),
                 Token(SpecTokens.VERSION, value="@3.1.4"),
             ],
-            "yaml-cpp@0.1.8%gcc@7.2.0 ^boost@3.1.4",
+            "yaml-cpp@0.1.8 %gcc@7.2.0 ^boost@3.1.4",
         ),
         (
             r"builtin.yaml-cpp ^testrepo.boost ^zlib",
@@ -485,12 +486,12 @@ def _specfile_for(spec_str, filename):
             "a@1:",
         ),
         (
-            "% intel @ 12.1:12.6 + debug",
+            "+ debug % intel @ 12.1:12.6",
             [
-                Token(SpecTokens.COMPILER_AND_VERSION, value="% intel @ 12.1:12.6"),
                 Token(SpecTokens.BOOL_VARIANT, value="+ debug"),
+                Token(SpecTokens.COMPILER_AND_VERSION, value="% intel @ 12.1:12.6"),
             ],
-            "%intel@12.1:12.6+debug",
+            "+debug %intel@12.1:12.6",
         ),
         (
             "@ 12.1:12.6 + debug - qt_4",
@@ -515,7 +516,7 @@ def _specfile_for(spec_str, filename):
                 Token(SpecTokens.VERSION, value="@:0.4"),
                 Token(SpecTokens.COMPILER, value="% nvhpc"),
             ],
-            "@:0.4%nvhpc",
+            "@:0.4 %nvhpc",
         ),
         (
             "^[virtuals=mpi] openmpi",
@@ -639,15 +640,15 @@ def test_parse_single_spec(spec_str, tokens, expected_roundtrip, mock_git_test_p
             ["mvapich cppflags=-O3", "emacs"],
         ),
         (
-            "mvapich emacs @1.1.1 %intel cflags=-O3",
+            "mvapich emacs @1.1.1 cflags=-O3 %intel",
             [
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich"),
                 Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="emacs"),
                 Token(SpecTokens.VERSION, value="@1.1.1"),
-                Token(SpecTokens.COMPILER, value="%intel"),
                 Token(SpecTokens.KEY_VALUE_PAIR, value="cflags=-O3"),
+                Token(SpecTokens.COMPILER, value="%intel"),
             ],
-            ["mvapich", "emacs @1.1.1 %intel cflags=-O3"],
+            ["mvapich", "emacs @1.1.1 cflags=-O3 %intel"],
         ),
         (
             'mvapich cflags="-O3 -fPIC" emacs^ncurses%intel',
@@ -877,9 +878,7 @@ def test_ambiguous_hash(mutable_database):
     x1 = spack.concretize.concretize_one("pkg-a")
     x2 = x1.copy()
     x1._hash = "xyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"
-    x1._process_hash = "xyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"
     x2._hash = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-    x2._process_hash = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 
     assert x1 != x2  # doesn't hold when only the dag hash is modified.
 
@@ -1231,7 +1230,7 @@ def test_compare_abstract_specs():
         "foo.foo@foo+foo",
         "foo.foo@foo+foo arch=foo-foo-foo",
         "foo.foo@foo+foo arch=foo-foo-foo %foo",
-        "foo.foo@foo+foo arch=foo-foo-foo %foo cflags=foo",
+        "foo.foo@foo+foo arch=foo-foo-foo cflags=foo %foo",
     ]
     specs = [SpecParser(s).next_spec() for s in constraints]
 
@@ -1285,3 +1284,19 @@ def test_git_ref_spec_equivalences(mock_packages, lhs_str, rhs_str, expected):
 def test_platform_is_none_if_not_present(spec_str):
     s = SpecParser(spec_str).next_spec()
     assert s.architecture.platform is None, s
+
+
+def test_parse_one_or_raise_error_message():
+    with pytest.raises(ValueError) as exc:
+        parse_one_or_raise("  x y   z")
+
+    msg = """\
+expected a single spec, but got more:
+  x y   z
+    ^\
+"""
+
+    assert str(exc.value) == msg
+
+    with pytest.raises(ValueError, match="expected a single spec, but got none"):
+        parse_one_or_raise("    ")

lib/spack/spack/test/spec_yaml.py

@@ -203,13 +203,6 @@ def test_ordered_read_not_required_for_consistent_dag_hash(
     spec = spec.copy(deps=ht.dag_hash.depflag)
 
     # specs and their hashes are equal to the original
-    assert (
-        spec.process_hash()
-        == from_yaml.process_hash()
-        == from_json.process_hash()
-        == from_yaml_rev.process_hash()
-        == from_json_rev.process_hash()
-    )
     assert (
         spec.dag_hash()
         == from_yaml.dag_hash()

lib/spack/spack/test/util/environment.py

@@ -128,17 +128,21 @@ def test_dump_environment(prepare_environment_for_tests, shell_as, shell, tmpdir
 
 
 def test_reverse_environment_modifications(working_env):
+    prepend_val = os.sep + os.path.join("new", "path", "prepended")
+    append_val = os.sep + os.path.join("new", "path", "appended")
+
     start_env = {
-        "PREPEND_PATH": os.sep + os.path.join("path", "to", "prepend", "to"),
-        "APPEND_PATH": os.sep + os.path.join("path", "to", "append", "to"),
+        "PREPEND_PATH": prepend_val + os.pathsep + os.path.join("path", "to", "prepend", "to"),
+        "APPEND_PATH": os.path.sep
+        + os.path.join("path", "to", "append", "to" + os.pathsep + append_val),
         "UNSET": "var_to_unset",
         "APPEND_FLAGS": "flags to append to",
     }
 
     to_reverse = envutil.EnvironmentModifications()
 
-    to_reverse.prepend_path("PREPEND_PATH", "/new/path/prepended")
-    to_reverse.append_path("APPEND_PATH", "/new/path/appended")
+    to_reverse.prepend_path("PREPEND_PATH", prepend_val)
+    to_reverse.append_path("APPEND_PATH", append_val)
     to_reverse.set_path("SET_PATH", ["/one/set/path", "/two/set/path"])
     to_reverse.set("SET", "a var")
     to_reverse.unset("UNSET")

lib/spack/spack/test/util/git.py

@@ -0,0 +1,15 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from llnl.util.filesystem import working_dir
+
+from spack.util.git import get_modified_files
+
+
+def test_modified_files(mock_git_package_changes):
+    repo_path, filename, commits = mock_git_package_changes
+
+    with working_dir(repo_path):
+        files = get_modified_files(from_ref="HEAD~1", to_ref="HEAD")
+        assert len(files) == 1
+        assert files[0] == filename

lib/spack/spack/test/util/path.py

@@ -134,3 +134,18 @@ def test_path_debug_padded_filter(debug, monkeypatch):
     monkeypatch.setattr(tty, "_debug", debug)
     with spack.config.override("config:install_tree", {"padded_length": 128}):
         assert expected == sup.debug_padded_filter(string)
+
+
+@pytest.mark.parametrize(
+    "path,expected",
+    [
+        ("/home/spack/path/to/file.txt", "/home/spack/path/to/file.txt"),
+        ("file:///home/another/config.yaml", "/home/another/config.yaml"),
+        ("path/to.txt", os.path.join(os.environ["SPACK_ROOT"], "path", "to.txt")),
+        (r"C:\Files (x86)\Windows\10", r"C:\Files (x86)\Windows\10"),
+        (r"E:/spack stage", "E:\\spack stage"),
+    ],
+)
+def test_canonicalize_file(path, expected):
+    """Confirm canonicalize path handles local files and file URLs."""
+    assert sup.canonicalize_path(path) == os.path.normpath(expected)

lib/spack/spack/test/util/remote_file_cache.py

@@ -0,0 +1,106 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os.path
+import sys
+
+import pytest
+
+import llnl.util.tty as tty
+from llnl.util.filesystem import join_path
+
+import spack.config
+import spack.util.remote_file_cache as rfc_util
+
+github_url = "https://github.com/fake/fake/{0}/develop"
+gitlab_url = "https://gitlab.fake.io/user/repo/-/blob/config/defaults"
+
+
+@pytest.mark.parametrize(
+    "path,err",
+    [
+        ("ssh://git@github.com:spack/", "Unsupported URL scheme"),
+        ("bad:///this/is/a/file/url/include.yaml", "Invalid URL scheme"),
+    ],
+)
+def test_rfc_local_path_bad_scheme(path, err):
+    with pytest.raises(ValueError, match=err):
+        _ = rfc_util.local_path(path, "")
+
+
+@pytest.mark.parametrize(
+    "path,expected",
+    [
+        ("/a/b/c/d/e/config.py", "/a/b/c/d/e/config.py"),
+        ("file:///this/is/a/file/url/include.yaml", "/this/is/a/file/url/include.yaml"),
+        (
+            "relative/packages.txt",
+            os.path.join(os.environ["SPACK_ROOT"], "relative", "packages.txt"),
+        ),
+        (r"C:\Files (x86)\Windows\10", r"C:\Files (x86)\Windows\10"),
+        (r"D:/spack stage", "D:\\spack stage"),
+    ],
+)
+def test_rfc_local_file(path, expected):
+    assert rfc_util.local_path(path, "") == os.path.normpath(expected)
+
+
+def test_rfc_remote_local_path_no_dest():
+    path = f"{gitlab_url}/packages.yaml"
+    with pytest.raises(ValueError, match="Requires the destination argument"):
+        _ = rfc_util.local_path(path, "")
+
+
+compilers_sha256 = (
+    "381732677538143a8f900406c0654f2730e2919a11740bdeaf35757ab3e1ef3e"
+    if sys.platform == "win32"
+    else "e91148ed5a0da7844e9f3f9cfce0fa60cce509461886bc3b006ee9eb711f69df"
+)
+
+
+@pytest.mark.parametrize(
+    "url,sha256,err,msg",
+    [
+        (
+            f"{join_path(github_url.format('tree'), 'config.yaml')}",
+            "",
+            ValueError,
+            "Requires sha256",
+        ),
+        (f"{gitlab_url}/compilers.yaml", compilers_sha256, None, ""),
+        (f"{gitlab_url}/packages.yaml", "abcdef", ValueError, "does not match"),
+        (f"{github_url.format('blob')}/README.md", "", OSError, "No such"),
+        (github_url.format("tree"), "", OSError, "No such"),
+        ("", "", ValueError, "argument is required"),
+    ],
+)
+def test_rfc_remote_local_path(
+    tmpdir, mutable_empty_config, mock_fetch_url_text, url, sha256, err, msg
+):
+    def _has_content(filename):
+        # The first element of all configuration files for this test happen to
+        # be the basename of the file so this check leverages that feature. If
+        # that changes, then this check will need to change accordingly.
+        element = f"{os.path.splitext(os.path.basename(filename))[0]}:"
+        with open(filename, "r", encoding="utf-8") as fd:
+            for line in fd:
+                if element in line:
+                    return True
+        tty.debug(f"Expected {element} in '{filename}'")
+        return False
+
+    def _dest_dir():
+        return join_path(tmpdir.strpath, "cache")
+
+    if err is not None:
+        with spack.config.override("config:url_fetch_method", "curl"):
+            with pytest.raises(err, match=msg):
+                rfc_util.local_path(url, sha256, _dest_dir)
+    else:
+        with spack.config.override("config:url_fetch_method", "curl"):
+            path = rfc_util.local_path(url, sha256, _dest_dir)
+            assert os.path.exists(path)
+            # Ensure correct file is "fetched"
+            assert os.path.basename(path) == os.path.basename(url)
+            # Ensure contents of the file contains expected config element
+            assert _has_content(path)

lib/spack/spack/util/environment.py

@@ -363,6 +363,30 @@ def execute(self, env: MutableMapping[str, str]):
         env[self.name] = self.separator.join(directories)
 
 
+class RemoveFirstPath(NameValueModifier):
+    def execute(self, env: MutableMapping[str, str]):
+        tty.debug(f"RemoveFirstPath: {self.name}-{str(self.value)}", level=3)
+        environment_value = env.get(self.name, "")
+        directories = environment_value.split(self.separator)
+        directories = [path_to_os_path(os.path.normpath(x)).pop() for x in directories]
+        val = path_to_os_path(os.path.normpath(self.value)).pop()
+        if val in directories:
+            directories.remove(val)
+        env[self.name] = self.separator.join(directories)
+
+
+class RemoveLastPath(NameValueModifier):
+    def execute(self, env: MutableMapping[str, str]):
+        tty.debug(f"RemoveLastPath: {self.name}-{str(self.value)}", level=3)
+        environment_value = env.get(self.name, "")
+        directories = environment_value.split(self.separator)[::-1]
+        directories = [path_to_os_path(os.path.normpath(x)).pop() for x in directories]
+        val = path_to_os_path(os.path.normpath(self.value)).pop()
+        if val in directories:
+            directories.remove(val)
+        env[self.name] = self.separator.join(directories[::-1])
+
+
 class RemovePath(NameValueModifier):
     def execute(self, env: MutableMapping[str, str]):
         tty.debug(f"RemovePath: {self.name}-{str(self.value)}", level=3)
@@ -533,6 +557,30 @@ def prepend_path(self, name: str, path: str, separator: str = os.pathsep):
         item = PrependPath(name, path, separator=separator, trace=self._trace())
         self.env_modifications.append(item)
 
+    @system_env_normalize
+    def remove_first_path(self, name: str, path: str, separator: str = os.pathsep):
+        """Stores a request to remove first instance of path from a list of paths.
+
+        Args:
+            name: name of the environment variable
+            path: path to be removed
+            separator: separator for the paths (default: os.pathsep)
+        """
+        item = RemoveFirstPath(name, path, separator=separator, trace=self._trace())
+        self.env_modifications.append(item)
+
+    @system_env_normalize
+    def remove_last_path(self, name: str, path: str, separator: str = os.pathsep):
+        """Stores a request to remove last instance of path from a list of paths.
+
+        Args:
+            name: name of the environment variable
+            path: path to be removed
+            separator: separator for the paths (default: os.pathsep)
+        """
+        item = RemoveLastPath(name, path, separator=separator, trace=self._trace())
+        self.env_modifications.append(item)
+
     @system_env_normalize
     def remove_path(self, name: str, path: str, separator: str = os.pathsep):
         """Stores a request to remove a path from a list of paths.
@@ -613,9 +661,9 @@ def reversed(self) -> "EnvironmentModifications":
                 tty.debug("Reversing `Set` environment operation may lose the original value")
                 rev.unset(envmod.name)
             elif isinstance(envmod, AppendPath):
-                rev.remove_path(envmod.name, envmod.value)
+                rev.remove_last_path(envmod.name, envmod.value)
             elif isinstance(envmod, PrependPath):
-                rev.remove_path(envmod.name, envmod.value)
+                rev.remove_first_path(envmod.name, envmod.value)
             elif isinstance(envmod, SetPath):
                 tty.debug("Reversing `SetPath` environment operation may lose the original value")
                 rev.unset(envmod.name)

lib/spack/spack/util/git.py

@@ -4,7 +4,7 @@
 """Single util module where Spack should get a git executable."""
 
 import sys
-from typing import Optional
+from typing import List, Optional
 
 import llnl.util.lang
 
@@ -26,3 +26,17 @@ def git(required: bool = False):
         git.add_default_arg("-c", "protocol.file.allow=always")
 
     return git
+
+
+def get_modified_files(from_ref: str = "HEAD~1", to_ref: str = "HEAD") -> List[str]:
+    """Get a list of files modified between `from_ref` and `to_ref`
+    Args:
+       from_ref (str): oldest git ref, defaults to `HEAD~1`
+       to_ref (str): newer git ref, defaults to `HEAD`
+    Returns: list of file paths
+    """
+    git_exe = git(required=True)
+
+    stdout = git_exe("diff", "--name-only", from_ref, to_ref, output=str)
+
+    return stdout.split()

lib/spack/spack/util/path.py

@@ -9,11 +9,13 @@
 import contextlib
 import getpass
 import os
+import pathlib
 import re
 import subprocess
 import sys
 import tempfile
 from datetime import date
+from typing import Optional
 
 import llnl.util.tty as tty
 from llnl.util.lang import memoized
@@ -235,7 +237,7 @@ def add_padding(path, length):
     return os.path.join(path, padding)
 
 
-def canonicalize_path(path, default_wd=None):
+def canonicalize_path(path: str, default_wd: Optional[str] = None) -> str:
     """Same as substitute_path_variables, but also take absolute path.
 
     If the string is a yaml object with file annotations, make absolute paths
@@ -243,28 +245,53 @@ def canonicalize_path(path, default_wd=None):
     Otherwise, use ``default_wd`` if specified, otherwise ``os.getcwd()``
 
     Arguments:
-        path (str): path being converted as needed
+        path: path being converted as needed
+        default_wd: optional working directory/root for non-yaml string paths
 
-    Returns:
-        (str): An absolute path with path variable substitution
+    Returns: An absolute path or non-file URL with path variable substitution
     """
+    import urllib.parse
+    import urllib.request
+
     # Get file in which path was written in case we need to make it absolute
     # relative to that path.
     filename = None
     if isinstance(path, syaml.syaml_str):
-        filename = os.path.dirname(path._start_mark.name)
-        assert path._start_mark.name == path._end_mark.name
+        filename = os.path.dirname(path._start_mark.name)  # type: ignore[attr-defined]
+        assert path._start_mark.name == path._end_mark.name  # type: ignore[attr-defined]
 
     path = substitute_path_variables(path)
-    if not os.path.isabs(path):
-        if filename:
-            path = os.path.join(filename, path)
-        else:
-            base = default_wd or os.getcwd()
-            path = os.path.join(base, path)
-            tty.debug("Using working directory %s as base for abspath" % base)
 
-    return os.path.normpath(path)
+    # Ensure properly process a Windows path
+    win_path = pathlib.PureWindowsPath(path)
+    if win_path.drive:
+        # Assume only absolute paths are supported with a Windows drive
+        # (though DOS does allow drive-relative paths).
+        return os.path.normpath(str(win_path))
+
+    # Now process linux-like paths and remote URLs
+    url = urllib.parse.urlparse(path)
+    url_path = urllib.request.url2pathname(url.path)
+    if url.scheme:
+        if url.scheme != "file":
+            # Have a remote URL so simply return it with substitutions
+            return os.path.normpath(path)
+
+        # Drop the URL scheme from the local path
+        path = url_path
+
+    if os.path.isabs(path):
+        return os.path.normpath(path)
+
+    # Have a relative path so prepend the appropriate dir to make it absolute
+    if filename:
+        # Prepend the directory of the syaml path
+        return os.path.normpath(os.path.join(filename, path))
+
+    # Prepend the default, if provided, or current working directory.
+    base = default_wd or os.getcwd()
+    tty.debug(f"Using working directory {base} as base for abspath")
+    return os.path.normpath(os.path.join(base, path))
 
 
 def longest_prefix_re(string, capture=True):
@@ -347,6 +374,7 @@ def filter_padding():
     This is needed because Spack's debug output gets extremely long when we use a
     long padded installation path.
     """
+    # circular import
     import spack.config
 
     padding = spack.config.get("config:install_tree:padded_length", None)

lib/spack/spack/util/remote_file_cache.py

@@ -0,0 +1,146 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import hashlib
+import os.path
+import pathlib
+import shutil
+import tempfile
+import urllib.parse
+import urllib.request
+from typing import Callable, Optional
+
+import llnl.util.tty as tty
+from llnl.util.filesystem import copy, join_path, mkdirp
+
+import spack.util.crypto
+from spack.util.path import canonicalize_path
+from spack.util.url import validate_scheme
+
+
+def raw_github_gitlab_url(url: str) -> str:
+    """Transform a github URL to the raw form to avoid undesirable html.
+
+    Args:
+        url: url to be converted to raw form
+
+    Returns:
+        Raw github/gitlab url or the original url
+    """
+    # Note we rely on GitHub to redirect the 'raw' URL returned here to the
+    # actual URL under https://raw.githubusercontent.com/ with '/blob'
+    # removed and or, '/blame' if needed.
+    if "github" in url or "gitlab" in url:
+        return url.replace("/blob/", "/raw/")
+
+    return url
+
+
+def fetch_remote_text_file(url: str, dest_dir: str) -> str:
+    """Retrieve the text file from the url into the destination directory.
+
+    Arguments:
+        url: URL for the remote text file
+        dest_dir: destination directory in which to stage the file locally
+
+    Returns:
+        Path to the fetched file
+
+    Raises:
+        ValueError: if there are missing required arguments
+    """
+    from spack.util.web import fetch_url_text  # circular import
+
+    if not url:
+        raise ValueError("Cannot retrieve the remote file without the URL")
+
+    raw_url = raw_github_gitlab_url(url)
+    tty.debug(f"Fetching file from {raw_url} into {dest_dir}")
+
+    return fetch_url_text(raw_url, dest_dir=dest_dir)
+
+
+def local_path(raw_path: str, sha256: str, make_dest: Optional[Callable[[], str]] = None) -> str:
+    """Determine the actual path and, if remote, stage its contents locally.
+
+    Args:
+        raw_path: raw path with possible variables needing substitution
+        sha256: the expected sha256 for the file
+        make_dest: function to create a stage for remote files, if needed (e.g., `mkdtemp`)
+
+    Returns: resolved, normalized local path
+
+    Raises:
+        ValueError: missing or mismatched arguments, unsupported URL scheme
+    """
+    if not raw_path:
+        raise ValueError("path argument is required to cache remote files")
+
+    file_schemes = ["", "file"]
+
+    # Allow paths (and URLs) to contain spack config/environment variables,
+    # etc.
+    path = canonicalize_path(raw_path)
+
+    # Save off the Windows drive of the canonicalized path (since now absolute)
+    # to ensure recognized by URL parsing as a valid file "scheme".
+    win_path = pathlib.PureWindowsPath(path)
+    if win_path.drive:
+        file_schemes.append(win_path.drive.lower().strip(":"))
+
+    url = urllib.parse.urlparse(path)
+
+    # Path isn't remote so return normalized, absolute path with substitutions.
+    if url.scheme in file_schemes:
+        return os.path.normpath(path)
+
+    # If scheme is not valid, path is not a supported url.
+    if validate_scheme(url.scheme):
+        # Fetch files from supported URL schemes.
+        if url.scheme in ("http", "https", "ftp"):
+            if make_dest is None:
+                raise ValueError("Requires the destination argument to cache remote files")
+
+            # Stage the remote configuration file
+            tmpdir = tempfile.mkdtemp()
+            try:
+                staged_path = fetch_remote_text_file(path, tmpdir)
+
+                # Ensure the sha256 is expected.
+                checksum = spack.util.crypto.checksum(hashlib.sha256, staged_path)
+                if sha256 and checksum != sha256:
+                    raise ValueError(
+                        f"Actual sha256 ('{checksum}') does not match expected ('{sha256}')"
+                    )
+
+                # Help the user by reporting the required checksum.
+                if not sha256:
+                    raise ValueError(f"Requires sha256 ('{checksum}') to cache remote files.")
+
+                # Copy the file to the destination directory
+                dest_dir = join_path(make_dest(), checksum)
+                if not os.path.exists(dest_dir):
+                    mkdirp(dest_dir)
+
+                cache_path = join_path(dest_dir, os.path.basename(staged_path))
+                copy(staged_path, cache_path)
+                tty.debug(f"Cached {raw_path} in {cache_path}")
+
+                # Stash the associated URL to aid with debugging
+                with open(join_path(dest_dir, "source_url.txt"), "w", encoding="utf-8") as f:
+                    f.write(f"{raw_path}\n")
+
+                return cache_path
+
+            except ValueError as err:
+                tty.warn(f"Unable to cache {raw_path}: {str(err)}")
+                raise
+
+            finally:
+                shutil.rmtree(tmpdir)
+
+        raise ValueError(f"Unsupported URL scheme ({url.scheme}) in {raw_path}")
+
+    else:
+        raise ValueError(f"Invalid URL scheme ({url.scheme}) in {raw_path}")

lib/spack/spack/util/spack_yaml.py

@@ -495,3 +495,25 @@ class SpackYAMLError(spack.error.SpackError):
 
     def __init__(self, msg, yaml_error):
         super().__init__(msg, str(yaml_error))
+
+
+def get_mark_from_yaml_data(obj):
+    """Try to get ``spack.util.spack_yaml`` mark from YAML data.
+
+    We try the object, and if that fails we try its first member (if it's a container).
+
+    Returns:
+        mark if one is found, otherwise None.
+    """
+    # mark of object itelf
+    mark = getattr(obj, "_start_mark", None)
+    if mark:
+        return mark
+
+    # mark of first member if it is a container
+    if isinstance(obj, (list, dict)):
+        first_member = next(iter(obj), None)
+        if first_member:
+            mark = getattr(first_member, "_start_mark", None)
+
+    return mark

share/spack/bootstrap/github-actions-v0.5/clingo.json

@@ -9,7 +9,7 @@
         ]
       ],
       "python": "python@3.10.13",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64"
+      "spec": "clingo-bootstrap platform=darwin target=aarch64 %apple-clang"
     },
     {
       "binaries": [
@@ -20,7 +20,7 @@
         ]
       ],
       "python": "python@3.11.5",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64"
+      "spec": "clingo-bootstrap platform=darwin target=aarch64 %apple-clang"
     },
     {
       "binaries": [
@@ -31,7 +31,7 @@
         ]
       ],
       "python": "python@3.12.0",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64"
+      "spec": "clingo-bootstrap platform=darwin target=aarch64 %apple-clang"
     },
     {
       "binaries": [
@@ -42,7 +42,7 @@
         ]
       ],
       "python": "python@3.6",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64"
+      "spec": "clingo-bootstrap platform=darwin target=aarch64 %apple-clang"
     },
     {
       "binaries": [
@@ -53,7 +53,7 @@
         ]
       ],
       "python": "python@3.7.17",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64"
+      "spec": "clingo-bootstrap platform=darwin target=aarch64 %apple-clang"
     },
     {
       "binaries": [
@@ -64,7 +64,7 @@
         ]
       ],
       "python": "python@3.8.18",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64"
+      "spec": "clingo-bootstrap platform=darwin target=aarch64 %apple-clang"
     },
     {
       "binaries": [
@@ -75,7 +75,7 @@
         ]
       ],
       "python": "python@3.9.18",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=aarch64"
+      "spec": "clingo-bootstrap platform=darwin target=aarch64 %apple-clang"
     },
     {
       "binaries": [
@@ -86,7 +86,7 @@
         ]
       ],
       "python": "python@3.10",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"
+      "spec": "clingo-bootstrap platform=darwin target=x86_64 %apple-clang"
     },
     {
       "binaries": [
@@ -97,7 +97,7 @@
         ]
       ],
       "python": "python@3.11",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"
+      "spec": "clingo-bootstrap platform=darwin target=x86_64 %apple-clang"
     },
     {
       "binaries": [
@@ -108,7 +108,7 @@
         ]
       ],
       "python": "python@3.12",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"
+      "spec": "clingo-bootstrap platform=darwin target=x86_64 %apple-clang"
     },
     {
       "binaries": [
@@ -119,7 +119,7 @@
         ]
       ],
       "python": "python@3.6",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"
+      "spec": "clingo-bootstrap platform=darwin target=x86_64 %apple-clang"
     },
     {
       "binaries": [
@@ -130,7 +130,7 @@
         ]
       ],
       "python": "python@3.7",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"
+      "spec": "clingo-bootstrap platform=darwin target=x86_64 %apple-clang"
     },
     {
       "binaries": [
@@ -141,7 +141,7 @@
         ]
       ],
       "python": "python@3.8",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"
+      "spec": "clingo-bootstrap platform=darwin target=x86_64 %apple-clang"
     },
     {
       "binaries": [
@@ -152,7 +152,7 @@
         ]
       ],
       "python": "python@3.9",
-      "spec": "clingo-bootstrap%apple-clang platform=darwin target=x86_64"
+      "spec": "clingo-bootstrap platform=darwin target=x86_64 %apple-clang"
     },
     {
       "binaries": [
@@ -163,7 +163,7 @@
         ]
       ],
       "python": "python@3.10.13",
-      "spec": "clingo-bootstrap%gcc platform=linux target=aarch64"
+      "spec": "clingo-bootstrap platform=linux target=aarch64 %gcc"
     },
     {
       "binaries": [
@@ -174,7 +174,7 @@
         ]
       ],
       "python": "python@3.11.5",
-      "spec": "clingo-bootstrap%gcc platform=linux target=aarch64"
+      "spec": "clingo-bootstrap platform=linux target=aarch64 %gcc"
     },
     {
       "binaries": [
@@ -185,7 +185,7 @@
         ]
       ],
       "python": "python@3.12.0",
-      "spec": "clingo-bootstrap%gcc platform=linux target=aarch64"
+      "spec": "clingo-bootstrap platform=linux target=aarch64 %gcc"
     },
     {
       "binaries": [
@@ -196,7 +196,7 @@
         ]
       ],
       "python": "python@3.6",
-      "spec": "clingo-bootstrap%gcc platform=linux target=aarch64"
+      "spec": "clingo-bootstrap platform=linux target=aarch64 %gcc"
     },
     {
       "binaries": [
@@ -207,7 +207,7 @@
         ]
       ],
       "python": "python@3.7.17",
-      "spec": "clingo-bootstrap%gcc platform=linux target=aarch64"
+      "spec": "clingo-bootstrap platform=linux target=aarch64 %gcc"
     },
     {
       "binaries": [
@@ -218,7 +218,7 @@
         ]
       ],
       "python": "python@3.8.18",
-      "spec": "clingo-bootstrap%gcc platform=linux target=aarch64"
+      "spec": "clingo-bootstrap platform=linux target=aarch64 %gcc"
     },
     {
       "binaries": [
@@ -229,7 +229,7 @@
         ]
       ],
       "python": "python@3.9.18",
-      "spec": "clingo-bootstrap%gcc platform=linux target=aarch64"
+      "spec": "clingo-bootstrap platform=linux target=aarch64 %gcc"
     },
     {
       "binaries": [
@@ -240,7 +240,7 @@
         ]
       ],
       "python": "python@3.10.13",
-      "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"
+      "spec": "clingo-bootstrap platform=linux target=ppc64le %gcc"
     },
     {
       "binaries": [
@@ -251,7 +251,7 @@
         ]
       ],
       "python": "python@3.11.5",
-      "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"
+      "spec": "clingo-bootstrap platform=linux target=ppc64le %gcc"
     },
     {
       "binaries": [
@@ -262,7 +262,7 @@
         ]
       ],
       "python": "python@3.12.0",
-      "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"
+      "spec": "clingo-bootstrap platform=linux target=ppc64le %gcc"
     },
     {
       "binaries": [
@@ -273,7 +273,7 @@
         ]
       ],
       "python": "python@3.6",
-      "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"
+      "spec": "clingo-bootstrap platform=linux target=ppc64le %gcc"
     },
     {
       "binaries": [
@@ -284,7 +284,7 @@
         ]
       ],
       "python": "python@3.7.17",
-      "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"
+      "spec": "clingo-bootstrap platform=linux target=ppc64le %gcc"
     },
     {
       "binaries": [
@@ -295,7 +295,7 @@
         ]
       ],
       "python": "python@3.8.18",
-      "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"
+      "spec": "clingo-bootstrap platform=linux target=ppc64le %gcc"
     },
     {
       "binaries": [
@@ -306,7 +306,7 @@
         ]
       ],
       "python": "python@3.9.18",
-      "spec": "clingo-bootstrap%gcc platform=linux target=ppc64le"
+      "spec": "clingo-bootstrap platform=linux target=ppc64le %gcc"
     },
     {
       "binaries": [
@@ -317,7 +317,7 @@
         ]
       ],
       "python": "python@3.10.13",
-      "spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
+      "spec": "clingo-bootstrap platform=linux target=x86_64 %gcc"
     },
     {
       "binaries": [
@@ -328,7 +328,7 @@
         ]
       ],
       "python": "python@3.11.5",
-      "spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
+      "spec": "clingo-bootstrap platform=linux target=x86_64 %gcc"
     },
     {
       "binaries": [
@@ -339,7 +339,7 @@
         ]
       ],
       "python": "python@3.12.0",
-      "spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
+      "spec": "clingo-bootstrap platform=linux target=x86_64 %gcc"
     },
     {
       "binaries": [
@@ -350,7 +350,7 @@
         ]
       ],
       "python": "python@3.6",
-      "spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
+      "spec": "clingo-bootstrap platform=linux target=x86_64 %gcc"
     },
     {
       "binaries": [
@@ -361,7 +361,7 @@
         ]
       ],
       "python": "python@3.7.17",
-      "spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
+      "spec": "clingo-bootstrap platform=linux target=x86_64 %gcc"
     },
     {
       "binaries": [
@@ -372,7 +372,7 @@
         ]
       ],
       "python": "python@3.8.18",
-      "spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
+      "spec": "clingo-bootstrap platform=linux target=x86_64 %gcc"
     },
     {
       "binaries": [
@@ -383,7 +383,7 @@
         ]
       ],
       "python": "python@3.9.18",
-      "spec": "clingo-bootstrap%gcc platform=linux target=x86_64"
+      "spec": "clingo-bootstrap platform=linux target=x86_64 %gcc"
     }
   ]
 }

share/spack/bootstrap/github-actions-v0.5/gnupg.json

@@ -48,7 +48,7 @@
           "23fdd223493f441fa2e5f82d7e02837ecfad831fbfa4c27c175b3e294ed977d1"
         ]
       ],
-      "spec": "gnupg@2.3: %apple-clang platform=darwin target=aarch64"
+      "spec": "gnupg@2.3: platform=darwin target=aarch64 %apple-clang"
     },
     {
       "binaries": [
@@ -98,7 +98,7 @@
           "b9481e122e2cb26f69b70505830d0fcc0d200aadbb6c6572339825f17ad1e52d"
         ]
       ],
-      "spec": "gnupg@2.3: %apple-clang platform=darwin target=x86_64"
+      "spec": "gnupg@2.3: platform=darwin target=x86_64 %apple-clang"
     },
     {
       "binaries": [
@@ -148,7 +148,7 @@
           "228ccb475932f7f40a64e9d87dec045931cc57f71b1dfd4b4c3926107222d96c"
         ]
       ],
-      "spec": "gnupg@2.3: %gcc platform=linux target=aarch64"
+      "spec": "gnupg@2.3: platform=linux target=aarch64 %gcc"
     },
     {
       "binaries": [
@@ -198,7 +198,7 @@
           "98e2bcb4064ec0830d896938bc1fe5264dac611da71ea546b9ca03349b752041"
         ]
       ],
-      "spec": "gnupg@2.3: %gcc platform=linux target=ppc64le"
+      "spec": "gnupg@2.3: platform=linux target=ppc64le %gcc"
     },
     {
       "binaries": [
@@ -248,7 +248,7 @@
           "054fac6eaad7c862ea4661461d847fb069876eb114209416b015748266f7d166"
         ]
       ],
-      "spec": "gnupg@2.3: %gcc platform=linux target=x86_64"
+      "spec": "gnupg@2.3: platform=linux target=x86_64 %gcc"
     }
   ]
 }

share/spack/bootstrap/github-actions-v0.5/patchelf.json

@@ -8,7 +8,7 @@
           "102800775f789cc293e244899f39a22f0b7a19373305ef0497ca3189223123f3"
         ]
       ],
-      "spec": "patchelf@0.13: %gcc platform=linux target=aarch64"
+      "spec": "patchelf@0.13: platform=linux target=aarch64 %gcc"
     },
     {
       "binaries": [
@@ -18,7 +18,7 @@
           "91cf0a9d4750c04575c5ed3bcdefc4754e1cf9d1cd1bf197eb1fe20ccaa869f1"
         ]
       ],
-      "spec": "patchelf@0.13: %gcc platform=linux target=ppc64le"
+      "spec": "patchelf@0.13: platform=linux target=ppc64le %gcc"
     },
     {
       "binaries": [
@@ -28,7 +28,7 @@
           "73f4bde46b843c96521e3f5c31ab94756491404c1ad6429c9f61dbafbbfa6470"
         ]
       ],
-      "spec": "patchelf@0.13: %gcc platform=linux target=x86_64"
+      "spec": "patchelf@0.13: platform=linux target=x86_64 %gcc"
     }
   ]
 }

share/spack/bootstrap/github-actions-v0.6/clingo.json

@@ -8,7 +8,7 @@
           "ff7f45db1645d1d857a315bf8d63c31447330552528bdf1fccdcf50735e62166"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=aarch64 ^python@3.10"
+      "spec": "clingo-bootstrap@spack platform=darwin target=aarch64 %apple-clang ^python@3.10"
     },
     {
       "binaries": [
@@ -18,7 +18,7 @@
           "e7491ac297cbb3f45c80aaf4ca5102e2b655b731e7b6ce7682807d302cb61f1c"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=aarch64 ^python@3.11"
+      "spec": "clingo-bootstrap@spack platform=darwin target=aarch64 %apple-clang ^python@3.11"
     },
     {
       "binaries": [
@@ -28,7 +28,7 @@
           "91214626a86c21fc0d76918884ec819050d4d52b4f78df7cc9769a83fbee2f71"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=aarch64 ^python@3.12"
+      "spec": "clingo-bootstrap@spack platform=darwin target=aarch64 %apple-clang ^python@3.12"
     },
     {
       "binaries": [
@@ -38,7 +38,7 @@
           "db596d9e6d8970d659f4be4cb510f9ba5dc2ec4ea42ecf2aed1325ec5ad72b45"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=aarch64 ^python@3.13"
+      "spec": "clingo-bootstrap@spack platform=darwin target=aarch64 %apple-clang ^python@3.13"
     },
     {
       "binaries": [
@@ -48,7 +48,7 @@
           "a7ed91aee1f8d5cfe2ca5ef2e3e74215953ffd2d8b5c722a670f2c303610e90c"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=aarch64 ^python@3.8"
+      "spec": "clingo-bootstrap@spack platform=darwin target=aarch64 %apple-clang ^python@3.8"
     },
     {
       "binaries": [
@@ -58,7 +58,7 @@
           "c856a98f92b9fa218377cea9272dffa736e93251d987b6386e6abf40058333dc"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=aarch64 ^python@3.9"
+      "spec": "clingo-bootstrap@spack platform=darwin target=aarch64 %apple-clang ^python@3.9"
     },
     {
       "binaries": [
@@ -68,7 +68,7 @@
           "d74cc0b44faa69473816dca16a3806123790e6eb9a59f611b1d80da7843f474a"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=x86_64 ^python@3.10"
+      "spec": "clingo-bootstrap@spack platform=darwin target=x86_64 %apple-clang ^python@3.10"
     },
     {
       "binaries": [
@@ -78,7 +78,7 @@
           "2cb12477504ca8e112522b6d56325ce32024c9466de5b8427fd70a1a81b15020"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=x86_64 ^python@3.11"
+      "spec": "clingo-bootstrap@spack platform=darwin target=x86_64 %apple-clang ^python@3.11"
     },
     {
       "binaries": [
@@ -88,7 +88,7 @@
           "4e73426599fa61df1a4faceafa38ade170a3dec45b6d8f333e6c2b6bfe697724"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=x86_64 ^python@3.12"
+      "spec": "clingo-bootstrap@spack platform=darwin target=x86_64 %apple-clang ^python@3.12"
     },
     {
       "binaries": [
@@ -98,7 +98,7 @@
           "4309b42e5642bc5c83ede90759b1a0b5d66fffa8991b6213208577626b588cde"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=x86_64 ^python@3.13"
+      "spec": "clingo-bootstrap@spack platform=darwin target=x86_64 %apple-clang ^python@3.13"
     },
     {
       "binaries": [
@@ -108,7 +108,7 @@
           "1feeab9e1a81ca56de838ccc234d60957e9ab14da038e38b6687732b7bae1ff6"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=x86_64 ^python@3.6"
+      "spec": "clingo-bootstrap@spack platform=darwin target=x86_64 %apple-clang ^python@3.6"
     },
     {
       "binaries": [
@@ -118,7 +118,7 @@
           "1149ab7d5f1c82d8de53f048af8aa5c5dbf0d21da4e4780c06e54b8ee902085b"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=x86_64 ^python@3.7"
+      "spec": "clingo-bootstrap@spack platform=darwin target=x86_64 %apple-clang ^python@3.7"
     },
     {
       "binaries": [
@@ -128,7 +128,7 @@
           "d6aeae2dbd7fa3c1d1c62f840a5c01f8e71b64afe2bdc9c90d4087694f7d3443"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=x86_64 ^python@3.8"
+      "spec": "clingo-bootstrap@spack platform=darwin target=x86_64 %apple-clang ^python@3.8"
     },
     {
       "binaries": [
@@ -138,7 +138,7 @@
           "81ef2beef78f46979965e5e69cd92b68ff4d2a59dbae1331c648d18b6ded1444"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%apple-clang platform=darwin target=x86_64 ^python@3.9"
+      "spec": "clingo-bootstrap@spack platform=darwin target=x86_64 %apple-clang ^python@3.9"
     },
     {
       "binaries": [
@@ -148,7 +148,7 @@
           "3d0830654f9e327fd7ec6dab214050295dbf0832f493937c0c133e516dd2a95a"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=aarch64 ^python@3.10"
+      "spec": "clingo-bootstrap@spack platform=linux target=aarch64 %gcc ^python@3.10"
     },
     {
       "binaries": [
@@ -158,7 +158,7 @@
           "941b93cd89d5271c740d1b1c870e85f32e5970f9f7b842ad99870399215a93db"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=aarch64 ^python@3.11"
+      "spec": "clingo-bootstrap@spack platform=linux target=aarch64 %gcc ^python@3.11"
     },
     {
       "binaries": [
@@ -168,7 +168,7 @@
           "8ca78e345da732643e3d1b077d8156ce89863c25095e4958d4ac6d1a458ae74b"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=aarch64 ^python@3.12"
+      "spec": "clingo-bootstrap@spack platform=linux target=aarch64 %gcc ^python@3.12"
     },
     {
       "binaries": [
@@ -178,7 +178,7 @@
           "f6ced988b515494d86a1069f13ae9030caeb40fe951c2460f532123c80399154"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=aarch64 ^python@3.13"
+      "spec": "clingo-bootstrap@spack platform=linux target=aarch64 %gcc ^python@3.13"
     },
     {
       "binaries": [
@@ -188,7 +188,7 @@
           "c00855b5cda99639b87c3912ee9c734c0b609dfe7d2c47ea947738c32bab6f03"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=aarch64 ^python@3.6"
+      "spec": "clingo-bootstrap@spack platform=linux target=aarch64 %gcc ^python@3.6"
     },
     {
       "binaries": [
@@ -198,7 +198,7 @@
           "aa861cfdf6001fc2da6e83eecc9ad35df424d86d71f6d73e480818942405ce4e"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=aarch64 ^python@3.7"
+      "spec": "clingo-bootstrap@spack platform=linux target=aarch64 %gcc ^python@3.7"
     },
     {
       "binaries": [
@@ -208,7 +208,7 @@
           "cb7807cd31fc5e0efe2acc1de1f74c7cef962bcadfc656b09ff853bc33c11bd0"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=aarch64 ^python@3.8"
+      "spec": "clingo-bootstrap@spack platform=linux target=aarch64 %gcc ^python@3.8"
     },
     {
       "binaries": [
@@ -218,7 +218,7 @@
           "36e5efb6b15b431b661e9e272904ab3c29ae7b87bf6250c158d545ccefc2f424"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=aarch64 ^python@3.9"
+      "spec": "clingo-bootstrap@spack platform=linux target=aarch64 %gcc ^python@3.9"
     },
     {
       "binaries": [
@@ -228,7 +228,7 @@
           "bd492c078b2cdaf327148eee5b0abd5b068dbb3ffa5dae0ec5d53257f471f7f7"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=ppc64le ^python@3.10"
+      "spec": "clingo-bootstrap@spack platform=linux target=ppc64le %gcc ^python@3.10"
     },
     {
       "binaries": [
@@ -238,7 +238,7 @@
           "0ebe5e05246c33fc8529e90e21529b29742b5dd6756dbc07534577a90394c0e6"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=ppc64le ^python@3.11"
+      "spec": "clingo-bootstrap@spack platform=linux target=ppc64le %gcc ^python@3.11"
     },
     {
       "binaries": [
@@ -248,7 +248,7 @@
           "9f97d3bf78b7642a775f12feb43781d46110793f58a7e69b0b68ac4fff47655c"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=ppc64le ^python@3.12"
+      "spec": "clingo-bootstrap@spack platform=linux target=ppc64le %gcc ^python@3.12"
     },
     {
       "binaries": [
@@ -258,7 +258,7 @@
           "e7295bb4bcb11a936f39665632ce68c751c9f6cddc44904392a1b33a5290bbbe"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=ppc64le ^python@3.13"
+      "spec": "clingo-bootstrap@spack platform=linux target=ppc64le %gcc ^python@3.13"
     },
     {
       "binaries": [
@@ -268,7 +268,7 @@
           "c44e7fbf721383aa8ee57d2305f41377e64a42ab8e02a9d3d6fc792d9b29ad08"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=ppc64le ^python@3.6"
+      "spec": "clingo-bootstrap@spack platform=linux target=ppc64le %gcc ^python@3.6"
     },
     {
       "binaries": [
@@ -278,7 +278,7 @@
           "965ba5c1a42f436001162a3f3a0d1715424f2ec8f65c42d6b66efcd4f4566b77"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=ppc64le ^python@3.7"
+      "spec": "clingo-bootstrap@spack platform=linux target=ppc64le %gcc ^python@3.7"
     },
     {
       "binaries": [
@@ -288,7 +288,7 @@
           "c8d31089d8f91718a5bde9c6b28cc67bdbadab401c8fdd07b296d588ece4ddfe"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=ppc64le ^python@3.8"
+      "spec": "clingo-bootstrap@spack platform=linux target=ppc64le %gcc ^python@3.8"
     },
     {
       "binaries": [
@@ -298,7 +298,7 @@
           "ef3f05d30333a39fd18714b87ee22605679f52fda97f5e592764d1591527bbf3"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=ppc64le ^python@3.9"
+      "spec": "clingo-bootstrap@spack platform=linux target=ppc64le %gcc ^python@3.9"
     },
     {
       "binaries": [
@@ -308,7 +308,7 @@
           "a4abec667660307ad5cff0a616d6651e187cc7b1386fd8cd4b6b288a01614076"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=x86_64 ^python@3.10"
+      "spec": "clingo-bootstrap@spack platform=linux target=x86_64 %gcc ^python@3.10"
     },
     {
       "binaries": [
@@ -318,7 +318,7 @@
           "a572ab6db954f4a850d1292bb1ef6d6055916784a894d149d657996fa98d0367"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=x86_64 ^python@3.11"
+      "spec": "clingo-bootstrap@spack platform=linux target=x86_64 %gcc ^python@3.11"
     },
     {
       "binaries": [
@@ -328,7 +328,7 @@
           "97f8ea17f3df3fb38904450114cbef9b4b0ea9c94da9de7a49b70b707012277a"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=x86_64 ^python@3.12"
+      "spec": "clingo-bootstrap@spack platform=linux target=x86_64 %gcc ^python@3.12"
     },
     {
       "binaries": [
@@ -338,7 +338,7 @@
           "6599ac06ade0cb3e80695f36492ea94a306f8bde0537482521510076c5981aa0"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=x86_64 ^python@3.13"
+      "spec": "clingo-bootstrap@spack platform=linux target=x86_64 %gcc ^python@3.13"
     },
     {
       "binaries": [
@@ -348,7 +348,7 @@
           "90b7cf4dd98e26c58578ad8604738cc32dfbb228cfb981bdfe103c99d0e7b5dd"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=x86_64 ^python@3.6"
+      "spec": "clingo-bootstrap@spack platform=linux target=x86_64 %gcc ^python@3.6"
     },
     {
       "binaries": [
@@ -358,7 +358,7 @@
           "dc5dbfd9c05b43c4992bf6666638ae96cee5548921e94eb793ba85727b25ec59"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=x86_64 ^python@3.7"
+      "spec": "clingo-bootstrap@spack platform=linux target=x86_64 %gcc ^python@3.7"
     },
     {
       "binaries": [
@@ -368,7 +368,7 @@
           "e8518de25baff7a74bdb42193e6e4b0496e7d0688434c42ce4bdc92fe4293a09"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=x86_64 ^python@3.8"
+      "spec": "clingo-bootstrap@spack platform=linux target=x86_64 %gcc ^python@3.8"
     },
     {
       "binaries": [
@@ -378,7 +378,7 @@
           "0c5831932608e7b4084fc6ce60e2b67b77dab76e5515303a049d4d30cd772321"
         ]
       ],
-      "spec": "clingo-bootstrap@spack%gcc platform=linux target=x86_64 ^python@3.9"
+      "spec": "clingo-bootstrap@spack platform=linux target=x86_64 %gcc ^python@3.9"
     }
   ]
 }

share/spack/bootstrap/github-actions-v0.6/gnupg.json

@@ -48,7 +48,7 @@
           "61bcb83dc3fc2ae06fde30b9f79c2596bd0457cf56b4d339c8c562a38ca1c31f"
         ]
       ],
-      "spec": "gnupg@2.4.5%apple-clang platform=darwin target=aarch64"
+      "spec": "gnupg@2.4.5 platform=darwin target=aarch64 %apple-clang"
     },
     {
       "binaries": [
@@ -98,7 +98,7 @@
           "3d36bce8bbd06134445aa3cefa00a80068317b6d082d2b43bb1e3be81ede5849"
         ]
       ],
-      "spec": "gnupg@2.4.5%apple-clang platform=darwin target=x86_64"
+      "spec": "gnupg@2.4.5 platform=darwin target=x86_64 %apple-clang"
     },
     {
       "binaries": [
@@ -153,7 +153,7 @@
           "8398592ab0812d8c76a21deca06da4277d05f4db784e129ead7535bb8988faa2"
         ]
       ],
-      "spec": "gnupg@2.4.5%gcc platform=linux target=aarch64"
+      "spec": "gnupg@2.4.5 platform=linux target=aarch64 %gcc"
     },
     {
       "binaries": [
@@ -208,7 +208,7 @@
           "cc7e4833af58913fa4ab2b7ce3fdb86d214594d54327c7e4eb4ca3f0784c046f"
         ]
       ],
-      "spec": "gnupg@2.4.5%gcc platform=linux target=ppc64le"
+      "spec": "gnupg@2.4.5 platform=linux target=ppc64le %gcc"
     },
     {
       "binaries": [
@@ -263,7 +263,7 @@
           "418b582f84547504b6464913fba5ba196482e86258081bdeb21af519fe8a2933"
         ]
       ],
-      "spec": "gnupg@2.4.5%gcc platform=linux target=x86_64"
+      "spec": "gnupg@2.4.5 platform=linux target=x86_64 %gcc"
     }
   ]
 }

share/spack/bootstrap/github-actions-v0.6/patchelf.json

@@ -13,7 +13,7 @@
           "820b8013b0b918ad85caa953740497e6c31c09d812bd34d087fc57128bfbdacb"
         ]
       ],
-      "spec": "patchelf@0.17.2%gcc platform=linux target=aarch64"
+      "spec": "patchelf@0.17.2 platform=linux target=aarch64 %gcc"
     },
     {
       "binaries": [
@@ -28,7 +28,7 @@
           "1569df037ea1ea316a50e89f5a0cafa0ce8e20629bbd07fcc3846d9fecd2451c"
         ]
       ],
-      "spec": "patchelf@0.17.2%gcc platform=linux target=ppc64le"
+      "spec": "patchelf@0.17.2 platform=linux target=ppc64le %gcc"
     },
     {
       "binaries": [
@@ -43,7 +43,7 @@
           "79dfb7064e7993a97474c5f6b7560254fe19465a6c4cfc44569852e5a6ab542b"
         ]
       ],
-      "spec": "patchelf@0.17.2%gcc platform=linux target=x86_64"
+      "spec": "patchelf@0.17.2 platform=linux target=x86_64 %gcc"
     }
   ]
 }

share/spack/gitlab/cloud_pipelines/configs/ci.yaml

@@ -20,7 +20,7 @@ ci:
         - k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust $k
         - k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust $k
       script::
-      - - if [ -n "$SPACK_EXTRA_MIRROR" ]; then spack mirror add local "$SPACK_EXTRA_MIRROR"; fi
+      - - if [ -n "$SPACK_EXTRA_MIRROR" ]; then spack mirror add local "${SPACK_EXTRA_MIRROR}/${SPACK_CI_STACK_NAME}"; fi
         - spack config blame mirrors
       - - spack --color=always --backtrace ci rebuild -j ${SPACK_BUILD_JOBS} --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
       after_script:

share/spack/gitlab/cloud_pipelines/configs/darwin/aarch64/config.yaml

@@ -1,5 +1,3 @@
 config:
-  build_stage:
-  - $spack/tmp/stage
   install_tree:
     root: $spack/opt/spack