Allow exago to use hiop@develop past v1.0.1 (#41384)

The patch for version 5.5.x still applies to 5.6.x.

.github/workflows/build-containers.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
 
-      - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934
+      - uses: docker/metadata-action@e6428a5c4e294a61438ed7f43155db912025b6b3
         id: docker_meta
         with:
           images: |

.github/workflows/style/requirements.txt

@@ -2,6 +2,6 @@ black==23.11.0
 clingo==5.6.2
 flake8==6.1.0
 isort==5.12.0
-mypy==1.6.1
+mypy==1.7.1
 types-six==1.16.21.9
-vermin==1.5.2
+vermin==1.6.0

.github/workflows/unit_tests.yaml

@@ -215,7 +215,7 @@ jobs:
         $(which spack) bootstrap disable spack-install
         $(which spack) solve zlib
         common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
-        $(which spack) unit-test --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
+        $(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
     - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
       with:
         flags: unittests,macos

CHANGELOG.md

@@ -1,3 +1,290 @@
+# v0.21.0 (2023-11-11)
+
+`v0.21.0` is a major feature release.
+
+## Features in this release
+
+1. **Better error messages with condition chaining**
+
+   In v0.18, we added better error messages that could tell you what problem happened,
+   but they couldn't tell you *why* it happened. `0.21` adds *condition chaining* to the
+   solver, and Spack can now trace back through the conditions that led to an error and
+   build a tree of causes potential causes and where they came from. For example:
+
+   ```console
+   $ spack solve hdf5 ^cmake@3.0.1
+   ==> Error: concretization failed for the following reasons:
+
+      1. Cannot satisfy 'cmake@3.0.1'
+      2. Cannot satisfy 'cmake@3.0.1'
+           required because hdf5 ^cmake@3.0.1 requested from CLI
+      3. Cannot satisfy 'cmake@3.18:' and 'cmake@3.0.1
+           required because hdf5 ^cmake@3.0.1 requested from CLI
+           required because hdf5 depends on cmake@3.18: when @1.13:
+             required because hdf5 ^cmake@3.0.1 requested from CLI
+      4. Cannot satisfy 'cmake@3.12:' and 'cmake@3.0.1
+           required because hdf5 depends on cmake@3.12:
+             required because hdf5 ^cmake@3.0.1 requested from CLI
+           required because hdf5 ^cmake@3.0.1 requested from CLI
+   ```
+
+   More details in #40173.
+
+2. **OCI build caches**
+
+   You can now use an arbitrary [OCI](https://opencontainers.org) registry as a build
+   cache:
+
+   ```console
+   $ spack mirror add my_registry oci://user/image # Dockerhub
+   $ spack mirror add my_registry oci://ghcr.io/haampie/spack-test # GHCR
+   $ spack mirror set --push --oci-username ... --oci-password ... my_registry  # set login creds
+   $ spack buildcache push my_registry [specs...]
+   ```
+
+   And you can optionally add a base image to get *runnable* images:
+
+   ```console
+   $ spack buildcache push --base-image ubuntu:23.04 my_registry python
+   Pushed ... as [image]:python-3.11.2-65txfcpqbmpawclvtasuog4yzmxwaoia.spack
+
+   $ docker run --rm -it [image]:python-3.11.2-65txfcpqbmpawclvtasuog4yzmxwaoia.spack
+   ```
+
+   This creates a container image from the Spack installations on the host system,
+   without the need to run `spack install` from a `Dockerfile` or `sif` file. It also
+   addresses the inconvenience of losing binaries of dependencies when `RUN spack
+   install` fails inside `docker build`.
+
+   Further, the container image layers and build cache tarballs are the same files. This
+   means that `spack install` and `docker pull` use the exact same underlying binaries.
+   If you previously used `spack install` inside of `docker build`, this feature helps
+   you save storage by a factor two.
+
+   More details in #38358.
+
+3. **Multiple versions of build dependencies**
+
+   Increasingly, complex package builds require multiple versions of some build
+   dependencies. For example, Python packages frequently require very specific versions
+   of `setuptools`, `cython`, and sometimes different physics packages require different
+   versions of Python to build. The concretizer enforced that every solve was *unified*,
+   i.e., that there only be one version of every package. The concretizer now supports
+   "duplicate" nodes for *build dependencies*, but enforces unification through
+   transitive link and run dependencies. This will allow it to better resolve complex
+   dependency graphs in ecosystems like Python, and it also gets us very close to
+   modeling compilers as proper dependencies.
+
+   This change required a major overhaul of the concretizer, as well as a number of
+   performance optimizations. See #38447, #39621.
+
+4. **Cherry-picking virtual dependencies**
+
+   You can now select only a subset of virtual dependencies from a spec that may provide
+   more. For example, if you want `mpich` to be your `mpi` provider, you can be explicit
+   by writing:
+
+   ```
+   hdf5 ^[virtuals=mpi] mpich
+   ```
+
+   Or, if you want to use, e.g., `intel-parallel-studio` for `blas` along with an external
+   `lapack` like `openblas`, you could write:
+
+   ```
+   strumpack ^[virtuals=mpi] intel-parallel-studio+mkl ^[virtuals=lapack] openblas
+   ```
+
+   The `virtuals=mpi` is an edge attribute, and dependency edges in Spack graphs now
+   track which virtuals they satisfied. More details in #17229 and #35322.
+
+   Note for packaging: in Spack 0.21 `spec.satisfies("^virtual")` is true if and only if
+   the package specifies `depends_on("virtual")`. This is different from Spack 0.20,
+   where depending on a provider implied depending on the virtual provided. See #41002
+   for an example where `^mkl` was being used to test for several `mkl` providers in a
+   package that did not depend on `mkl`.
+
+5. **License directive**
+
+   Spack packages can now have license metadata, with the new `license()` directive:
+
+   ```python
+       license("Apache-2.0")
+   ```
+
+   Licenses use [SPDX identifiers](https://spdx.org/licenses), and you can use SPDX
+   expressions to combine them:
+
+   ```python
+       license("Apache-2.0 OR MIT")
+   ```
+
+   Like other directives in Spack, it's conditional, so you can handle complex cases like
+   Spack itself:
+
+   ```python
+      license("LGPL-2.1", when="@:0.11")
+      license("Apache-2.0 OR MIT", when="@0.12:")
+   ```
+
+   More details in #39346, #40598.
+
+6. **`spack deconcretize` command**
+
+   We are getting close to having a `spack update` command for environments, but we're
+   not quite there yet. This is the next best thing. `spack deconcretize` gives you
+   control over what you want to update in an already concrete environment. If you have
+   an environment built with, say, `meson`, and you want to update your `meson` version,
+   you can run:
+
+   ```console
+   spack deconcretize meson
+   ```
+
+   and have everything that depends on `meson` rebuilt the next time you run `spack
+   concretize`. In a future Spack version, we'll handle all of this in a single command,
+   but for now you can use this to drop bits of your lockfile and resolve your
+   dependencies again. More in #38803.
+
+7. **UI Improvements**
+
+   The venerable `spack info` command was looking shabby compared to the rest of Spack's
+   UI, so we reworked it to have a bit more flair. `spack info` now makes much better
+   use of terminal space and shows variants, their values, and their descriptions much
+   more clearly. Conditional variants are grouped separately so you can more easily
+   understand how packages are structured. More in #40998.
+
+   `spack checksum` now allows you to filter versions from your editor, or by version
+   range. It also notifies you about potential download URL changes. See #40403.
+
+8. **Environments can include definitions**
+
+   Spack did not previously support using `include:` with The
+   [definitions](https://spack.readthedocs.io/en/latest/environments.html#spec-list-references)
+   section of an environment, but now it does. You can use this to curate lists of specs
+   and more easily reuse them across environments. See #33960.
+
+9. **Aliases**
+
+   You can now add aliases to Spack commands in `config.yaml`, e.g. this might enshrine
+   your favorite args to `spack find` as `spack f`:
+
+   ```yaml
+   config:
+     aliases:
+       f: find -lv
+   ```
+
+   See #17229.
+
+10. **Improved autoloading of modules**
+
+    Spack 0.20 was the first release to enable autoloading of direct dependencies in
+    module files.
+
+    The downside of this was that `module avail` and `module load` tab completion would
+    show users too many modules to choose from, and many users disabled generating
+    modules for dependencies through `exclude_implicits: true`. Further, it was
+    necessary to keep hashes in module names to avoid file name clashes.
+
+    In this release, you can start using `hide_implicits: true` instead, which exposes
+    only explicitly installed packages to the user, while still autoloading
+    dependencies. On top of that, you can safely use `hash_length: 0`, as this config
+    now only applies to the modules exposed to the user -- you don't have to worry about
+    file name clashes for hidden dependencies.
+
+   Note: for `tcl` this feature requires Modules 4.7 or higher
+
+11. **Updated container labeling**
+
+    Nightly Docker images from the `develop` branch will now be tagged as `:develop` and
+    `:nightly`. The `:latest` tag is no longer associated with `:develop`, but with the
+    latest stable release. Releases will be tagged with `:{major}`, `:{major}.{minor}`
+    and `:{major}.{minor}.{patch}`. `ubuntu:18.04` has also been removed from the list of
+    generated Docker images, as it is no longer supported. See #40593.
+
+## Other new commands and directives
+
+* `spack env activate` without arguments now loads a `default` environment that you do
+  not have to create (#40756).
+* `spack find -H` / `--hashes`: a new shortcut for piping `spack find` output to
+  other commands (#38663)
+* Add `spack checksum --verify`, fix `--add` (#38458)
+* New `default_args` context manager factors out common args for directives (#39964)
+* `spack compiler find --[no]-mixed-toolchain` lets you easily mix `clang` and
+  `gfortran` on Linux (#40902)
+
+## Performance improvements
+
+* `spack external find` execution is now much faster (#39843)
+* `spack location -i` now much faster on success (#40898)
+* Drop redundant rpaths post install (#38976)
+* ASP-based solver: avoid cycles in clingo using hidden directive (#40720)
+* Fix multiple quadratic complexity issues in environments (#38771)
+
+## Other new features of note
+
+* archspec: update to v0.2.2, support for Sapphire Rapids, Power10, Neoverse V2 (#40917)
+* Propagate variants across nodes that don't have that variant (#38512)
+* Implement fish completion (#29549)
+* Can now distinguish between source/binary mirror; don't ping mirror.spack.io as much (#34523)
+* Improve status reporting on install (add [n/total] display) (#37903)
+
+## Windows
+
+This release has the best Windows support of any Spack release yet, with numerous
+improvements and much larger swaths of tests passing:
+
+* MSVC and SDK improvements (#37711, #37930, #38500, #39823, #39180)
+* Windows external finding: update default paths; treat .bat as executable on Windows (#39850)
+* Windows decompression: fix removal of intermediate file (#38958)
+* Windows: executable/path handling (#37762)
+* Windows build systems: use ninja and enable tests (#33589)
+* Windows testing (#36970, #36972, #36973, #36840, #36977, #36792, #36834, #34696, #36971)
+* Windows PowerShell support (#39118, #37951)
+* Windows symlinking and libraries (#39933, #38599, #34701, #38578, #34701)
+
+## Notable refactors
+* User-specified flags take precedence over others in Spack compiler wrappers (#37376)
+* Improve setup of build, run, and test environments (#35737, #40916)
+* `make` is no longer a required system dependency of Spack (#40380)
+* Support Python 3.12 (#40404, #40155, #40153)
+* docs: Replace package list with packages.spack.io (#40251)
+* Drop Python 2 constructs in Spack (#38720, #38718, #38703)
+
+## Binary cache and stack updates
+* e4s arm stack: duplicate and target neoverse v1 (#40369)
+* Add macOS ML CI stacks (#36586)
+* E4S Cray CI Stack (#37837)
+* e4s cray: expand spec list (#38947)
+* e4s cray sles ci: expand spec list (#39081)
+
+## Removals, deprecations, and syntax changes
+* ASP: targets, compilers and providers soft-preferences are only global (#31261)
+* Parser: fix ambiguity with whitespace in version ranges (#40344)
+* Module file generation is disabled by default; you'll need to enable it to use it (#37258)
+* Remove deprecated "extra_instructions" option for containers (#40365)
+* Stand-alone test feature deprecation postponed to v0.22 (#40600)
+* buildcache push: make `--allow-root` the default and deprecate the option (#38878)
+
+## Notable Bugfixes
+* Bugfix: propagation of multivalued variants (#39833)
+* Allow `/` in git versions (#39398)
+* Fetch & patch: actually acquire stage lock, and many more issues (#38903)
+* Environment/depfile: better escaping of targets with Git versions (#37560)
+* Prevent "spack external find" to error out on wrong permissions (#38755)
+* lmod: allow core compiler to be specified with a version range (#37789)
+
+## Spack community stats
+
+* 7,469 total packages, 303 new since `v0.20.0`
+    * 150 new Python packages
+    * 34 new R packages
+* 353 people contributed to this release
+    * 336 committers to packages
+    * 65 committers to core
+
+
 # v0.20.3 (2023-10-31)
 
 ## Bugfixes

etc/spack/defaults/darwin/packages.yaml

@@ -50,4 +50,4 @@ packages:
     # Apple bundles libuuid in libsystem_c version 1353.100.2,
     # although the version number used here isn't critical
     - spec: apple-libuuid@1353.100.2
-      prefix: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk
+      prefix: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk

lib/spack/docs/binary_caches.rst

@@ -182,6 +182,7 @@ section of the configuration:
        padded_length: 128
 
 
+.. _binary_caches_oci:
 
 -----------------------------------------
 OCI / Docker V2 registries as build cache

lib/spack/docs/build_systems/cmakepackage.rst

@@ -82,7 +82,7 @@ class already contains:
 
 .. code-block:: python
 
-   depends_on('cmake', type='build')
+   depends_on("cmake", type="build")
 
 
 If you need to specify a particular version requirement, you can
@@ -90,7 +90,7 @@ override this in your package:
 
 .. code-block:: python
 
-   depends_on('cmake@2.8.12:', type='build')
+   depends_on("cmake@2.8.12:", type="build")
 
 
 ^^^^^^^^^^^^^^^^^^^
@@ -137,10 +137,10 @@ and without the :meth:`~spack.build_systems.cmake.CMakeBuilder.define` and
 
    def cmake_args(self):
        args = [
-           '-DWHATEVER:STRING=somevalue',
-           self.define('ENABLE_BROKEN_FEATURE', False),
-           self.define_from_variant('DETECT_HDF5', 'hdf5'),
-           self.define_from_variant('THREADS'), # True if +threads
+           "-DWHATEVER:STRING=somevalue",
+           self.define("ENABLE_BROKEN_FEATURE", False),
+           self.define_from_variant("DETECT_HDF5", "hdf5"),
+           self.define_from_variant("THREADS"), # True if +threads
        ]
 
        return args
@@ -151,10 +151,10 @@ and CMake simply ignores the empty command line argument. For example the follow
 
 .. code-block:: python
 
-   variant('example', default=True, when='@2.0:')
+   variant("example", default=True, when="@2.0:")
 
    def cmake_args(self):
-      return [self.define_from_variant('EXAMPLE', 'example')]
+      return [self.define_from_variant("EXAMPLE", "example")]
 
 will generate ``'cmake' '-DEXAMPLE=ON' ...`` when `@2.0: +example` is met, but will
 result in ``'cmake' '' ...`` when the spec version is below ``2.0``.
@@ -193,9 +193,9 @@ a variant to control this:
 
 .. code-block:: python
 
-   variant('build_type', default='RelWithDebInfo',
-           description='CMake build type',
-           values=('Debug', 'Release', 'RelWithDebInfo', 'MinSizeRel'))
+   variant("build_type", default="RelWithDebInfo",
+           description="CMake build type",
+           values=("Debug", "Release", "RelWithDebInfo", "MinSizeRel"))
 
 However, not every CMake package accepts all four of these options.
 Grep the ``CMakeLists.txt`` file to see if the default values are
@@ -205,9 +205,9 @@ package overrides the default variant with:
 
 .. code-block:: python
 
-   variant('build_type', default='DebugRelease',
-           description='The build type to build',
-           values=('Debug', 'Release', 'DebugRelease'))
+   variant("build_type", default="DebugRelease",
+           description="The build type to build",
+           values=("Debug", "Release", "DebugRelease"))
 
 For more information on ``CMAKE_BUILD_TYPE``, see:
 https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html
@@ -250,7 +250,7 @@ generator is Ninja. To switch to the Ninja generator, simply add:
 
 .. code-block:: python
 
-   generator = 'Ninja'
+   generator = "Ninja"
 
 
 ``CMakePackage`` defaults to "Unix Makefiles". If you switch to the
@@ -258,7 +258,7 @@ Ninja generator, make sure to add:
 
 .. code-block:: python
 
-   depends_on('ninja', type='build')
+   depends_on("ninja", type="build")
 
 to the package as well. Aside from that, you shouldn't need to do
 anything else. Spack will automatically detect that you are using
@@ -288,7 +288,7 @@ like so:
 
 .. code-block:: python
 
-   root_cmakelists_dir = 'src'
+   root_cmakelists_dir = "src"
 
 
 Note that this path is relative to the root of the extracted tarball,
@@ -304,7 +304,7 @@ different sub-directory, simply override ``build_directory`` like so:
 
 .. code-block:: python
 
-   build_directory = 'my-build'
+   build_directory = "my-build"
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 Build and install targets
@@ -324,8 +324,8 @@ library or build the documentation, you can add these like so:
 
 .. code-block:: python
 
-   build_targets = ['all', 'docs']
-   install_targets = ['install', 'docs']
+   build_targets = ["all", "docs"]
+   install_targets = ["install", "docs"]
 
 ^^^^^^^
 Testing

lib/spack/docs/build_systems/inteloneapipackage.rst

@@ -53,18 +53,24 @@ Install the oneAPI compilers::
 
 Add the compilers to your ``compilers.yaml`` so spack can use them::
 
-  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64
-  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/bin
 
 Verify that the compilers are available::
 
   spack compiler list
 
+Note that 2024 and later releases do not include ``icc``. Before 2024,
+the package layout was different::
+  
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin
+
 The ``intel-oneapi-compilers`` package includes 2 families of
 compilers:
 
 * ``intel``: ``icc``, ``icpc``, ``ifort``. Intel's *classic*
-  compilers.
+  compilers. 2024 and later releases contain ``ifort``, but not
+  ``icc`` and ``icpc``.
 * ``oneapi``: ``icx``, ``icpx``, ``ifx``. Intel's new generation of
   compilers based on LLVM.
 
@@ -89,8 +95,8 @@ Install the oneAPI compilers::
 
 Add the compilers to your ``compilers.yaml`` so Spack can use them::
 
-  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64
-  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/bin
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/bin
 
 Verify that the compilers are available::
 
@@ -146,8 +152,7 @@ Compilers
 To use the compilers, add some information about the installation to
 ``compilers.yaml``. For most users, it is sufficient to do::
 
-  spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin/intel64
-  spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin
+  spack compiler add /opt/intel/oneapi/compiler/latest/bin
 
 Adapt the paths above if you did not install the tools in the default
 location. After adding the compilers, using them is the same
@@ -156,6 +161,12 @@ Another option is to manually add the configuration to
 ``compilers.yaml`` as described in :ref:`Compiler configuration
 <compiler-config>`.
 
+Before 2024, the directory structure was different::
+  
+  spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin/intel64
+  spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin
+
+
 Libraries
 ---------
 

lib/spack/docs/build_systems/intelpackage.rst

@@ -934,9 +934,9 @@ a *virtual* ``mkl`` package is declared in Spack.
   .. code-block:: python
 
      # Examples for absolute and conditional dependencies:
-     depends_on('mkl')
-     depends_on('mkl', when='+mkl')
-     depends_on('mkl', when='fftw=mkl')
+     depends_on("mkl")
+     depends_on("mkl", when="+mkl")
+     depends_on("mkl", when="fftw=mkl")
 
   The ``MKLROOT`` environment variable (part of the documented API) will be set
   during all stages of client package installation, and is available to both
@@ -972,8 +972,8 @@ a *virtual* ``mkl`` package is declared in Spack.
       def configure_args(self):
           args = []
           ...
-          args.append('--with-blas=%s' % self.spec['blas'].libs.ld_flags)
-          args.append('--with-lapack=%s' % self.spec['lapack'].libs.ld_flags)
+          args.append("--with-blas=%s" % self.spec["blas"].libs.ld_flags)
+          args.append("--with-lapack=%s" % self.spec["lapack"].libs.ld_flags)
           ...
 
   .. tip::
@@ -989,13 +989,13 @@ a *virtual* ``mkl`` package is declared in Spack.
 
   .. code-block:: python
 
-    self.spec['blas'].headers.include_flags
+    self.spec["blas"].headers.include_flags
 
   and to generate linker options (``-L<dir> -llibname ...``), use the same as above,
 
   .. code-block:: python
 
-    self.spec['blas'].libs.ld_flags
+    self.spec["blas"].libs.ld_flags
 
   See
   :ref:`MakefilePackage <makefilepackage>`

lib/spack/docs/build_systems/luapackage.rst

@@ -88,7 +88,7 @@ override the ``luarocks_args`` method like so:
 .. code-block:: python
 
     def luarocks_args(self):
-        return ['flag1', 'flag2']
+        return ["flag1", "flag2"]
 
 One common use of this is to override warnings or flags for newer compilers, as in:
 

lib/spack/docs/build_systems/mavenpackage.rst

@@ -48,8 +48,8 @@ class automatically adds the following dependencies:
 
 .. code-block:: python
 
-   depends_on('java', type=('build', 'run'))
-   depends_on('maven', type='build')
+   depends_on("java", type=("build", "run"))
+   depends_on("maven", type="build")
 
 
 In the ``pom.xml`` file, you may see sections like:
@@ -72,8 +72,8 @@ should add:
 
 .. code-block:: python
 
-   depends_on('java@7:', type='build')
-   depends_on('maven@3.5.4:', type='build')
+   depends_on("java@7:", type="build")
+   depends_on("maven@3.5.4:", type="build")
 
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -88,9 +88,9 @@ the build phase. For example:
 
    def build_args(self):
        return [
-           '-Pdist,native',
-           '-Dtar',
-           '-Dmaven.javadoc.skip=true'
+           "-Pdist,native",
+           "-Dtar",
+           "-Dmaven.javadoc.skip=true"
        ]
 
 

lib/spack/docs/build_systems/mesonpackage.rst

@@ -86,8 +86,8 @@ the ``MesonPackage`` base class already contains:
 
 .. code-block:: python
 
-   depends_on('meson', type='build')
-   depends_on('ninja', type='build')
+   depends_on("meson", type="build")
+   depends_on("ninja", type="build")
 
 
 If you need to specify a particular version requirement, you can
@@ -95,8 +95,8 @@ override this in your package:
 
 .. code-block:: python
 
-   depends_on('meson@0.43.0:', type='build')
-   depends_on('ninja', type='build')
+   depends_on("meson@0.43.0:", type="build")
+   depends_on("ninja", type="build")
 
 
 ^^^^^^^^^^^^^^^^^^^
@@ -121,7 +121,7 @@ override the ``meson_args`` method like so:
 .. code-block:: python
 
    def meson_args(self):
-       return ['--warnlevel=3']
+       return ["--warnlevel=3"]
 
 
 This method can be used to pass flags as well as variables.

lib/spack/docs/build_systems/perlpackage.rst

@@ -118,7 +118,7 @@ so ``PerlPackage`` contains:
 
 .. code-block:: python
 
-   extends('perl')
+   extends("perl")
 
 
 If your package requires a specific version of Perl, you should
@@ -132,14 +132,14 @@ properly. If your package uses ``Makefile.PL`` to build, add:
 
 .. code-block:: python
 
-   depends_on('perl-extutils-makemaker', type='build')
+   depends_on("perl-extutils-makemaker", type="build")
 
 
 If your package uses ``Build.PL`` to build, add:
 
 .. code-block:: python
 
-   depends_on('perl-module-build', type='build')
+   depends_on("perl-module-build", type="build")
 
 
 ^^^^^^^^^^^^^^^^^
@@ -165,11 +165,11 @@ arguments to ``Makefile.PL`` or ``Build.PL`` by overriding
 .. code-block:: python
 
    def configure_args(self):
-       expat = self.spec['expat'].prefix
+       expat = self.spec["expat"].prefix
 
        return [
-           'EXPATLIBPATH={0}'.format(expat.lib),
-           'EXPATINCPATH={0}'.format(expat.include),
+           "EXPATLIBPATH={0}".format(expat.lib),
+           "EXPATINCPATH={0}".format(expat.include),
        ]
 
 

lib/spack/docs/build_systems/qmakepackage.rst

@@ -83,7 +83,7 @@ base class already contains:
 
 .. code-block:: python
 
-   depends_on('qt', type='build')
+   depends_on("qt", type="build")
 
 
 If you want to specify a particular version requirement, or need to
@@ -91,7 +91,7 @@ link to the ``qt`` libraries, you can override this in your package:
 
 .. code-block:: python
 
-   depends_on('qt@5.6.0:')
+   depends_on("qt@5.6.0:")
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 Passing arguments to qmake
@@ -103,7 +103,7 @@ override the ``qmake_args`` method like so:
 .. code-block:: python
 
    def qmake_args(self):
-       return ['-recursive']
+       return ["-recursive"]
 
 
 This method can be used to pass flags as well as variables.
@@ -118,7 +118,7 @@ sub-directory by adding the following to the package:
 
 .. code-block:: python
 
-   build_directory = 'src'
+   build_directory = "src"
 
 
 ^^^^^^^^^^^^^^^^^^^^^^

lib/spack/docs/build_systems/rpackage.rst

@@ -163,28 +163,28 @@ attributes that can be used to set ``homepage``, ``url``, ``list_url``, and
 
 .. code-block:: python
 
-   cran = 'caret'
+   cran = "caret"
 
 is equivalent to:
 
 .. code-block:: python
 
-   homepage = 'https://cloud.r-project.org/package=caret'
-   url      = 'https://cloud.r-project.org/src/contrib/caret_6.0-86.tar.gz'
-   list_url = 'https://cloud.r-project.org/src/contrib/Archive/caret'
+   homepage = "https://cloud.r-project.org/package=caret"
+   url      = "https://cloud.r-project.org/src/contrib/caret_6.0-86.tar.gz"
+   list_url = "https://cloud.r-project.org/src/contrib/Archive/caret"
 
 Likewise, the following ``bioc`` attribute:
 
 .. code-block:: python
 
-   bioc = 'BiocVersion'
+   bioc = "BiocVersion"
 
 is equivalent to:
 
 .. code-block:: python
 
-   homepage = 'https://bioconductor.org/packages/BiocVersion/'
-   git      = 'https://git.bioconductor.org/packages/BiocVersion'
+   homepage = "https://bioconductor.org/packages/BiocVersion/"
+   git      = "https://git.bioconductor.org/packages/BiocVersion"
 
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -200,7 +200,7 @@ base class contains:
 
 .. code-block:: python
 
-   extends('r')
+   extends("r")
 
 
 Take a close look at the homepage for ``caret``. If you look at the
@@ -209,7 +209,7 @@ You should add this to your package like so:
 
 .. code-block:: python
 
-   depends_on('r@3.2.0:', type=('build', 'run'))
+   depends_on("r@3.2.0:", type=("build", "run"))
 
 
 ^^^^^^^^^^^^^^
@@ -227,7 +227,7 @@ and list all of their dependencies in the following sections:
 * LinkingTo
 
 As far as Spack is concerned, all 3 of these dependency types
-correspond to ``type=('build', 'run')``, so you don't have to worry
+correspond to ``type=("build", "run")``, so you don't have to worry
 about the details. If you are curious what they mean,
 https://github.com/spack/spack/issues/2951 has a pretty good summary:
 
@@ -330,7 +330,7 @@ the dependency:
 
 .. code-block:: python
 
-   depends_on('r-lattice@0.20:', type=('build', 'run'))
+   depends_on("r-lattice@0.20:", type=("build", "run"))
 
 
 ^^^^^^^^^^^^^^^^^^
@@ -361,20 +361,20 @@ like so:
 .. code-block:: python
 
    def configure_args(self):
-       mpi_name = self.spec['mpi'].name
+       mpi_name = self.spec["mpi"].name
 
        # The type of MPI. Supported values are:
        # OPENMPI, LAM, MPICH, MPICH2, or CRAY
-       if mpi_name == 'openmpi':
-           Rmpi_type = 'OPENMPI'
-       elif mpi_name == 'mpich':
-           Rmpi_type = 'MPICH2'
+       if mpi_name == "openmpi":
+           Rmpi_type = "OPENMPI"
+       elif mpi_name == "mpich":
+           Rmpi_type = "MPICH2"
        else:
-           raise InstallError('Unsupported MPI type')
+           raise InstallError("Unsupported MPI type")
 
        return [
-           '--with-Rmpi-type={0}'.format(Rmpi_type),
-           '--with-mpi={0}'.format(spec['mpi'].prefix),
+           "--with-Rmpi-type={0}".format(Rmpi_type),
+           "--with-mpi={0}".format(spec["mpi"].prefix),
        ]
 
 

lib/spack/docs/build_systems/rubypackage.rst

@@ -84,8 +84,8 @@ The ``*.gemspec`` file may contain something like:
 
 .. code-block:: ruby
 
-   summary = 'An implementation of the AsciiDoc text processor and publishing toolchain'
-   description = 'A fast, open source text processor and publishing toolchain for converting AsciiDoc content to HTML 5, DocBook 5, and other formats.'
+   summary = "An implementation of the AsciiDoc text processor and publishing toolchain"
+   description = "A fast, open source text processor and publishing toolchain for converting AsciiDoc content to HTML 5, DocBook 5, and other formats."
 
 
 Either of these can be used for the description of the Spack package.
@@ -98,7 +98,7 @@ The ``*.gemspec`` file may contain something like:
 
 .. code-block:: ruby
 
-   homepage = 'https://asciidoctor.org'
+   homepage = "https://asciidoctor.org"
 
 
 This should be used as the official homepage of the Spack package.
@@ -112,21 +112,21 @@ the base class contains:
 
 .. code-block:: python
 
-   extends('ruby')
+   extends("ruby")
 
 
 The ``*.gemspec`` file may contain something like:
 
 .. code-block:: ruby
 
-   required_ruby_version = '>= 2.3.0'
+   required_ruby_version = ">= 2.3.0"
 
 
 This can be added to the Spack package using:
 
 .. code-block:: python
 
-   depends_on('ruby@2.3.0:', type=('build', 'run'))
+   depends_on("ruby@2.3.0:", type=("build", "run"))
 
 
 ^^^^^^^^^^^^^^^^^

lib/spack/docs/build_systems/sippackage.rst

@@ -124,7 +124,7 @@ are wrong, you can provide the names yourself by overriding
 
 .. code-block:: python
 
-   import_modules = ['PyQt5']
+   import_modules = ["PyQt5"]
 
 
 These tests often catch missing dependencies and non-RPATHed

lib/spack/docs/build_systems/wafpackage.rst

@@ -63,8 +63,8 @@ run package-specific unit tests.
 .. code-block:: python
 
    def installtest(self):
-       with working_dir('test'):
-           pytest = which('py.test')
+       with working_dir("test"):
+           pytest = which("py.test")
            pytest()
 
 
@@ -93,7 +93,7 @@ the following dependency automatically:
 
 .. code-block:: python
 
-   depends_on('python@2.5:', type='build')
+   depends_on("python@2.5:", type="build")
 
 
 Waf only supports Python 2.5 and up.
@@ -113,7 +113,7 @@ phase, you can use:
        args = []
 
        if self.run_tests:
-           args.append('--test')
+           args.append("--test")
 
        return args
 

lib/spack/docs/containers.rst

@@ -24,6 +24,16 @@ image, or to set up a proper entrypoint to run the image. These tasks are
 usually both necessary and repetitive, so Spack comes with a command
 to generate recipes for container images starting from a ``spack.yaml``.
 
+.. seealso::
+
+  This page is a reference for generating recipes to build container images.
+  It means that your environment is built from scratch inside the container
+  runtime.
+
+  Since v0.21, Spack can also create container images from existing package installations
+  on your host system. See :ref:`binary_caches_oci` for more information on
+  that topic.
+
 --------------------
 A Quick Introduction
 --------------------

lib/spack/docs/packages_yaml.rst

@@ -383,7 +383,33 @@ like this:
 
 which means every spec will be required to use ``clang`` as a compiler.
 
-Note that in this case ``all`` represents a *default set of requirements* -
+Requirements on variants for all packages are possible too, but note that they
+are only enforced for those packages that define these variants, otherwise they
+are disregarded. For example:
+
+.. code-block:: yaml
+
+   packages:
+     all:
+       require:
+       - "+shared"
+       - "+cuda"
+
+will just enforce ``+shared`` on ``zlib``, which has a boolean ``shared`` variant but
+no ``cuda`` variant.
+
+Constraints in a single spec literal are always considered as a whole, so in a case like:
+
+.. code-block:: yaml
+
+   packages:
+     all:
+       require: "+shared +cuda"
+
+the default requirement will be enforced only if a package has both a ``cuda`` and
+a ``shared`` variant, and will never be partially enforced.
+
+Finally, ``all`` represents a *default set of requirements* -
 if there are specific package requirements, then the default requirements
 under ``all`` are disregarded. For example, with a configuration like this:
 
@@ -391,12 +417,18 @@ under ``all`` are disregarded. For example, with a configuration like this:
 
    packages:
      all:
-       require: '%clang'
+       require:
+       - 'build_type=Debug'
+       - '%clang'
      cmake:
-       require: '%gcc'
+       require:
+       - 'build_type=Debug'
+       - '%gcc'
 
 Spack requires ``cmake`` to use ``gcc`` and all other nodes (including ``cmake``
-dependencies) to use ``clang``.
+dependencies) to use ``clang``. If enforcing ``build_type=Debug`` is needed also
+on ``cmake``, it must be repeated in the specific ``cmake`` requirements.
+
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Setting requirements on virtual specs

lib/spack/docs/requirements.txt

@@ -1,13 +1,13 @@
 sphinx==7.2.6
 sphinxcontrib-programoutput==0.17
 sphinx_design==0.5.0
-sphinx-rtd-theme==1.3.0
+sphinx-rtd-theme==2.0.0
 python-levenshtein==0.23.0
-docutils==0.18.1
-pygments==2.16.1
+docutils==0.20.1
+pygments==2.17.2
 urllib3==2.1.0
 pytest==7.4.3
 isort==5.12.0
 black==23.11.0
 flake8==6.1.0
-mypy==1.7.0
+mypy==1.7.1

lib/spack/spack/audit.py

@@ -286,6 +286,54 @@ def make_error(attribute_name, config_data, summary):
     return errors
 
 
+@config_packages
+def _avoid_mismatched_variants(error_cls):
+    """Warns if variant preferences have mismatched types or names."""
+    errors = []
+    packages_yaml = spack.config.CONFIG.get_config("packages")
+
+    def make_error(config_data, summary):
+        s = io.StringIO()
+        s.write("Occurring in the following file:\n")
+        syaml.dump_config(config_data, stream=s, blame=True)
+        return error_cls(summary=summary, details=[s.getvalue()])
+
+    for pkg_name in packages_yaml:
+        # 'all:' must be more forgiving, since it is setting defaults for everything
+        if pkg_name == "all" or "variants" not in packages_yaml[pkg_name]:
+            continue
+
+        preferences = packages_yaml[pkg_name]["variants"]
+        if not isinstance(preferences, list):
+            preferences = [preferences]
+
+        for variants in preferences:
+            current_spec = spack.spec.Spec(variants)
+            pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
+            for variant in current_spec.variants.values():
+                # Variant does not exist at all
+                if variant.name not in pkg_cls.variants:
+                    summary = (
+                        f"Setting a preference for the '{pkg_name}' package to the "
+                        f"non-existing variant '{variant.name}'"
+                    )
+                    errors.append(make_error(preferences, summary))
+                    continue
+
+                # Variant cannot accept this value
+                s = spack.spec.Spec(pkg_name)
+                try:
+                    s.update_variant_validate(variant.name, variant.value)
+                except Exception:
+                    summary = (
+                        f"Setting the variant '{variant.name}' of the '{pkg_name}' package "
+                        f"to the invalid value '{str(variant)}'"
+                    )
+                    errors.append(make_error(preferences, summary))
+
+    return errors
+
+
 #: Sanity checks on package directives
 package_directives = AuditClass(
     group="packages",

lib/spack/spack/binary_distribution.py

@@ -230,7 +230,11 @@ def _associate_built_specs_with_mirror(self, cache_key, mirror_url):
                 )
                 return
 
-            spec_list = db.query_local(installed=False, in_buildcache=True)
+            spec_list = [
+                s
+                for s in db.query_local(installed=any, in_buildcache=any)
+                if s.external or db.query_local_by_spec_hash(s.dag_hash()).in_buildcache
+            ]
 
             for indexed_spec in spec_list:
                 dag_hash = indexed_spec.dag_hash()

lib/spack/spack/bootstrap/_common.py

@@ -213,7 +213,8 @@ def _root_spec(spec_str: str) -> str:
     if str(spack.platforms.host()) == "darwin":
         spec_str += " %apple-clang"
     elif str(spack.platforms.host()) == "windows":
-        spec_str += " %msvc"
+        # TODO (johnwparent): Remove version constraint when clingo patch is up
+        spec_str += " %msvc@:19.37"
     else:
         spec_str += " %gcc"
 

lib/spack/spack/bootstrap/environment.py

@@ -161,7 +161,7 @@ def _write_spack_yaml_file(self) -> None:
 
 def isort_root_spec() -> str:
     """Return the root spec used to bootstrap isort"""
-    return _root_spec("py-isort@4.3.5:")
+    return _root_spec("py-isort@5")
 
 
 def mypy_root_spec() -> str:

lib/spack/spack/build_environment.py

@@ -324,19 +324,29 @@ def set_compiler_environment_variables(pkg, env):
     # ttyout, ttyerr, etc.
     link_dir = spack.paths.build_env_path
 
-    # Set SPACK compiler variables so that our wrapper knows what to call
+    # Set SPACK compiler variables so that our wrapper knows what to
+    # call.  If there is no compiler configured then use a default
+    # wrapper which will emit an error if it is used.
     if compiler.cc:
         env.set("SPACK_CC", compiler.cc)
         env.set("CC", os.path.join(link_dir, compiler.link_paths["cc"]))
+    else:
+        env.set("CC", os.path.join(link_dir, "cc"))
     if compiler.cxx:
         env.set("SPACK_CXX", compiler.cxx)
         env.set("CXX", os.path.join(link_dir, compiler.link_paths["cxx"]))
+    else:
+        env.set("CC", os.path.join(link_dir, "c++"))
     if compiler.f77:
         env.set("SPACK_F77", compiler.f77)
         env.set("F77", os.path.join(link_dir, compiler.link_paths["f77"]))
+    else:
+        env.set("F77", os.path.join(link_dir, "f77"))
     if compiler.fc:
         env.set("SPACK_FC", compiler.fc)
         env.set("FC", os.path.join(link_dir, compiler.link_paths["fc"]))
+    else:
+        env.set("FC", os.path.join(link_dir, "fc"))
 
     # Set SPACK compiler rpath flags so that our wrapper knows what to use
     env.set("SPACK_CC_RPATH_ARG", compiler.cc_rpath_arg)
@@ -743,15 +753,16 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD):
         set_compiler_environment_variables(pkg, env_mods)
         set_wrapper_variables(pkg, env_mods)
 
-    tty.debug("setup_package: grabbing modifications from dependencies")
-    env_mods.extend(setup_context.get_env_modifications())
-    tty.debug("setup_package: collected all modifications from dependencies")
-
-    # architecture specific setup
+    # Platform specific setup goes before package specific setup. This is for setting
+    # defaults like MACOSX_DEPLOYMENT_TARGET on macOS.
     platform = spack.platforms.by_name(pkg.spec.architecture.platform)
     target = platform.target(pkg.spec.architecture.target)
     platform.setup_platform_environment(pkg, env_mods)
 
+    tty.debug("setup_package: grabbing modifications from dependencies")
+    env_mods.extend(setup_context.get_env_modifications())
+    tty.debug("setup_package: collected all modifications from dependencies")
+
     if context == Context.TEST:
         env_mods.prepend_path("PATH", ".")
     elif context == Context.BUILD and not dirty and not env_mods.is_unset("CPATH"):

lib/spack/spack/build_systems/cargo.py

@@ -0,0 +1,89 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import inspect
+
+import llnl.util.filesystem as fs
+
+import spack.builder
+import spack.package_base
+from spack.directives import build_system, depends_on
+from spack.multimethod import when
+
+from ._checks import BaseBuilder, execute_install_time_tests
+
+
+class CargoPackage(spack.package_base.PackageBase):
+    """Specialized class for packages built using a Makefiles."""
+
+    #: This attribute is used in UI queries that need to know the build
+    #: system base class
+    build_system_class = "CargoPackage"
+
+    build_system("cargo")
+
+    with when("build_system=cargo"):
+        depends_on("rust", type="build")
+
+
+@spack.builder.builder("cargo")
+class CargoBuilder(BaseBuilder):
+    """The Cargo builder encodes the most common way of building software with
+    a rust Cargo.toml file. It has two phases that can be overridden, if need be:
+
+            1. :py:meth:`~.CargoBuilder.build`
+            2. :py:meth:`~.CargoBuilder.install`
+
+    For a finer tuning you may override:
+
+        +-----------------------------------------------+----------------------+
+        | **Method**                                    | **Purpose**          |
+        +===============================================+======================+
+        | :py:meth:`~.CargoBuilder.build_args`          | Specify arguments    |
+        |                                               | to ``cargo install`` |
+        +-----------------------------------------------+----------------------+
+        | :py:meth:`~.CargoBuilder.check_args`          | Specify arguments    |
+        |                                               | to ``cargo test``    |
+        +-----------------------------------------------+----------------------+
+    """
+
+    phases = ("build", "install")
+
+    #: Callback names for install-time test
+    install_time_test_callbacks = ["check"]
+
+    @property
+    def build_directory(self):
+        """Return the directory containing the main Cargo.toml."""
+        return self.pkg.stage.source_path
+
+    @property
+    def build_args(self):
+        """Arguments for ``cargo build``."""
+        return []
+
+    @property
+    def check_args(self):
+        """Argument for ``cargo test`` during check phase"""
+        return []
+
+    def build(self, pkg, spec, prefix):
+        """Runs ``cargo install`` in the source directory"""
+        with fs.working_dir(self.build_directory):
+            inspect.getmodule(pkg).cargo(
+                "install", "--root", "out", "--path", ".", *self.build_args
+            )
+
+    def install(self, pkg, spec, prefix):
+        """Copy build files into package prefix."""
+        with fs.working_dir(self.build_directory):
+            fs.install_tree("out", prefix)
+
+    spack.builder.run_after("install")(execute_install_time_tests)
+
+    def check(self):
+        """Run "cargo test"."""
+        with fs.working_dir(self.build_directory):
+            inspect.getmodule(self.pkg).cargo("test", *self.check_args)

lib/spack/spack/build_systems/cuda.py

@@ -136,12 +136,12 @@ def cuda_flags(arch_list):
         conflicts("%gcc@11:", when="+cuda ^cuda@:11.4.0")
         conflicts("%gcc@11.2:", when="+cuda ^cuda@:11.5")
         conflicts("%gcc@12:", when="+cuda ^cuda@:11.8")
-        conflicts("%gcc@13:", when="+cuda ^cuda@:12.1")
+        conflicts("%gcc@13:", when="+cuda ^cuda@:12.3")
         conflicts("%clang@12:", when="+cuda ^cuda@:11.4.0")
         conflicts("%clang@13:", when="+cuda ^cuda@:11.5")
         conflicts("%clang@14:", when="+cuda ^cuda@:11.7")
         conflicts("%clang@15:", when="+cuda ^cuda@:12.0")
-        conflicts("%clang@16:", when="+cuda ^cuda@:12.1")
+        conflicts("%clang@16:", when="+cuda ^cuda@:12.3")
 
         # https://gist.github.com/ax3l/9489132#gistcomment-3860114
         conflicts("%gcc@10", when="+cuda ^cuda@:11.4.0")

lib/spack/spack/build_systems/go.py

@@ -0,0 +1,98 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import inspect
+
+import llnl.util.filesystem as fs
+
+import spack.builder
+import spack.package_base
+from spack.directives import build_system, extends
+from spack.multimethod import when
+
+from ._checks import BaseBuilder, execute_install_time_tests
+
+
+class GoPackage(spack.package_base.PackageBase):
+    """Specialized class for packages built using the Go toolchain."""
+
+    #: This attribute is used in UI queries that need to know the build
+    #: system base class
+    build_system_class = "GoPackage"
+
+    #: Legacy buildsystem attribute used to deserialize and install old specs
+    legacy_buildsystem = "go"
+
+    build_system("go")
+
+    with when("build_system=go"):
+        # TODO: this seems like it should be depends_on, see
+        # setup_dependent_build_environment in go for why I kept it like this
+        extends("go@1.14:", type="build")
+
+
+@spack.builder.builder("go")
+class GoBuilder(BaseBuilder):
+    """The Go builder encodes the most common way of building software with
+    a golang go.mod file. It has two phases that can be overridden, if need be:
+
+            1. :py:meth:`~.GoBuilder.build`
+            2. :py:meth:`~.GoBuilder.install`
+
+    For a finer tuning you may override:
+
+        +-----------------------------------------------+--------------------+
+        | **Method**                                    | **Purpose**        |
+        +===============================================+====================+
+        | :py:meth:`~.GoBuilder.build_args`             | Specify arguments  |
+        |                                               | to ``go build``    |
+        +-----------------------------------------------+--------------------+
+        | :py:meth:`~.GoBuilder.check_args`             | Specify arguments  |
+        |                                               | to ``go test``     |
+        +-----------------------------------------------+--------------------+
+    """
+
+    phases = ("build", "install")
+
+    #: Callback names for install-time test
+    install_time_test_callbacks = ["check"]
+
+    def setup_build_environment(self, env):
+        env.set("GO111MODULE", "on")
+        env.set("GOTOOLCHAIN", "local")
+
+    @property
+    def build_directory(self):
+        """Return the directory containing the main go.mod."""
+        return self.pkg.stage.source_path
+
+    @property
+    def build_args(self):
+        """Arguments for ``go build``."""
+        # Pass ldflags -s = --strip-all and -w = --no-warnings by default
+        return ["-ldflags", "-s -w", "-o", f"{self.pkg.name}"]
+
+    @property
+    def check_args(self):
+        """Argument for ``go test`` during check phase"""
+        return []
+
+    def build(self, pkg, spec, prefix):
+        """Runs ``go build`` in the source directory"""
+        with fs.working_dir(self.build_directory):
+            inspect.getmodule(pkg).go("build", *self.build_args)
+
+    def install(self, pkg, spec, prefix):
+        """Install built binaries into prefix bin."""
+        with fs.working_dir(self.build_directory):
+            fs.mkdirp(prefix.bin)
+            fs.install(pkg.name, prefix.bin)
+
+    spack.builder.run_after("install")(execute_install_time_tests)
+
+    def check(self):
+        """Run ``go test .`` in the source directory"""
+        with fs.working_dir(self.build_directory):
+            inspect.getmodule(self.pkg).go("test", *self.check_args)

lib/spack/spack/build_systems/oneapi.py

@@ -7,9 +7,9 @@
 import os
 import platform
 import shutil
-from os.path import basename, dirname, isdir
+from os.path import basename, isdir
 
-from llnl.util.filesystem import find_headers, find_libraries, join_path, mkdirp
+from llnl.util.filesystem import HeaderList, find_libraries, join_path, mkdirp
 from llnl.util.link_tree import LinkTree
 
 from spack.directives import conflicts, variant
@@ -55,10 +55,21 @@ def component_dir(self):
         """Subdirectory for this component in the install prefix."""
         raise NotImplementedError
 
+    @property
+    def v2_layout_versions(self):
+        """Version that implements the v2 directory layout."""
+        raise NotImplementedError
+
+    @property
+    def v2_layout(self):
+        """Returns true if this version implements the v2 directory layout."""
+        return self.spec.satisfies(self.v2_layout_versions)
+
     @property
     def component_prefix(self):
         """Path to component <prefix>/<component>/<version>."""
-        return self.prefix.join(join_path(self.component_dir, self.spec.version))
+        v = self.spec.version.up_to(2) if self.v2_layout else self.spec.version
+        return self.prefix.join(self.component_dir).join(str(v))
 
     @property
     def env_script_args(self):
@@ -112,8 +123,9 @@ def install_component(self, installer_path):
                 shutil.rmtree("/var/intel/installercache", ignore_errors=True)
 
         # Some installers have a bug and do not return an error code when failing
-        if not isdir(join_path(self.prefix, self.component_dir)):
-            raise RuntimeError("install failed")
+        install_dir = self.component_prefix
+        if not isdir(install_dir):
+            raise RuntimeError("install failed to directory: {0}".format(install_dir))
 
     def setup_run_environment(self, env):
         """Adds environment variables to the generated module file.
@@ -128,7 +140,7 @@ def setup_run_environment(self, env):
         if "~envmods" not in self.spec:
             env.extend(
                 EnvironmentModifications.from_sourcing_file(
-                    join_path(self.component_prefix, "env", "vars.sh"), *self.env_script_args
+                    self.component_prefix.env.join("vars.sh"), *self.env_script_args
                 )
             )
 
@@ -167,16 +179,21 @@ class IntelOneApiLibraryPackage(IntelOneApiPackage):
 
     """
 
+    def header_directories(self, dirs):
+        h = HeaderList([])
+        h.directories = dirs
+        return h
+
     @property
     def headers(self):
-        include_path = join_path(self.component_prefix, "include")
-        return find_headers("*", include_path, recursive=True)
+        return self.header_directories(
+            [self.component_prefix.include, self.component_prefix.include.join(self.component_dir)]
+        )
 
     @property
     def libs(self):
-        lib_path = join_path(self.component_prefix, "lib", "intel64")
-        lib_path = lib_path if isdir(lib_path) else dirname(lib_path)
-        return find_libraries("*", root=lib_path, shared=True, recursive=True)
+        # for v2_layout all libraries are in the top level, v1 sometimes put them in intel64
+        return find_libraries("*", root=self.component_prefix.lib, recursive=not self.v2_layout)
 
 
 class IntelOneApiLibraryPackageWithSdk(IntelOneApiPackage):
@@ -189,23 +206,13 @@ class IntelOneApiLibraryPackageWithSdk(IntelOneApiPackage):
 
     """
 
-    @property
-    def include(self):
-        return join_path(self.component_prefix, "sdk", "include")
-
     @property
     def headers(self):
-        return find_headers("*", self.include, recursive=True)
-
-    @property
-    def lib(self):
-        lib_path = join_path(self.component_prefix, "sdk", "lib64")
-        lib_path = lib_path if isdir(lib_path) else dirname(lib_path)
-        return lib_path
+        return self.header_directories([self.component_prefix.sdk.include])
 
     @property
     def libs(self):
-        return find_libraries("*", root=self.lib, shared=True, recursive=True)
+        return find_libraries("*", self.component_prefix.sdk.lib64)
 
 
 class IntelOneApiStaticLibraryList:

lib/spack/spack/cmd/add.py

@@ -6,7 +6,7 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
+from spack.cmd.common import arguments
 
 description = "add a spec to an environment"
 section = "environments"

lib/spack/spack/cmd/bootstrap.py

@@ -15,13 +15,13 @@
 import spack.bootstrap
 import spack.bootstrap.config
 import spack.bootstrap.core
-import spack.cmd.common.arguments
 import spack.config
 import spack.main
 import spack.mirror
 import spack.spec
 import spack.stage
 import spack.util.path
+from spack.cmd.common import arguments
 
 description = "manage bootstrap configuration"
 section = "system"
@@ -68,12 +68,8 @@
 
 
 def _add_scope_option(parser):
-    scopes = spack.config.scopes()
     parser.add_argument(
-        "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        help="configuration scope to read/modify",
+        "--scope", action=arguments.ConfigScope, help="configuration scope to read/modify"
     )
 
 
@@ -106,7 +102,7 @@ def setup_parser(subparser):
     disable.add_argument("name", help="name of the source to be disabled", nargs="?", default=None)
 
     reset = sp.add_parser("reset", help="reset bootstrapping configuration to Spack defaults")
-    spack.cmd.common.arguments.add_common_arguments(reset, ["yes_to_all"])
+    arguments.add_common_arguments(reset, ["yes_to_all"])
 
     root = sp.add_parser("root", help="get/set the root bootstrap directory")
     _add_scope_option(root)

lib/spack/spack/cmd/buildcache.py

@@ -21,7 +21,6 @@
 
 import spack.binary_distribution as bindist
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
 import spack.error
@@ -40,6 +39,7 @@
 import spack.util.web as web_util
 from spack.build_environment import determine_number_of_jobs
 from spack.cmd import display_specs
+from spack.cmd.common import arguments
 from spack.oci.image import (
     Digest,
     ImageReference,
@@ -182,13 +182,10 @@ def setup_parser(subparser: argparse.ArgumentParser):
     )
 
     # used to construct scope arguments below
-    scopes = spack.config.scopes()
-
     check.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope containing mirrors to check",
     )
     check_spec_or_specfile = check.add_mutually_exclusive_group(required=True)

lib/spack/spack/cmd/change.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
+from spack.cmd.common import arguments
 
 description = "change an existing spec in an environment"
 section = "environments"

lib/spack/spack/cmd/clean.py

@@ -12,13 +12,13 @@
 
 import spack.bootstrap
 import spack.caches
-import spack.cmd.common.arguments as arguments
 import spack.cmd.test
 import spack.config
 import spack.repo
 import spack.stage
 import spack.store
 import spack.util.path
+from spack.cmd.common import arguments
 from spack.paths import lib_path, var_path
 
 description = "remove temporary build files and/or downloaded archives"

lib/spack/spack/cmd/common/arguments.py

@@ -124,6 +124,33 @@ def __call__(self, parser, namespace, values, option_string=None):
         setattr(namespace, self.dest, deptype)
 
 
+class ConfigScope(argparse.Action):
+    """Pick the currently configured config scopes."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        kwargs.setdefault("metavar", spack.config.SCOPES_METAVAR)
+        super().__init__(*args, **kwargs)
+
+    @property
+    def default(self):
+        return self._default() if callable(self._default) else self._default
+
+    @default.setter
+    def default(self, value):
+        self._default = value
+
+    @property
+    def choices(self):
+        return spack.config.scopes().keys()
+
+    @choices.setter
+    def choices(self, value):
+        pass
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, values)
+
+
 def _cdash_reporter(namespace):
     """Helper function to create a CDash reporter. This function gets an early reference to the
     argparse namespace under construction, so it can later use it to create the object.

lib/spack/spack/cmd/common/env_utility.py

@@ -8,13 +8,13 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.deptypes as dt
 import spack.error
 import spack.paths
 import spack.spec
 import spack.store
 from spack import build_environment, traverse
+from spack.cmd.common import arguments
 from spack.context import Context
 from spack.util.environment import dump_environment, pickle_environment
 

lib/spack/spack/cmd/compiler.py

@@ -14,6 +14,7 @@
 import spack.compilers
 import spack.config
 import spack.spec
+from spack.cmd.common import arguments
 
 description = "manage compilers"
 section = "system"
@@ -23,8 +24,6 @@
 def setup_parser(subparser):
     sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="compiler_command")
 
-    scopes = spack.config.scopes()
-
     # Find
     find_parser = sp.add_parser(
         "find",
@@ -47,9 +46,8 @@ def setup_parser(subparser):
     find_parser.add_argument("add_paths", nargs=argparse.REMAINDER)
     find_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope("compilers"),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope("compilers"),
         help="configuration scope to modify",
     )
 
@@ -60,20 +58,15 @@ def setup_parser(subparser):
     )
     remove_parser.add_argument("compiler_spec")
     remove_parser.add_argument(
-        "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=None,
-        help="configuration scope to modify",
+        "--scope", action=arguments.ConfigScope, default=None, help="configuration scope to modify"
     )
 
     # List
     list_parser = sp.add_parser("list", help="list available compilers")
     list_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_list_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_list_scope(),
         help="configuration scope to read from",
     )
 
@@ -82,9 +75,8 @@ def setup_parser(subparser):
     info_parser.add_argument("compiler_spec")
     info_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_list_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_list_scope(),
         help="configuration scope to read from",
     )
 

lib/spack/spack/cmd/compilers.py

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import spack.config
+from spack.cmd.common import arguments
 from spack.cmd.compiler import compiler_list
 
 description = "list available compilers"
@@ -12,13 +12,8 @@
 
 
 def setup_parser(subparser):
-    scopes = spack.config.scopes()
-
     subparser.add_argument(
-        "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        help="configuration scope to read/modify",
+        "--scope", action=arguments.ConfigScope, help="configuration scope to read/modify"
     )
 
 

lib/spack/spack/cmd/config.py

@@ -10,7 +10,6 @@
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 
-import spack.cmd.common.arguments
 import spack.config
 import spack.environment as ev
 import spack.repo
@@ -18,6 +17,7 @@
 import spack.schema.packages
 import spack.store
 import spack.util.spack_yaml as syaml
+from spack.cmd.common import arguments
 from spack.util.editor import editor
 
 description = "get and set configuration options"
@@ -26,14 +26,9 @@
 
 
 def setup_parser(subparser):
-    scopes = spack.config.scopes()
-
     # User can only choose one
     subparser.add_argument(
-        "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        help="configuration scope to read/modify",
+        "--scope", action=arguments.ConfigScope, help="configuration scope to read/modify"
     )
 
     sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="config_command")
@@ -101,13 +96,13 @@ def setup_parser(subparser):
     setup_parser.add_parser = add_parser
 
     update = sp.add_parser("update", help="update configuration files to the latest format")
-    spack.cmd.common.arguments.add_common_arguments(update, ["yes_to_all"])
+    arguments.add_common_arguments(update, ["yes_to_all"])
     update.add_argument("section", help="section to update")
 
     revert = sp.add_parser(
         "revert", help="revert configuration files to their state before update"
     )
-    spack.cmd.common.arguments.add_common_arguments(revert, ["yes_to_all"])
+    arguments.add_common_arguments(revert, ["yes_to_all"])
     revert.add_argument("section", help="section to update")
 
 

lib/spack/spack/cmd/create.py

@@ -172,6 +172,14 @@ def configure_args(self):
         return args"""
 
 
+class CargoPackageTemplate(PackageTemplate):
+    """Provides appropriate overrides for cargo-based packages"""
+
+    base_class_name = "CargoPackage"
+
+    body_def = ""
+
+
 class CMakePackageTemplate(PackageTemplate):
     """Provides appropriate overrides for CMake-based packages"""
 
@@ -186,6 +194,14 @@ def cmake_args(self):
         return args"""
 
 
+class GoPackageTemplate(PackageTemplate):
+    """Provides appropriate overrides for Go-module-based packages"""
+
+    base_class_name = "GoPackage"
+
+    body_def = ""
+
+
 class LuaPackageTemplate(PackageTemplate):
     """Provides appropriate overrides for LuaRocks-based packages"""
 
@@ -575,28 +591,30 @@ def __init__(self, name, *args, **kwargs):
 
 
 templates = {
-    "autotools": AutotoolsPackageTemplate,
     "autoreconf": AutoreconfPackageTemplate,
-    "cmake": CMakePackageTemplate,
-    "bundle": BundlePackageTemplate,
-    "qmake": QMakePackageTemplate,
-    "maven": MavenPackageTemplate,
-    "scons": SconsPackageTemplate,
-    "waf": WafPackageTemplate,
+    "autotools": AutotoolsPackageTemplate,
     "bazel": BazelPackageTemplate,
+    "bundle": BundlePackageTemplate,
+    "cargo": CargoPackageTemplate,
+    "cmake": CMakePackageTemplate,
+    "generic": PackageTemplate,
+    "go": GoPackageTemplate,
+    "intel": IntelPackageTemplate,
+    "lua": LuaPackageTemplate,
+    "makefile": MakefilePackageTemplate,
+    "maven": MavenPackageTemplate,
+    "meson": MesonPackageTemplate,
+    "octave": OctavePackageTemplate,
+    "perlbuild": PerlbuildPackageTemplate,
+    "perlmake": PerlmakePackageTemplate,
     "python": PythonPackageTemplate,
+    "qmake": QMakePackageTemplate,
     "r": RPackageTemplate,
     "racket": RacketPackageTemplate,
-    "perlmake": PerlmakePackageTemplate,
-    "perlbuild": PerlbuildPackageTemplate,
-    "octave": OctavePackageTemplate,
     "ruby": RubyPackageTemplate,
-    "makefile": MakefilePackageTemplate,
-    "intel": IntelPackageTemplate,
-    "meson": MesonPackageTemplate,
-    "lua": LuaPackageTemplate,
+    "scons": SconsPackageTemplate,
     "sip": SIPPackageTemplate,
-    "generic": PackageTemplate,
+    "waf": WafPackageTemplate,
 }
 
 
@@ -679,6 +697,8 @@ def __call__(self, stage, url):
         clues = [
             (r"/CMakeLists\.txt$", "cmake"),
             (r"/NAMESPACE$", "r"),
+            (r"/Cargo\.toml$", "cargo"),
+            (r"/go\.mod$", "go"),
             (r"/configure$", "autotools"),
             (r"/configure\.(in|ac)$", "autoreconf"),
             (r"/Makefile\.am$", "autoreconf"),

lib/spack/spack/cmd/deconcretize.py

@@ -10,10 +10,10 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.cmd.common.confirmation as confirmation
 import spack.environment as ev
 import spack.spec
+from spack.cmd.common import arguments
 
 description = "remove specs from the concretized lockfile of an environment"
 section = "environments"

lib/spack/spack/cmd/dependencies.py

@@ -9,11 +9,11 @@
 from llnl.util.tty.colify import colify
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.package_base
 import spack.repo
 import spack.store
+from spack.cmd.common import arguments
 
 description = "show dependencies of a package"
 section = "basic"

lib/spack/spack/cmd/dependents.py

@@ -9,10 +9,10 @@
 from llnl.util.tty.colify import colify
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.repo
 import spack.store
+from spack.cmd.common import arguments
 
 description = "show packages that depend on another"
 section = "basic"

lib/spack/spack/cmd/deprecate.py

@@ -20,9 +20,9 @@
 from llnl.util.symlink import symlink
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.store
+from spack.cmd.common import arguments
 from spack.database import InstallStatuses
 from spack.error import SpackError
 

lib/spack/spack/cmd/dev_build.py

@@ -9,9 +9,9 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.repo
+from spack.cmd.common import arguments
 
 description = "developer build: build from code in current working directory"
 section = "build"

lib/spack/spack/cmd/develop.py

@@ -8,10 +8,10 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.spec
 import spack.util.path
 import spack.version
+from spack.cmd.common import arguments
 from spack.error import SpackError
 
 description = "add a spec to an environment's dev-build information"

lib/spack/spack/cmd/diff.py

@@ -10,11 +10,11 @@
 from llnl.util.tty.color import cprint, get_color_when
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.solver.asp as asp
 import spack.util.environment
 import spack.util.spack_json as sjson
+from spack.cmd.common import arguments
 
 description = "compare two specs"
 section = "basic"

lib/spack/spack/cmd/env.py

@@ -20,7 +20,6 @@
 import spack.cmd
 import spack.cmd.common
 import spack.cmd.common.arguments
-import spack.cmd.common.arguments as arguments
 import spack.cmd.install
 import spack.cmd.modules
 import spack.cmd.uninstall
@@ -31,6 +30,7 @@
 import spack.schema.env
 import spack.spec
 import spack.tengine
+from spack.cmd.common import arguments
 from spack.util.environment import EnvironmentModifications
 
 description = "manage virtual environments"

lib/spack/spack/cmd/extensions.py

@@ -10,10 +10,10 @@
 from llnl.util.tty.colify import colify
 
 import spack.cmd as cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.repo
 import spack.store
+from spack.cmd.common import arguments
 
 description = "list extensions for package"
 section = "extensions"

lib/spack/spack/cmd/external.py

@@ -14,12 +14,12 @@
 
 import spack
 import spack.cmd
-import spack.cmd.common.arguments
 import spack.config
 import spack.cray_manifest as cray_manifest
 import spack.detection
 import spack.error
 import spack.util.environment
+from spack.cmd.common import arguments
 
 description = "manage external packages in Spack configuration"
 section = "config"
@@ -29,8 +29,6 @@
 def setup_parser(subparser):
     sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="external_command")
 
-    scopes = spack.config.scopes()
-
     find_parser = sp.add_parser("find", help="add external packages to packages.yaml")
     find_parser.add_argument(
         "--not-buildable",
@@ -48,15 +46,14 @@ def setup_parser(subparser):
     )
     find_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope("packages"),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope("packages"),
         help="configuration scope to modify",
     )
     find_parser.add_argument(
         "--all", action="store_true", help="search for all packages that Spack knows about"
     )
-    spack.cmd.common.arguments.add_common_arguments(find_parser, ["tags", "jobs"])
+    arguments.add_common_arguments(find_parser, ["tags", "jobs"])
     find_parser.add_argument("packages", nargs=argparse.REMAINDER)
     find_parser.epilog = (
         'The search is by default on packages tagged with the "build-tools" or '

lib/spack/spack/cmd/fetch.py

@@ -6,11 +6,11 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
 import spack.repo
 import spack.traverse
+from spack.cmd.common import arguments
 
 description = "fetch archives for packages"
 section = "build"

lib/spack/spack/cmd/find.py

@@ -12,9 +12,9 @@
 
 import spack.bootstrap
 import spack.cmd as cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.repo
+from spack.cmd.common import arguments
 from spack.database import InstallStatuses
 
 description = "list and search installed packages"

lib/spack/spack/cmd/gpg.py

@@ -7,11 +7,11 @@
 import os
 
 import spack.binary_distribution
-import spack.cmd.common.arguments as arguments
 import spack.mirror
 import spack.paths
 import spack.util.gpg
 import spack.util.url
+from spack.cmd.common import arguments
 
 description = "handle GPG actions for spack"
 section = "packaging"

lib/spack/spack/cmd/graph.py

@@ -5,10 +5,10 @@
 from llnl.util import tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
 import spack.store
+from spack.cmd.common import arguments
 from spack.graph import DAGWithDependencyTypes, SimpleDAG, graph_ascii, graph_dot, static_graph_dot
 
 description = "generate graphs of package dependency relationships"
@@ -61,7 +61,7 @@ def graph(parser, args):
         args.dot = True
         env = ev.active_environment()
         if env:
-            specs = env.all_specs()
+            specs = env.concrete_roots()
         else:
             specs = spack.store.STORE.db.query()
 

lib/spack/spack/cmd/info.py

@@ -11,13 +11,13 @@
 import llnl.util.tty.color as color
 from llnl.util.tty.colify import colify
 
-import spack.cmd.common.arguments as arguments
 import spack.deptypes as dt
 import spack.fetch_strategy as fs
 import spack.install_test
 import spack.repo
 import spack.spec
 import spack.version
+from spack.cmd.common import arguments
 from spack.package_base import preferred_version
 
 description = "get detailed information on a particular package"

lib/spack/spack/cmd/install.py

@@ -14,7 +14,6 @@
 
 import spack.build_environment
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
 import spack.fetch_strategy
@@ -23,6 +22,7 @@
 import spack.report
 import spack.spec
 import spack.store
+from spack.cmd.common import arguments
 from spack.error import SpackError
 from spack.installer import PackageInstaller
 

lib/spack/spack/cmd/list.py

@@ -15,9 +15,9 @@
 import llnl.util.tty as tty
 from llnl.util.tty.colify import colify
 
-import spack.cmd.common.arguments as arguments
 import spack.deptypes as dt
 import spack.repo
+from spack.cmd.common import arguments
 from spack.version import VersionList
 
 description = "list and search available packages"

lib/spack/spack/cmd/load.py

@@ -8,12 +8,12 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.cmd.find
 import spack.environment as ev
 import spack.store
 import spack.user_environment as uenv
 import spack.util.environment
+from spack.cmd.common import arguments
 
 description = "add package to the user environment"
 section = "user environment"

lib/spack/spack/cmd/location.py

@@ -9,11 +9,11 @@
 
 import spack.builder
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.paths
 import spack.repo
 import spack.stage
+from spack.cmd.common import arguments
 
 description = "print out locations of packages and spack directories"
 section = "basic"

lib/spack/spack/cmd/mark.py

@@ -8,11 +8,11 @@
 from llnl.util import tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.error
 import spack.package_base
 import spack.repo
 import spack.store
+from spack.cmd.common import arguments
 from spack.database import InstallStatuses
 
 description = "mark packages as explicitly or implicitly installed"

lib/spack/spack/cmd/mirror.py

@@ -11,7 +11,6 @@
 
 import spack.caches
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.concretize
 import spack.config
 import spack.environment as ev
@@ -20,6 +19,7 @@
 import spack.spec
 import spack.util.path
 import spack.util.web as web_util
+from spack.cmd.common import arguments
 from spack.error import SpackError
 
 description = "manage mirrors (source and binary)"
@@ -88,18 +88,14 @@ def setup_parser(subparser):
         "--mirror-url", metavar="mirror_url", type=str, help="find mirror to destroy by url"
     )
 
-    # used to construct scope arguments below
-    scopes = spack.config.scopes()
-
     # Add
     add_parser = sp.add_parser("add", help=mirror_add.__doc__)
     add_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror")
     add_parser.add_argument("url", help="url of mirror directory from 'spack mirror create'")
     add_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope to modify",
     )
     add_parser.add_argument(
@@ -117,9 +113,8 @@ def setup_parser(subparser):
     remove_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror")
     remove_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope to modify",
     )
 
@@ -136,9 +131,8 @@ def setup_parser(subparser):
     )
     set_url_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope to modify",
     )
     arguments.add_connection_args(set_url_parser, False)
@@ -165,9 +159,8 @@ def setup_parser(subparser):
     set_parser.add_argument("--url", help="url of mirror directory from 'spack mirror create'")
     set_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope to modify",
     )
     arguments.add_connection_args(set_parser, False)
@@ -176,9 +169,8 @@ def setup_parser(subparser):
     list_parser = sp.add_parser("list", help=mirror_list.__doc__)
     list_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_list_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_list_scope(),
         help="configuration scope to read from",
     )
 

lib/spack/spack/cmd/modules/__init__.py

@@ -14,11 +14,11 @@
 from llnl.util.tty import color
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.modules
 import spack.modules.common
 import spack.repo
+from spack.cmd.common import arguments
 
 description = "manipulate module files"
 section = "environment"

lib/spack/spack/cmd/patch.py

@@ -6,12 +6,12 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
 import spack.package_base
 import spack.repo
 import spack.traverse
+from spack.cmd.common import arguments
 
 description = "patch expanded archive sources in preparation for install"
 section = "build"

lib/spack/spack/cmd/pkg.py

@@ -12,11 +12,11 @@
 from llnl.util.tty.colify import colify
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.paths
 import spack.repo
 import spack.util.executable as exe
 import spack.util.package_hash as ph
+from spack.cmd.common import arguments
 
 description = "query packages associated with particular git revisions"
 section = "developer"

lib/spack/spack/cmd/remove.py

@@ -6,7 +6,7 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
+from spack.cmd.common import arguments
 
 description = "remove specs from an environment"
 section = "environments"

lib/spack/spack/cmd/repo.py

@@ -11,6 +11,7 @@
 import spack.config
 import spack.repo
 import spack.util.path
+from spack.cmd.common import arguments
 
 description = "manage package source repositories"
 section = "config"
@@ -19,7 +20,6 @@
 
 def setup_parser(subparser):
     sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="repo_command")
-    scopes = spack.config.scopes()
 
     # Create
     create_parser = sp.add_parser("create", help=repo_create.__doc__)
@@ -43,9 +43,8 @@ def setup_parser(subparser):
     list_parser = sp.add_parser("list", help=repo_list.__doc__)
     list_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_list_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_list_scope(),
         help="configuration scope to read from",
     )
 
@@ -54,9 +53,8 @@ def setup_parser(subparser):
     add_parser.add_argument("path", help="path to a Spack package repository directory")
     add_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope to modify",
     )
 
@@ -67,9 +65,8 @@ def setup_parser(subparser):
     )
     remove_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope to modify",
     )
 

lib/spack/spack/cmd/restage.py

@@ -6,8 +6,8 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.repo
+from spack.cmd.common import arguments
 
 description = "revert checked out package source code"
 section = "build"

lib/spack/spack/cmd/solve.py

@@ -12,12 +12,12 @@
 
 import spack
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment
 import spack.hash_types as ht
 import spack.package_base
 import spack.solver.asp as asp
+from spack.cmd.common import arguments
 
 description = "concretize a specs using an ASP solver"
 section = "developer"

lib/spack/spack/cmd/spec.py

@@ -10,11 +10,11 @@
 
 import spack
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.hash_types as ht
 import spack.spec
 import spack.store
+from spack.cmd.common import arguments
 
 description = "show what would be installed, given a spec"
 section = "build"

lib/spack/spack/cmd/stage.py

@@ -8,13 +8,13 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
 import spack.package_base
 import spack.repo
 import spack.stage
 import spack.traverse
+from spack.cmd.common import arguments
 
 description = "expand downloaded archive in preparation for install"
 section = "build"

lib/spack/spack/cmd/test.py

@@ -15,12 +15,12 @@
 from llnl.util.tty import colify
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.install_test
 import spack.package_base
 import spack.repo
 import spack.report
+from spack.cmd.common import arguments
 
 description = "run spack's tests for an install"
 section = "admin"

lib/spack/spack/cmd/tutorial.py

@@ -10,11 +10,11 @@
 from llnl.util.filesystem import working_dir
 
 import spack
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.paths
 import spack.util.git
 import spack.util.gpg
+from spack.cmd.common import arguments
 from spack.util.spack_yaml import syaml_dict
 
 description = "set up spack for our tutorial (WARNING: modifies config!)"

lib/spack/spack/cmd/undevelop.py

@@ -6,7 +6,7 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
+from spack.cmd.common import arguments
 
 description = "remove specs from an environment"
 section = "environments"

lib/spack/spack/cmd/uninstall.py

@@ -10,13 +10,13 @@
 from llnl.util.tty.colify import colify
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.cmd.common.confirmation as confirmation
 import spack.environment as ev
 import spack.package_base
 import spack.spec
 import spack.store
 import spack.traverse as traverse
+from spack.cmd.common import arguments
 from spack.database import InstallStatuses
 
 description = "remove installed packages"

lib/spack/spack/cmd/unload.py

@@ -7,10 +7,10 @@
 import sys
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.error
 import spack.user_environment as uenv
 import spack.util.environment
+from spack.cmd.common import arguments
 
 description = "remove package from the user environment"
 section = "user environment"

lib/spack/spack/cmd/versions.py

@@ -8,9 +8,9 @@
 import llnl.util.tty as tty
 from llnl.util.tty.colify import colify
 
-import spack.cmd.common.arguments as arguments
 import spack.repo
 import spack.spec
+from spack.cmd.common import arguments
 from spack.version import infinity_versions, ver
 
 description = "list available versions of a package"

lib/spack/spack/compilers/__init__.py

@@ -154,6 +154,14 @@ def add_compilers_to_config(compilers, scope=None, init_config=True):
     """
     compiler_config = get_compiler_config(scope, init_config)
     for compiler in compilers:
+        if not compiler.cc:
+            tty.debug(f"{compiler.spec} does not have a C compiler")
+        if not compiler.cxx:
+            tty.debug(f"{compiler.spec} does not have a C++ compiler")
+        if not compiler.f77:
+            tty.debug(f"{compiler.spec} does not have a Fortran77 compiler")
+        if not compiler.fc:
+            tty.debug(f"{compiler.spec} does not have a Fortran compiler")
         compiler_config.append(_to_dict(compiler))
     spack.config.set("compilers", compiler_config, scope=scope)
 

lib/spack/spack/compilers/intel.py

@@ -85,6 +85,14 @@ def cxx14_flag(self):
         else:
             return "-std=c++14"
 
+    @property
+    def cxx17_flag(self):
+        # https://www.intel.com/content/www/us/en/developer/articles/news/c17-features-supported-by-c-compiler.html
+        if self.real_version < Version("19"):
+            raise UnsupportedCompilerFlag(self, "the C++17 standard", "cxx17_flag", "< 19")
+        else:
+            return "-std=c++17"
+
     @property
     def c99_flag(self):
         if self.real_version < Version("12"):

lib/spack/spack/compilers/oneapi.py

@@ -6,6 +6,8 @@
 import os
 from os.path import dirname
 
+from llnl.util import tty
+
 from spack.compiler import Compiler
 
 
@@ -135,3 +137,13 @@ def setup_custom_environment(self, pkg, env):
         #   Executable "sycl-post-link" doesn't exist!
         if self.cxx:
             env.prepend_path("PATH", dirname(self.cxx))
+
+        # 2024 release bumped the libsycl version because of an ABI
+        # change, 2024 compilers are required.  You will see this
+        # error:
+        #
+        # /usr/bin/ld: warning: libsycl.so.7, needed by ...., not found
+        if pkg.spec.satisfies("%oneapi@:2023"):
+            for c in ["dnn"]:
+                if pkg.spec.satisfies(f"^intel-oneapi-{c}@2024:"):
+                    tty.warn(f"intel-oneapi-{c}@2024 SYCL APIs requires %oneapi@2024:")

lib/spack/spack/main.py

@@ -1016,14 +1016,16 @@ def _main(argv=None):
         bootstrap_context = bootstrap.ensure_bootstrap_configuration()
 
     with bootstrap_context:
-        return finish_parse_and_run(parser, cmd_name, args.command, env_format_error)
+        return finish_parse_and_run(parser, cmd_name, args, env_format_error)
 
 
-def finish_parse_and_run(parser, cmd_name, cmd, env_format_error):
+def finish_parse_and_run(parser, cmd_name, main_args, env_format_error):
     """Finish parsing after we know the command to run."""
     # add the found command to the parser and re-run then re-parse
     command = parser.add_command(cmd_name)
-    args, unknown = parser.parse_known_args()
+    args, unknown = parser.parse_known_args(main_args.command)
+    # we need to inherit verbose since the install command checks for it
+    args.verbose = main_args.verbose
 
     # Now that we know what command this is and what its args are, determine
     # whether we can continue with a bad environment and raise if not.

lib/spack/spack/package.py

@@ -37,10 +37,12 @@
     cmake_cache_path,
     cmake_cache_string,
 )
+from spack.build_systems.cargo import CargoPackage
 from spack.build_systems.cmake import CMakePackage, generator
 from spack.build_systems.cuda import CudaPackage
 from spack.build_systems.generic import Package
 from spack.build_systems.gnu import GNUMirrorPackage
+from spack.build_systems.go import GoPackage
 from spack.build_systems.intel import IntelPackage
 from spack.build_systems.lua import LuaPackage
 from spack.build_systems.makefile import MakefilePackage

lib/spack/spack/parser.py

@@ -206,11 +206,15 @@ def tokenize(text: str) -> Iterator[Token]:
     scanner = ALL_TOKENS.scanner(text)  # type: ignore[attr-defined]
     match: Optional[Match] = None
     for match in iter(scanner.match, None):
+        # The following two assertions are to help mypy
+        msg = (
+            "unexpected value encountered during parsing. Please submit a bug report "
+            "at https://github.com/spack/spack/issues/new/choose"
+        )
+        assert match is not None, msg
+        assert match.lastgroup is not None, msg
         yield Token(
-            TokenType.__members__[match.lastgroup],  # type: ignore[attr-defined]
-            match.group(),  # type: ignore[attr-defined]
-            match.start(),  # type: ignore[attr-defined]
-            match.end(),  # type: ignore[attr-defined]
+            TokenType.__members__[match.lastgroup], match.group(), match.start(), match.end()
         )
 
     if match is None and not text:

lib/spack/spack/solver/asp.py

@@ -1013,14 +1013,6 @@ def on_model(model):
             # record the possible dependencies in the solve
             result.possible_dependencies = setup.pkgs
 
-            # print any unknown functions in the model
-            for sym in best_model:
-                if sym.name not in ("attr", "error", "opt_criterion"):
-                    tty.debug(
-                        "UNKNOWN SYMBOL: %s(%s)"
-                        % (sym.name, ", ".join([str(s) for s in intermediate_repr(sym.arguments)]))
-                    )
-
         elif cores:
             result.control = self.control
             result.cores.extend(cores)
@@ -1125,11 +1117,8 @@ def __init__(self, tests=False):
 
         self.reusable_and_possible = ConcreteSpecsByHash()
 
-        # id for dummy variables
-        self._condition_id_counter = itertools.count()
-        self._trigger_id_counter = itertools.count()
+        self._id_counter = itertools.count()
         self._trigger_cache = collections.defaultdict(dict)
-        self._effect_id_counter = itertools.count()
         self._effect_cache = collections.defaultdict(dict)
 
         # Caches to optimize the setup phase of the solver
@@ -1299,52 +1288,70 @@ def requirement_rules_from_packages_yaml(self, pkg):
             kind = RequirementKind.DEFAULT
         return self._rules_from_requirements(pkg_name, requirements, kind=kind)
 
-    def _rules_from_requirements(self, pkg_name: str, requirements, *, kind: RequirementKind):
+    def _rules_from_requirements(
+        self, pkg_name: str, requirements, *, kind: RequirementKind
+    ) -> List[RequirementRule]:
         """Manipulate requirements from packages.yaml, and return a list of tuples
         with a uniform structure (name, policy, requirements).
         """
         if isinstance(requirements, str):
-            rules = [self._rule_from_str(pkg_name, requirements, kind)]
-        else:
-            rules = []
-            for requirement in requirements:
-                if isinstance(requirement, str):
-                    # A string represents a spec that must be satisfied. It is
-                    # equivalent to a one_of group with a single element
-                    rules.append(self._rule_from_str(pkg_name, requirement, kind))
-                else:
-                    for policy in ("spec", "one_of", "any_of"):
-                        if policy in requirement:
-                            constraints = requirement[policy]
+            requirements = [requirements]
 
-                            # "spec" is for specifying a single spec
-                            if policy == "spec":
-                                constraints = [constraints]
-                                policy = "one_of"
+        rules = []
+        for requirement in requirements:
+            # A string is equivalent to a one_of group with a single element
+            if isinstance(requirement, str):
+                requirement = {"one_of": [requirement]}
 
-                            rules.append(
-                                RequirementRule(
-                                    pkg_name=pkg_name,
-                                    policy=policy,
-                                    requirements=constraints,
-                                    kind=kind,
-                                    message=requirement.get("message"),
-                                    condition=requirement.get("when"),
-                                )
-                            )
+            for policy in ("spec", "one_of", "any_of"):
+                if policy not in requirement:
+                    continue
+
+                constraints = requirement[policy]
+                # "spec" is for specifying a single spec
+                if policy == "spec":
+                    constraints = [constraints]
+                    policy = "one_of"
+
+                constraints = [
+                    x
+                    for x in constraints
+                    if not self.reject_requirement_constraint(pkg_name, constraint=x, kind=kind)
+                ]
+                if not constraints:
+                    continue
+
+                rules.append(
+                    RequirementRule(
+                        pkg_name=pkg_name,
+                        policy=policy,
+                        requirements=constraints,
+                        kind=kind,
+                        message=requirement.get("message"),
+                        condition=requirement.get("when"),
+                    )
+                )
         return rules
 
-    def _rule_from_str(
-        self, pkg_name: str, requirements: str, kind: RequirementKind
-    ) -> RequirementRule:
-        return RequirementRule(
-            pkg_name=pkg_name,
-            policy="one_of",
-            requirements=[requirements],
-            kind=kind,
-            condition=None,
-            message=None,
-        )
+    def reject_requirement_constraint(
+        self, pkg_name: str, *, constraint: str, kind: RequirementKind
+    ) -> bool:
+        """Returns True if a requirement constraint should be rejected"""
+        if kind == RequirementKind.DEFAULT:
+            # Requirements under all: are applied only if they are satisfiable considering only
+            # package rules, so e.g. variants must exist etc. Otherwise, they are rejected.
+            try:
+                s = spack.spec.Spec(pkg_name)
+                s.constrain(constraint)
+                s.validate_or_raise()
+            except spack.error.SpackError as e:
+                tty.debug(
+                    f"[SETUP] Rejecting the default '{constraint}' requirement "
+                    f"on '{pkg_name}': {str(e)}",
+                    level=2,
+                )
+                return True
+        return False
 
     def pkg_rules(self, pkg, tests):
         pkg = packagize(pkg)
@@ -1525,7 +1532,7 @@ def condition(
         # In this way, if a condition can't be emitted but the exception is handled in the caller,
         # we won't emit partial facts.
 
-        condition_id = next(self._condition_id_counter)
+        condition_id = next(self._id_counter)
         self.gen.fact(fn.pkg_fact(named_cond.name, fn.condition(condition_id)))
         self.gen.fact(fn.condition_reason(condition_id, msg))
 
@@ -1533,7 +1540,7 @@ def condition(
 
         named_cond_key = (str(named_cond), transform_required)
         if named_cond_key not in cache:
-            trigger_id = next(self._trigger_id_counter)
+            trigger_id = next(self._id_counter)
             requirements = self.spec_clauses(named_cond, body=True, required_from=name)
 
             if transform_required:
@@ -1549,7 +1556,7 @@ def condition(
         cache = self._effect_cache[named_cond.name]
         imposed_spec_key = (str(imposed_spec), transform_imposed)
         if imposed_spec_key not in cache:
-            effect_id = next(self._effect_id_counter)
+            effect_id = next(self._id_counter)
             requirements = self.spec_clauses(imposed_spec, body=False, required_from=name)
 
             if transform_imposed:
@@ -1839,7 +1846,13 @@ def preferred_variants(self, pkg_name):
 
             # perform validation of the variant and values
             spec = spack.spec.Spec(pkg_name)
-            spec.update_variant_validate(variant_name, values)
+            try:
+                spec.update_variant_validate(variant_name, values)
+            except (spack.variant.InvalidVariantValueError, KeyError, ValueError) as e:
+                tty.debug(
+                    f"[SETUP]: rejected {str(variant)} as a preference for {pkg_name}: {str(e)}"
+                )
+                continue
 
             for value in values:
                 self.variant_values_from_specs.add((pkg_name, variant.name, value))
@@ -2557,12 +2570,8 @@ def setup(
             reuse: list of concrete specs that can be reused
             allow_deprecated: if True adds deprecated versions into the solve
         """
-        self._condition_id_counter = itertools.count()
-
-        # preliminary checks
         check_packages_exist(specs)
 
-        # get list of all possible dependencies
         self.possible_virtuals = set(x.name for x in specs if x.virtual)
 
         node_counter = _create_counter(specs, tests=self.tests)
@@ -2682,23 +2691,21 @@ def setup(
     def literal_specs(self, specs):
         for spec in specs:
             self.gen.h2("Spec: %s" % str(spec))
-            condition_id = next(self._condition_id_counter)
-            trigger_id = next(self._trigger_id_counter)
+            condition_id = next(self._id_counter)
+            trigger_id = next(self._id_counter)
 
             # Special condition triggered by "literal_solved"
             self.gen.fact(fn.literal(trigger_id))
             self.gen.fact(fn.pkg_fact(spec.name, fn.condition_trigger(condition_id, trigger_id)))
             self.gen.fact(fn.condition_reason(condition_id, f"{spec} requested from CLI"))
 
-            # Effect imposes the spec
             imposed_spec_key = str(spec), None
             cache = self._effect_cache[spec.name]
-            msg = (
-                "literal specs have different requirements. clear cache before computing literals"
-            )
-            assert imposed_spec_key not in cache, msg
-            effect_id = next(self._effect_id_counter)
-            requirements = self.spec_clauses(spec)
+            if imposed_spec_key in cache:
+                effect_id, requirements = cache[imposed_spec_key]
+            else:
+                effect_id = next(self._id_counter)
+                requirements = self.spec_clauses(spec)
             root_name = spec.name
             for clause in requirements:
                 clause_name = clause.args[0]
@@ -2799,9 +2806,11 @@ class SpecBuilder:
                 r"^.*_propagate$",
                 r"^.*_satisfies$",
                 r"^.*_set$",
+                r"^dependency_holds$",
                 r"^node_compiler$",
                 r"^package_hash$",
                 r"^root$",
+                r"^track_dependencies$",
                 r"^variant_default_value_from_cli$",
                 r"^virtual_node$",
                 r"^virtual_root$",
@@ -3125,6 +3134,25 @@ def _develop_specs_from_env(spec, env):
     spec.constrain(dev_info["spec"])
 
 
+def _is_reusable_external(packages, spec: spack.spec.Spec) -> bool:
+    """Returns true iff spec is an external that can be reused.
+
+    Arguments:
+        packages: the packages configuration
+        spec: the spec to check
+    """
+    for name in {spec.name, *(p.name for p in spec.package.provided)}:
+        for entry in packages.get(name, {}).get("externals", []):
+            if (
+                spec.satisfies(entry["spec"])
+                and spec.external_path == entry.get("prefix")
+                and spec.external_modules == entry.get("modules")
+            ):
+                return True
+
+    return False
+
+
 class Solver:
     """This is the main external interface class for solving.
 
@@ -3172,8 +3200,18 @@ def _reusable_specs(self, specs):
 
             # Specs from buildcaches
             try:
-                index = spack.binary_distribution.update_cache_and_get_specs()
-                reusable_specs.extend(index)
+                # Specs in a build cache that depend on externals are reusable as long as local
+                # config has matching externals. This should guard against picking up binaries
+                # linked against externals not available locally, while still supporting the use
+                # case of distributing binaries across machines with similar externals.
+                packages = spack.config.get("packages")
+                reusable_specs.extend(
+                    [
+                        s
+                        for s in spack.binary_distribution.update_cache_and_get_specs()
+                        if not s.external or _is_reusable_external(packages, s)
+                    ]
+                )
             except (spack.binary_distribution.FetchCacheError, IndexError):
                 # this is raised when no mirrors had indices.
                 # TODO: update mirror configuration so it can indicate that the

lib/spack/spack/solver/concretize.lp

@@ -20,7 +20,7 @@
 % Integrity constraints on DAG nodes
 :- attr("root", PackageNode), not attr("node", PackageNode).
 :- attr("version", PackageNode, _), not attr("node", PackageNode), not attr("virtual_node", PackageNode).
-:- attr("node_version_satisfies", PackageNode), not attr("node", PackageNode).
+:- attr("node_version_satisfies", PackageNode, _), not attr("node", PackageNode), not attr("virtual_node", PackageNode).
 :- attr("hash", PackageNode, _), not attr("node", PackageNode).
 :- attr("node_platform", PackageNode, _), not attr("node", PackageNode).
 :- attr("node_os", PackageNode, _), not attr("node", PackageNode).

lib/spack/spack/test/cmd/env.py

@@ -3540,7 +3540,7 @@ def test_environment_created_in_users_location(mutable_mock_env_path, tmp_path):
     assert os.path.isdir(os.path.join(env_dir, dir_name))
 
 
-def test_environment_created_from_lockfile_has_view(mock_packages, tmpdir):
+def test_environment_created_from_lockfile_has_view(mock_packages, temporary_store, tmpdir):
     """When an env is created from a lockfile, a view should be generated for it"""
     env_a = str(tmpdir.join("a"))
     env_b = str(tmpdir.join("b"))

lib/spack/spack/test/cmd/gpg.py

@@ -43,7 +43,7 @@ def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch):
                 f.write(TEMPLATE.format(version=version))
             fs.set_executable(fname)
 
-    monkeypatch.setitem(os.environ, "PATH", str(tmpdir))
+    monkeypatch.setenv("PATH", str(tmpdir))
     if version == "undetectable" or version.endswith("1.3.4"):
         with pytest.raises(spack.util.gpg.SpackGPGError):
             spack.util.gpg.init(force=True)
@@ -54,7 +54,7 @@ def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch):
 
 
 def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch, mutable_config):
-    monkeypatch.setitem(os.environ, "PATH", str(tmpdir))
+    monkeypatch.setenv("PATH", str(tmpdir))
     bootstrap("disable")
     with pytest.raises(RuntimeError):
         spack.util.gpg.init(force=True)

lib/spack/spack/test/cmd/install.py

@@ -904,13 +904,12 @@ def test_install_help_cdash():
 
 
 @pytest.mark.disable_clean_stage_check
-def test_cdash_auth_token(tmpdir, mock_fetch, install_mockery, capfd):
+def test_cdash_auth_token(tmpdir, mock_fetch, install_mockery, monkeypatch, capfd):
     # capfd interferes with Spack's capturing
-    with tmpdir.as_cwd():
-        with capfd.disabled():
-            os.environ["SPACK_CDASH_AUTH_TOKEN"] = "asdf"
-            out = install("-v", "--log-file=cdash_reports", "--log-format=cdash", "a")
-            assert "Using CDash auth token from environment" in out
+    with tmpdir.as_cwd(), capfd.disabled():
+        monkeypatch.setenv("SPACK_CDASH_AUTH_TOKEN", "asdf")
+        out = install("-v", "--log-file=cdash_reports", "--log-format=cdash", "a")
+        assert "Using CDash auth token from environment" in out
 
 
 @pytest.mark.not_on_windows("Windows log_output logs phase header out of order")

lib/spack/spack/test/cmd/spec.py

@@ -104,8 +104,7 @@ def test_spec_parse_unquoted_flags_report():
         spec("gcc cflags=-Os -pipe")
     cm = str(cm.value)
     assert cm.startswith(
-        'trying to set variant "pipe" in package "gcc", but the package has no such '
-        'variant [happened during concretization of gcc cflags="-Os" ~pipe]'
+        'trying to set variant "pipe" in package "gcc", but the package has no such variant'
     )
     assert cm.endswith('(1) cflags=-Os -pipe => cflags="-Os -pipe"')
 

lib/spack/spack/test/concretize.py

@@ -203,7 +203,9 @@ def change(self, changes=None):
                 # TODO: in case tests using this fixture start failing.
                 if sys.modules.get("spack.pkg.changing.changing"):
                     del sys.modules["spack.pkg.changing.changing"]
+                if sys.modules.get("spack.pkg.changing.root"):
                     del sys.modules["spack.pkg.changing.root"]
+                if sys.modules.get("spack.pkg.changing"):
                     del sys.modules["spack.pkg.changing"]
 
                 # Change the recipe
@@ -1604,7 +1606,9 @@ def test_installed_version_is_selected_only_for_reuse(
         assert not new_root["changing"].satisfies("@1.0")
 
     @pytest.mark.regression("28259")
-    def test_reuse_with_unknown_namespace_dont_raise(self, mock_custom_repository):
+    def test_reuse_with_unknown_namespace_dont_raise(
+        self, temporary_store, mock_custom_repository
+    ):
         with spack.repo.use_repositories(mock_custom_repository, override=False):
             s = Spec("c").concretized()
             assert s.namespace != "builtin.mock"
@@ -1615,8 +1619,8 @@ def test_reuse_with_unknown_namespace_dont_raise(self, mock_custom_repository):
         assert s.namespace == "builtin.mock"
 
     @pytest.mark.regression("28259")
-    def test_reuse_with_unknown_package_dont_raise(self, tmpdir, monkeypatch):
-        builder = spack.repo.MockRepositoryBuilder(tmpdir, namespace="myrepo")
+    def test_reuse_with_unknown_package_dont_raise(self, tmpdir, temporary_store, monkeypatch):
+        builder = spack.repo.MockRepositoryBuilder(tmpdir.mkdir("mock.repo"), namespace="myrepo")
         builder.add_package("c")
         with spack.repo.use_repositories(builder.root, override=False):
             s = Spec("c").concretized()
@@ -2423,3 +2427,81 @@ def test_virtuals_provided_together_but_only_one_required_in_dag(self):
         s = Spec("blas-only-client ^openblas").concretized()
         assert s.satisfies("^[virtuals=blas] openblas")
         assert not s.satisfies("^[virtuals=blas,lapack] openblas")
+
+
+def test_reusable_externals_match(mock_packages, tmpdir):
+    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec.external_path = tmpdir.strpath
+    spec.external_modules = ["mpich/4.1"]
+    spec._mark_concrete()
+    assert spack.solver.asp._is_reusable_external(
+        {
+            "mpich": {
+                "externals": [
+                    {"spec": "mpich@4.1", "prefix": tmpdir.strpath, "modules": ["mpich/4.1"]}
+                ]
+            }
+        },
+        spec,
+    )
+
+
+def test_reusable_externals_match_virtual(mock_packages, tmpdir):
+    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec.external_path = tmpdir.strpath
+    spec.external_modules = ["mpich/4.1"]
+    spec._mark_concrete()
+    assert spack.solver.asp._is_reusable_external(
+        {
+            "mpi": {
+                "externals": [
+                    {"spec": "mpich@4.1", "prefix": tmpdir.strpath, "modules": ["mpich/4.1"]}
+                ]
+            }
+        },
+        spec,
+    )
+
+
+def test_reusable_externals_different_prefix(mock_packages, tmpdir):
+    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec.external_path = "/other/path"
+    spec.external_modules = ["mpich/4.1"]
+    spec._mark_concrete()
+    assert not spack.solver.asp._is_reusable_external(
+        {
+            "mpich": {
+                "externals": [
+                    {"spec": "mpich@4.1", "prefix": tmpdir.strpath, "modules": ["mpich/4.1"]}
+                ]
+            }
+        },
+        spec,
+    )
+
+
+@pytest.mark.parametrize("modules", [None, ["mpich/4.1", "libfabric/1.19"]])
+def test_reusable_externals_different_modules(mock_packages, tmpdir, modules):
+    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec.external_path = tmpdir.strpath
+    spec.external_modules = modules
+    spec._mark_concrete()
+    assert not spack.solver.asp._is_reusable_external(
+        {
+            "mpich": {
+                "externals": [
+                    {"spec": "mpich@4.1", "prefix": tmpdir.strpath, "modules": ["mpich/4.1"]}
+                ]
+            }
+        },
+        spec,
+    )
+
+
+def test_reusable_externals_different_spec(mock_packages, tmpdir):
+    spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
+    spec.external_path = tmpdir.strpath
+    spec._mark_concrete()
+    assert not spack.solver.asp._is_reusable_external(
+        {"mpich": {"externals": [{"spec": "mpich@4.1 +debug", "prefix": tmpdir.strpath}]}}, spec
+    )

lib/spack/spack/test/concretize_preferences.py

@@ -504,3 +504,13 @@ def test_sticky_variant_accounts_for_packages_yaml(self):
         with spack.config.override("packages:sticky-variant", {"variants": "+allow-gcc"}):
             s = Spec("sticky-variant %gcc").concretized()
             assert s.satisfies("%gcc") and s.satisfies("+allow-gcc")
+
+    @pytest.mark.regression("41134")
+    @pytest.mark.only_clingo("Not backporting the fix to the old concretizer")
+    def test_default_preference_variant_different_type_does_not_error(self):
+        """Tests that a different type for an existing variant in the 'all:' section of
+        packages.yaml doesn't fail with an error.
+        """
+        with spack.config.override("packages:all", {"variants": "+foo"}):
+            s = Spec("a").concretized()
+            assert s.satisfies("foo=bar")

lib/spack/spack/test/concretize_requirements.py

@@ -896,3 +896,134 @@ def test_requires_directive(concretize_scope, mock_packages):
     # This package can only be compiled with clang
     with pytest.raises(spack.error.SpackError, match="can only be compiled with Clang"):
         Spec("requires_clang").concretized()
+
+
+@pytest.mark.parametrize(
+    "packages_yaml",
+    [
+        # Simple string
+        """
+        packages:
+          all:
+            require: "+shared"
+    """,
+        # List of strings
+        """
+        packages:
+          all:
+            require:
+            - "+shared"
+    """,
+        # Objects with attributes
+        """
+        packages:
+          all:
+            require:
+            - spec: "+shared"
+    """,
+        """
+        packages:
+          all:
+            require:
+            - one_of: ["+shared"]
+    """,
+    ],
+)
+def test_default_requirements_semantic(packages_yaml, concretize_scope, mock_packages):
+    """Tests that requirements under 'all:' are by default applied only if the variant/property
+    required exists, but are strict otherwise.
+
+    For example:
+
+      packages:
+        all:
+          require: "+shared"
+
+    should enforce the value of "+shared" when a Boolean variant named "shared" exists. This is
+    not overridable from the command line, so with the configuration above:
+
+    > spack spec zlib~shared
+
+    is unsatisfiable.
+    """
+    update_packages_config(packages_yaml)
+
+    # Regular zlib concretize to +shared
+    s = Spec("zlib").concretized()
+    assert s.satisfies("+shared")
+
+    # If we specify the variant we can concretize only the one matching the constraint
+    s = Spec("zlib +shared").concretized()
+    assert s.satisfies("+shared")
+    with pytest.raises(UnsatisfiableSpecError):
+        Spec("zlib ~shared").concretized()
+
+    # A spec without the shared variant still concretize
+    s = Spec("a").concretized()
+    assert not s.satisfies("a +shared")
+    assert not s.satisfies("a ~shared")
+
+
+@pytest.mark.parametrize(
+    "packages_yaml,spec_str,expected,not_expected",
+    [
+        # The package has a 'libs' mv variant defaulting to 'libs=shared'
+        (
+            """
+        packages:
+          all:
+            require: "+libs"
+    """,
+            "multivalue-variant",
+            ["libs=shared"],
+            ["libs=static", "+libs"],
+        ),
+        (
+            """
+        packages:
+          all:
+            require: "libs=foo"
+    """,
+            "multivalue-variant",
+            ["libs=shared"],
+            ["libs=static", "libs=foo"],
+        ),
+        (
+            # (TODO): revisit this case when we'll have exact value semantic for mv variants
+            """
+        packages:
+          all:
+            require: "libs=static"
+    """,
+            "multivalue-variant",
+            ["libs=static", "libs=shared"],
+            [],
+        ),
+        (
+            # Constraint apply as a whole, so having a non-existing variant
+            # invalidate the entire constraint
+            """
+        packages:
+          all:
+            require: "libs=static +feefoo"
+    """,
+            "multivalue-variant",
+            ["libs=shared"],
+            ["libs=static"],
+        ),
+    ],
+)
+def test_default_requirements_semantic_with_mv_variants(
+    packages_yaml, spec_str, expected, not_expected, concretize_scope, mock_packages
+):
+    """Tests that requirements under 'all:' are behaving correctly under cases that could stem
+    from MV variants.
+    """
+    update_packages_config(packages_yaml)
+    s = Spec(spec_str).concretized()
+
+    for constraint in expected:
+        assert s.satisfies(constraint), constraint
+
+    for constraint in not_expected:
+        assert not s.satisfies(constraint), constraint

lib/spack/spack/test/config.py

@@ -1239,11 +1239,11 @@ def test_user_config_path_is_default_when_env_var_is_empty(working_env):
     assert os.path.expanduser("~%s.spack" % os.sep) == spack.paths._get_user_config_path()
 
 
-def test_default_install_tree(monkeypatch):
+def test_default_install_tree(monkeypatch, default_config):
     s = spack.spec.Spec("nonexistent@x.y.z %none@a.b.c arch=foo-bar-baz")
     monkeypatch.setattr(s, "dag_hash", lambda: "abc123")
-    projection = spack.config.get("config:install_tree:projections:all", scope="defaults")
-    assert s.format(projection) == "foo-bar-baz/none-a.b.c/nonexistent-x.y.z-abc123"
+    _, _, projections = spack.store.parse_install_tree(spack.config.get("config"))
+    assert s.format(projections["all"]) == "foo-bar-baz/none-a.b.c/nonexistent-x.y.z-abc123"
 
 
 def test_local_config_can_be_disabled(working_env):

lib/spack/spack/test/conftest.py

@@ -630,7 +630,7 @@ def platform_config():
     spack.config.add_default_platform_scope(spack.platforms.real_host().name)
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture
 def default_config():
     """Isolates the default configuration from the user configs.
 
@@ -714,19 +714,16 @@ def configuration_dir(tmpdir_factory, linux_os):
     t.write(content)
     yield tmpdir
 
-    # Once done, cleanup the directory
-    shutil.rmtree(str(tmpdir))
-
 
 def _create_mock_configuration_scopes(configuration_dir):
     """Create the configuration scopes used in `config` and `mutable_config`."""
-    scopes = [spack.config.InternalConfigScope("_builtin", spack.config.CONFIG_DEFAULTS)]
-    scopes += [
-        spack.config.ConfigScope(name, str(configuration_dir.join(name)))
-        for name in ["site", "system", "user"]
+    return [
+        spack.config.InternalConfigScope("_builtin", spack.config.CONFIG_DEFAULTS),
+        spack.config.ConfigScope("site", str(configuration_dir.join("site"))),
+        spack.config.ConfigScope("system", str(configuration_dir.join("system"))),
+        spack.config.ConfigScope("user", str(configuration_dir.join("user"))),
+        spack.config.InternalConfigScope("command_line"),
     ]
-    scopes += [spack.config.InternalConfigScope("command_line")]
-    return scopes
 
 
 @pytest.fixture(scope="session")

lib/spack/spack/test/env.py

@@ -695,7 +695,7 @@ def test_removing_spec_from_manifest_with_exact_duplicates(
 
 @pytest.mark.regression("35298")
 @pytest.mark.only_clingo("Propagation not supported in the original concretizer")
-def test_variant_propagation_with_unify_false(tmp_path, mock_packages):
+def test_variant_propagation_with_unify_false(tmp_path, mock_packages, config):
     """Spack distributes concretizations to different processes, when unify:false is selected and
     the number of roots is 2 or more. When that happens, the specs to be concretized need to be
     properly reconstructed on the worker process, if variant propagation was requested.
@@ -778,3 +778,32 @@ def test_env_with_include_def_missing(mutable_mock_env_path, mock_packages):
     with e:
         with pytest.raises(UndefinedReferenceError, match=r"which does not appear"):
             e.concretize()
+
+
+@pytest.mark.regression("41292")
+def test_deconcretize_then_concretize_does_not_error(mutable_mock_env_path, mock_packages):
+    """Tests that, after having deconcretized a spec, we can reconcretize an environment which
+    has 2 or more user specs mapping to the same concrete spec.
+    """
+    mutable_mock_env_path.mkdir()
+    spack_yaml = mutable_mock_env_path / ev.manifest_name
+    spack_yaml.write_text(
+        """spack:
+      specs:
+      # These two specs concretize to the same hash
+      - c
+      - c@1.0
+      # Spec used to trigger the bug
+      - a
+      concretizer:
+        unify: true
+    """
+    )
+    e = ev.Environment(mutable_mock_env_path)
+    with e:
+        e.concretize()
+        e.deconcretize(spack.spec.Spec("a"), concrete=False)
+        e.concretize()
+    assert len(e.concrete_roots()) == 3
+    all_root_hashes = set(x.dag_hash() for x in e.concrete_roots())
+    assert len(all_root_hashes) == 2

lib/spack/spack/test/make_executable.py

@@ -9,10 +9,7 @@
 This just tests whether the right args are getting passed to make.
 """
 import os
-import shutil
 import sys
-import tempfile
-import unittest
 
 import pytest
 
@@ -20,110 +17,104 @@
 from spack.util.environment import path_put_first
 
 pytestmark = pytest.mark.skipif(
-    sys.platform == "win32",
-    reason="MakeExecutable \
-                                        not supported on Windows",
+    sys.platform == "win32", reason="MakeExecutable not supported on Windows"
 )
 
 
-class MakeExecutableTest(unittest.TestCase):
-    def setUp(self):
-        self.tmpdir = tempfile.mkdtemp()
+@pytest.fixture(autouse=True)
+def make_executable(tmp_path, working_env):
+    make_exe = tmp_path / "make"
+    with open(make_exe, "w") as f:
+        f.write("#!/bin/sh\n")
+        f.write('echo "$@"')
+    os.chmod(make_exe, 0o700)
 
-        make_exe = os.path.join(self.tmpdir, "make")
-        with open(make_exe, "w") as f:
-            f.write("#!/bin/sh\n")
-            f.write('echo "$@"')
-        os.chmod(make_exe, 0o700)
+    path_put_first("PATH", [tmp_path])
 
-        path_put_first("PATH", [self.tmpdir])
 
-    def tearDown(self):
-        shutil.rmtree(self.tmpdir)
+def test_make_normal():
+    make = MakeExecutable("make", 8)
+    assert make(output=str).strip() == "-j8"
+    assert make("install", output=str).strip() == "-j8 install"
 
-    def test_make_normal(self):
-        make = MakeExecutable("make", 8)
-        self.assertEqual(make(output=str).strip(), "-j8")
-        self.assertEqual(make("install", output=str).strip(), "-j8 install")
 
-    def test_make_explicit(self):
-        make = MakeExecutable("make", 8)
-        self.assertEqual(make(parallel=True, output=str).strip(), "-j8")
-        self.assertEqual(make("install", parallel=True, output=str).strip(), "-j8 install")
+def test_make_explicit():
+    make = MakeExecutable("make", 8)
+    assert make(parallel=True, output=str).strip() == "-j8"
+    assert make("install", parallel=True, output=str).strip() == "-j8 install"
 
-    def test_make_one_job(self):
-        make = MakeExecutable("make", 1)
-        self.assertEqual(make(output=str).strip(), "-j1")
-        self.assertEqual(make("install", output=str).strip(), "-j1 install")
 
-    def test_make_parallel_false(self):
-        make = MakeExecutable("make", 8)
-        self.assertEqual(make(parallel=False, output=str).strip(), "-j1")
-        self.assertEqual(make("install", parallel=False, output=str).strip(), "-j1 install")
+def test_make_one_job():
+    make = MakeExecutable("make", 1)
+    assert make(output=str).strip() == "-j1"
+    assert make("install", output=str).strip() == "-j1 install"
 
-    def test_make_parallel_disabled(self):
-        make = MakeExecutable("make", 8)
 
-        os.environ["SPACK_NO_PARALLEL_MAKE"] = "true"
-        self.assertEqual(make(output=str).strip(), "-j1")
-        self.assertEqual(make("install", output=str).strip(), "-j1 install")
+def test_make_parallel_false():
+    make = MakeExecutable("make", 8)
+    assert make(parallel=False, output=str).strip() == "-j1"
+    assert make("install", parallel=False, output=str).strip() == "-j1 install"
 
-        os.environ["SPACK_NO_PARALLEL_MAKE"] = "1"
-        self.assertEqual(make(output=str).strip(), "-j1")
-        self.assertEqual(make("install", output=str).strip(), "-j1 install")
 
-        # These don't disable (false and random string)
-        os.environ["SPACK_NO_PARALLEL_MAKE"] = "false"
-        self.assertEqual(make(output=str).strip(), "-j8")
-        self.assertEqual(make("install", output=str).strip(), "-j8 install")
+def test_make_parallel_disabled(monkeypatch):
+    make = MakeExecutable("make", 8)
 
-        os.environ["SPACK_NO_PARALLEL_MAKE"] = "foobar"
-        self.assertEqual(make(output=str).strip(), "-j8")
-        self.assertEqual(make("install", output=str).strip(), "-j8 install")
+    monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "true")
+    assert make(output=str).strip() == "-j1"
+    assert make("install", output=str).strip() == "-j1 install"
 
-        del os.environ["SPACK_NO_PARALLEL_MAKE"]
+    monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "1")
+    assert make(output=str).strip() == "-j1"
+    assert make("install", output=str).strip() == "-j1 install"
 
-    def test_make_parallel_precedence(self):
-        make = MakeExecutable("make", 8)
+    # These don't disable (false and random string)
+    monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "false")
+    assert make(output=str).strip() == "-j8"
+    assert make("install", output=str).strip() == "-j8 install"
 
-        # These should work
-        os.environ["SPACK_NO_PARALLEL_MAKE"] = "true"
-        self.assertEqual(make(parallel=True, output=str).strip(), "-j1")
-        self.assertEqual(make("install", parallel=True, output=str).strip(), "-j1 install")
+    monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "foobar")
+    assert make(output=str).strip() == "-j8"
+    assert make("install", output=str).strip() == "-j8 install"
 
-        os.environ["SPACK_NO_PARALLEL_MAKE"] = "1"
-        self.assertEqual(make(parallel=True, output=str).strip(), "-j1")
-        self.assertEqual(make("install", parallel=True, output=str).strip(), "-j1 install")
 
-        # These don't disable (false and random string)
-        os.environ["SPACK_NO_PARALLEL_MAKE"] = "false"
-        self.assertEqual(make(parallel=True, output=str).strip(), "-j8")
-        self.assertEqual(make("install", parallel=True, output=str).strip(), "-j8 install")
+def test_make_parallel_precedence(monkeypatch):
+    make = MakeExecutable("make", 8)
 
-        os.environ["SPACK_NO_PARALLEL_MAKE"] = "foobar"
-        self.assertEqual(make(parallel=True, output=str).strip(), "-j8")
-        self.assertEqual(make("install", parallel=True, output=str).strip(), "-j8 install")
+    # These should work
+    monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "true")
+    assert make(parallel=True, output=str).strip() == "-j1"
+    assert make("install", parallel=True, output=str).strip() == "-j1 install"
 
-        del os.environ["SPACK_NO_PARALLEL_MAKE"]
+    monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "1")
+    assert make(parallel=True, output=str).strip() == "-j1"
+    assert make("install", parallel=True, output=str).strip() == "-j1 install"
 
-    def test_make_jobs_env(self):
-        make = MakeExecutable("make", 8)
-        dump_env = {}
-        self.assertEqual(
-            make(output=str, jobs_env="MAKE_PARALLELISM", _dump_env=dump_env).strip(), "-j8"
-        )
-        self.assertEqual(dump_env["MAKE_PARALLELISM"], "8")
+    # These don't disable (false and random string)
+    monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "false")
+    assert make(parallel=True, output=str).strip() == "-j8"
+    assert make("install", parallel=True, output=str).strip() == "-j8 install"
 
-    def test_make_jobserver(self):
-        make = MakeExecutable("make", 8)
-        os.environ["MAKEFLAGS"] = "--jobserver-auth=X,Y"
-        self.assertEqual(make(output=str).strip(), "")
-        self.assertEqual(make(parallel=False, output=str).strip(), "-j1")
-        del os.environ["MAKEFLAGS"]
+    monkeypatch.setenv("SPACK_NO_PARALLEL_MAKE", "foobar")
+    assert make(parallel=True, output=str).strip() == "-j8"
+    assert make("install", parallel=True, output=str).strip() == "-j8 install"
 
-    def test_make_jobserver_not_supported(self):
-        make = MakeExecutable("make", 8, supports_jobserver=False)
-        os.environ["MAKEFLAGS"] = "--jobserver-auth=X,Y"
-        # Currently fallback on default job count, Maybe it should force -j1 ?
-        self.assertEqual(make(output=str).strip(), "-j8")
-        del os.environ["MAKEFLAGS"]
+
+def test_make_jobs_env():
+    make = MakeExecutable("make", 8)
+    dump_env = {}
+    assert make(output=str, jobs_env="MAKE_PARALLELISM", _dump_env=dump_env).strip() == "-j8"
+    assert dump_env["MAKE_PARALLELISM"] == "8"
+
+
+def test_make_jobserver(monkeypatch):
+    make = MakeExecutable("make", 8)
+    monkeypatch.setenv("MAKEFLAGS", "--jobserver-auth=X,Y")
+    assert make(output=str).strip() == ""
+    assert make(parallel=False, output=str).strip() == "-j1"
+
+
+def test_make_jobserver_not_supported(monkeypatch):
+    make = MakeExecutable("make", 8, supports_jobserver=False)
+    monkeypatch.setenv("MAKEFLAGS", "--jobserver-auth=X,Y")
+    # Currently fallback on default job count, Maybe it should force -j1 ?
+    assert make(output=str).strip() == "-j8"

lib/spack/spack/test/module_parsing.py

@@ -27,16 +27,13 @@
 ]
 
 
-def test_module_function_change_env(tmpdir, working_env):
-    src_file = str(tmpdir.join("src_me"))
-    with open(src_file, "w") as f:
-        f.write("export TEST_MODULE_ENV_VAR=TEST_SUCCESS\n")
-
-    os.environ["NOT_AFFECTED"] = "NOT_AFFECTED"
-    module("load", src_file, module_template=". {0} 2>&1".format(src_file))
-
-    assert os.environ["TEST_MODULE_ENV_VAR"] == "TEST_SUCCESS"
-    assert os.environ["NOT_AFFECTED"] == "NOT_AFFECTED"
+def test_module_function_change_env(tmp_path):
+    environb = {b"TEST_MODULE_ENV_VAR": b"TEST_FAIL", b"NOT_AFFECTED": b"NOT_AFFECTED"}
+    src_file = tmp_path / "src_me"
+    src_file.write_text("export TEST_MODULE_ENV_VAR=TEST_SUCCESS\n")
+    module("load", str(src_file), module_template=f". {src_file} 2>&1", environb=environb)
+    assert environb[b"TEST_MODULE_ENV_VAR"] == b"TEST_SUCCESS"
+    assert environb[b"NOT_AFFECTED"] == b"NOT_AFFECTED"
 
 
 def test_module_function_no_change(tmpdir):

lib/spack/spack/test/util/environment.py

@@ -147,7 +147,8 @@ def test_reverse_environment_modifications(working_env):
 
     reversal = to_reverse.reversed()
 
-    os.environ = start_env.copy()
+    os.environ.clear()
+    os.environ.update(start_env)
 
     print(os.environ)
     to_reverse.apply_modifications()

lib/spack/spack/test/util/executable.py

@@ -89,8 +89,8 @@ def test_which_with_slash_ignores_path(tmpdir, working_env):
         assert exe.path == path
 
 
-def test_which(tmpdir):
-    os.environ["PATH"] = str(tmpdir)
+def test_which(tmpdir, monkeypatch):
+    monkeypatch.setenv("PATH", str(tmpdir))
     assert ex.which("spack-test-exe") is None
 
     with pytest.raises(ex.CommandNotFoundError):

lib/spack/spack/util/module_cmd.py

@@ -10,6 +10,7 @@
 import os
 import re
 import subprocess
+from typing import MutableMapping, Optional
 
 import llnl.util.tty as tty
 
@@ -21,8 +22,13 @@
 awk_cmd = r"""awk 'BEGIN{for(name in ENVIRON)""" r"""printf("%s=%s%c", name, ENVIRON[name], 0)}'"""
 
 
-def module(*args, **kwargs):
-    module_cmd = kwargs.get("module_template", "module " + " ".join(args))
+def module(
+    *args,
+    module_template: Optional[str] = None,
+    environb: Optional[MutableMapping[bytes, bytes]] = None,
+):
+    module_cmd = module_template or ("module " + " ".join(args))
+    environb = environb or os.environb
 
     if args[0] in module_change_commands:
         # Suppress module output
@@ -33,10 +39,10 @@ def module(*args, **kwargs):
             stderr=subprocess.STDOUT,
             shell=True,
             executable="/bin/bash",
+            env=environb,
         )
 
-        # In Python 3, keys and values of `environ` are byte strings.
-        environ = {}
+        new_environb = {}
         output = module_p.communicate()[0]
 
         # Loop over each environment variable key=value byte string
@@ -45,11 +51,11 @@ def module(*args, **kwargs):
             parts = entry.split(b"=", 1)
             if len(parts) != 2:
                 continue
-            environ[parts[0]] = parts[1]
+            new_environb[parts[0]] = parts[1]
 
         # Update os.environ with new dict
-        os.environ.clear()
-        os.environb.update(environ)  # novermin
+        environb.clear()
+        environb.update(new_environb)  # novermin
 
     else:
         # Simply execute commands that don't change state and return output