damask: add version 3.0.0-alpha8 (#41444)

* damask 3.0.0-alpha8

* ensuring correct Python versions for py-damask

.github/workflows/audit.yaml

@@ -23,7 +23,7 @@ jobs:
         operating_system: ["ubuntu-latest", "macos-latest"]
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
       with:
         python-version: ${{inputs.python_version}}
     - name: Install Python packages

.github/workflows/bootstrap.yml

@@ -159,7 +159,7 @@ jobs:
           brew install cmake bison@2.7 tree
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
-      - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
+      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
         with:
           python-version: "3.12"
       - name: Bootstrap clingo

.github/workflows/build-containers.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
 
-      - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934
+      - uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e
         id: docker_meta
         with:
           images: |
@@ -113,7 +113,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build & Deploy ${{ matrix.dockerfile[0] }}
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56
         with:
           context: dockerfiles/${{ matrix.dockerfile[0] }}
           platforms: ${{ matrix.dockerfile[1] }}

.github/workflows/nightly-win-builds.yml

@@ -17,7 +17,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: 3.9
     - name: Install Python packages

.github/workflows/style/requirements.txt

@@ -1,7 +1,7 @@
-black==23.10.1
+black==23.12.0
 clingo==5.6.2
 flake8==6.1.0
-isort==5.12.0
-mypy==1.6.1
+isort==5.13.2
+mypy==1.7.1
 types-six==1.16.21.9
-vermin==1.5.2
+vermin==1.6.0

.github/workflows/unit_tests.yaml

@@ -54,7 +54,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install System packages
@@ -101,7 +101,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -159,7 +159,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -194,7 +194,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Python packages
@@ -215,7 +215,7 @@ jobs:
         $(which spack) bootstrap disable spack-install
         $(which spack) solve zlib
         common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
-        $(which spack) unit-test --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
+        $(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
     - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
       with:
         flags: unittests,macos

.github/workflows/valid-style.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: '3.11'
         cache: 'pip'
@@ -38,7 +38,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: '3.11'
         cache: 'pip'

.github/workflows/windows_python.yml

@@ -18,7 +18,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: 3.9
     - name: Install Python packages
@@ -42,7 +42,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: 3.9
     - name: Install Python packages
@@ -66,7 +66,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: 3.9
     - name: Install Python packages

CHANGELOG.md

@@ -1,3 +1,290 @@
+# v0.21.0 (2023-11-11)
+
+`v0.21.0` is a major feature release.
+
+## Features in this release
+
+1. **Better error messages with condition chaining**
+
+   In v0.18, we added better error messages that could tell you what problem happened,
+   but they couldn't tell you *why* it happened. `0.21` adds *condition chaining* to the
+   solver, and Spack can now trace back through the conditions that led to an error and
+   build a tree of causes potential causes and where they came from. For example:
+
+   ```console
+   $ spack solve hdf5 ^cmake@3.0.1
+   ==> Error: concretization failed for the following reasons:
+
+      1. Cannot satisfy 'cmake@3.0.1'
+      2. Cannot satisfy 'cmake@3.0.1'
+           required because hdf5 ^cmake@3.0.1 requested from CLI
+      3. Cannot satisfy 'cmake@3.18:' and 'cmake@3.0.1
+           required because hdf5 ^cmake@3.0.1 requested from CLI
+           required because hdf5 depends on cmake@3.18: when @1.13:
+             required because hdf5 ^cmake@3.0.1 requested from CLI
+      4. Cannot satisfy 'cmake@3.12:' and 'cmake@3.0.1
+           required because hdf5 depends on cmake@3.12:
+             required because hdf5 ^cmake@3.0.1 requested from CLI
+           required because hdf5 ^cmake@3.0.1 requested from CLI
+   ```
+
+   More details in #40173.
+
+2. **OCI build caches**
+
+   You can now use an arbitrary [OCI](https://opencontainers.org) registry as a build
+   cache:
+
+   ```console
+   $ spack mirror add my_registry oci://user/image # Dockerhub
+   $ spack mirror add my_registry oci://ghcr.io/haampie/spack-test # GHCR
+   $ spack mirror set --push --oci-username ... --oci-password ... my_registry  # set login creds
+   $ spack buildcache push my_registry [specs...]
+   ```
+
+   And you can optionally add a base image to get *runnable* images:
+
+   ```console
+   $ spack buildcache push --base-image ubuntu:23.04 my_registry python
+   Pushed ... as [image]:python-3.11.2-65txfcpqbmpawclvtasuog4yzmxwaoia.spack
+
+   $ docker run --rm -it [image]:python-3.11.2-65txfcpqbmpawclvtasuog4yzmxwaoia.spack
+   ```
+
+   This creates a container image from the Spack installations on the host system,
+   without the need to run `spack install` from a `Dockerfile` or `sif` file. It also
+   addresses the inconvenience of losing binaries of dependencies when `RUN spack
+   install` fails inside `docker build`.
+
+   Further, the container image layers and build cache tarballs are the same files. This
+   means that `spack install` and `docker pull` use the exact same underlying binaries.
+   If you previously used `spack install` inside of `docker build`, this feature helps
+   you save storage by a factor two.
+
+   More details in #38358.
+
+3. **Multiple versions of build dependencies**
+
+   Increasingly, complex package builds require multiple versions of some build
+   dependencies. For example, Python packages frequently require very specific versions
+   of `setuptools`, `cython`, and sometimes different physics packages require different
+   versions of Python to build. The concretizer enforced that every solve was *unified*,
+   i.e., that there only be one version of every package. The concretizer now supports
+   "duplicate" nodes for *build dependencies*, but enforces unification through
+   transitive link and run dependencies. This will allow it to better resolve complex
+   dependency graphs in ecosystems like Python, and it also gets us very close to
+   modeling compilers as proper dependencies.
+
+   This change required a major overhaul of the concretizer, as well as a number of
+   performance optimizations. See #38447, #39621.
+
+4. **Cherry-picking virtual dependencies**
+
+   You can now select only a subset of virtual dependencies from a spec that may provide
+   more. For example, if you want `mpich` to be your `mpi` provider, you can be explicit
+   by writing:
+
+   ```
+   hdf5 ^[virtuals=mpi] mpich
+   ```
+
+   Or, if you want to use, e.g., `intel-parallel-studio` for `blas` along with an external
+   `lapack` like `openblas`, you could write:
+
+   ```
+   strumpack ^[virtuals=mpi] intel-parallel-studio+mkl ^[virtuals=lapack] openblas
+   ```
+
+   The `virtuals=mpi` is an edge attribute, and dependency edges in Spack graphs now
+   track which virtuals they satisfied. More details in #17229 and #35322.
+
+   Note for packaging: in Spack 0.21 `spec.satisfies("^virtual")` is true if and only if
+   the package specifies `depends_on("virtual")`. This is different from Spack 0.20,
+   where depending on a provider implied depending on the virtual provided. See #41002
+   for an example where `^mkl` was being used to test for several `mkl` providers in a
+   package that did not depend on `mkl`.
+
+5. **License directive**
+
+   Spack packages can now have license metadata, with the new `license()` directive:
+
+   ```python
+       license("Apache-2.0")
+   ```
+
+   Licenses use [SPDX identifiers](https://spdx.org/licenses), and you can use SPDX
+   expressions to combine them:
+
+   ```python
+       license("Apache-2.0 OR MIT")
+   ```
+
+   Like other directives in Spack, it's conditional, so you can handle complex cases like
+   Spack itself:
+
+   ```python
+      license("LGPL-2.1", when="@:0.11")
+      license("Apache-2.0 OR MIT", when="@0.12:")
+   ```
+
+   More details in #39346, #40598.
+
+6. **`spack deconcretize` command**
+
+   We are getting close to having a `spack update` command for environments, but we're
+   not quite there yet. This is the next best thing. `spack deconcretize` gives you
+   control over what you want to update in an already concrete environment. If you have
+   an environment built with, say, `meson`, and you want to update your `meson` version,
+   you can run:
+
+   ```console
+   spack deconcretize meson
+   ```
+
+   and have everything that depends on `meson` rebuilt the next time you run `spack
+   concretize`. In a future Spack version, we'll handle all of this in a single command,
+   but for now you can use this to drop bits of your lockfile and resolve your
+   dependencies again. More in #38803.
+
+7. **UI Improvements**
+
+   The venerable `spack info` command was looking shabby compared to the rest of Spack's
+   UI, so we reworked it to have a bit more flair. `spack info` now makes much better
+   use of terminal space and shows variants, their values, and their descriptions much
+   more clearly. Conditional variants are grouped separately so you can more easily
+   understand how packages are structured. More in #40998.
+
+   `spack checksum` now allows you to filter versions from your editor, or by version
+   range. It also notifies you about potential download URL changes. See #40403.
+
+8. **Environments can include definitions**
+
+   Spack did not previously support using `include:` with The
+   [definitions](https://spack.readthedocs.io/en/latest/environments.html#spec-list-references)
+   section of an environment, but now it does. You can use this to curate lists of specs
+   and more easily reuse them across environments. See #33960.
+
+9. **Aliases**
+
+   You can now add aliases to Spack commands in `config.yaml`, e.g. this might enshrine
+   your favorite args to `spack find` as `spack f`:
+
+   ```yaml
+   config:
+     aliases:
+       f: find -lv
+   ```
+
+   See #17229.
+
+10. **Improved autoloading of modules**
+
+    Spack 0.20 was the first release to enable autoloading of direct dependencies in
+    module files.
+
+    The downside of this was that `module avail` and `module load` tab completion would
+    show users too many modules to choose from, and many users disabled generating
+    modules for dependencies through `exclude_implicits: true`. Further, it was
+    necessary to keep hashes in module names to avoid file name clashes.
+
+    In this release, you can start using `hide_implicits: true` instead, which exposes
+    only explicitly installed packages to the user, while still autoloading
+    dependencies. On top of that, you can safely use `hash_length: 0`, as this config
+    now only applies to the modules exposed to the user -- you don't have to worry about
+    file name clashes for hidden dependencies.
+
+   Note: for `tcl` this feature requires Modules 4.7 or higher
+
+11. **Updated container labeling**
+
+    Nightly Docker images from the `develop` branch will now be tagged as `:develop` and
+    `:nightly`. The `:latest` tag is no longer associated with `:develop`, but with the
+    latest stable release. Releases will be tagged with `:{major}`, `:{major}.{minor}`
+    and `:{major}.{minor}.{patch}`. `ubuntu:18.04` has also been removed from the list of
+    generated Docker images, as it is no longer supported. See #40593.
+
+## Other new commands and directives
+
+* `spack env activate` without arguments now loads a `default` environment that you do
+  not have to create (#40756).
+* `spack find -H` / `--hashes`: a new shortcut for piping `spack find` output to
+  other commands (#38663)
+* Add `spack checksum --verify`, fix `--add` (#38458)
+* New `default_args` context manager factors out common args for directives (#39964)
+* `spack compiler find --[no]-mixed-toolchain` lets you easily mix `clang` and
+  `gfortran` on Linux (#40902)
+
+## Performance improvements
+
+* `spack external find` execution is now much faster (#39843)
+* `spack location -i` now much faster on success (#40898)
+* Drop redundant rpaths post install (#38976)
+* ASP-based solver: avoid cycles in clingo using hidden directive (#40720)
+* Fix multiple quadratic complexity issues in environments (#38771)
+
+## Other new features of note
+
+* archspec: update to v0.2.2, support for Sapphire Rapids, Power10, Neoverse V2 (#40917)
+* Propagate variants across nodes that don't have that variant (#38512)
+* Implement fish completion (#29549)
+* Can now distinguish between source/binary mirror; don't ping mirror.spack.io as much (#34523)
+* Improve status reporting on install (add [n/total] display) (#37903)
+
+## Windows
+
+This release has the best Windows support of any Spack release yet, with numerous
+improvements and much larger swaths of tests passing:
+
+* MSVC and SDK improvements (#37711, #37930, #38500, #39823, #39180)
+* Windows external finding: update default paths; treat .bat as executable on Windows (#39850)
+* Windows decompression: fix removal of intermediate file (#38958)
+* Windows: executable/path handling (#37762)
+* Windows build systems: use ninja and enable tests (#33589)
+* Windows testing (#36970, #36972, #36973, #36840, #36977, #36792, #36834, #34696, #36971)
+* Windows PowerShell support (#39118, #37951)
+* Windows symlinking and libraries (#39933, #38599, #34701, #38578, #34701)
+
+## Notable refactors
+* User-specified flags take precedence over others in Spack compiler wrappers (#37376)
+* Improve setup of build, run, and test environments (#35737, #40916)
+* `make` is no longer a required system dependency of Spack (#40380)
+* Support Python 3.12 (#40404, #40155, #40153)
+* docs: Replace package list with packages.spack.io (#40251)
+* Drop Python 2 constructs in Spack (#38720, #38718, #38703)
+
+## Binary cache and stack updates
+* e4s arm stack: duplicate and target neoverse v1 (#40369)
+* Add macOS ML CI stacks (#36586)
+* E4S Cray CI Stack (#37837)
+* e4s cray: expand spec list (#38947)
+* e4s cray sles ci: expand spec list (#39081)
+
+## Removals, deprecations, and syntax changes
+* ASP: targets, compilers and providers soft-preferences are only global (#31261)
+* Parser: fix ambiguity with whitespace in version ranges (#40344)
+* Module file generation is disabled by default; you'll need to enable it to use it (#37258)
+* Remove deprecated "extra_instructions" option for containers (#40365)
+* Stand-alone test feature deprecation postponed to v0.22 (#40600)
+* buildcache push: make `--allow-root` the default and deprecate the option (#38878)
+
+## Notable Bugfixes
+* Bugfix: propagation of multivalued variants (#39833)
+* Allow `/` in git versions (#39398)
+* Fetch & patch: actually acquire stage lock, and many more issues (#38903)
+* Environment/depfile: better escaping of targets with Git versions (#37560)
+* Prevent "spack external find" to error out on wrong permissions (#38755)
+* lmod: allow core compiler to be specified with a version range (#37789)
+
+## Spack community stats
+
+* 7,469 total packages, 303 new since `v0.20.0`
+    * 150 new Python packages
+    * 34 new R packages
+* 353 people contributed to this release
+    * 336 committers to packages
+    * 65 committers to core
+
+
 # v0.20.3 (2023-10-31)
 
 ## Bugfixes

README.md

@@ -1,13 +1,34 @@
-# <img src="https://cdn.rawgit.com/spack/spack/develop/share/spack/logo/spack-logo.svg" width="64" valign="middle" alt="Spack"/> Spack
+<div align="left">
 
-[![Unit Tests](https://github.com/spack/spack/workflows/linux%20tests/badge.svg)](https://github.com/spack/spack/actions)
-[![Bootstrapping](https://github.com/spack/spack/actions/workflows/bootstrap.yml/badge.svg)](https://github.com/spack/spack/actions/workflows/bootstrap.yml)
-[![codecov](https://codecov.io/gh/spack/spack/branch/develop/graph/badge.svg)](https://codecov.io/gh/spack/spack)
-[![Containers](https://github.com/spack/spack/actions/workflows/build-containers.yml/badge.svg)](https://github.com/spack/spack/actions/workflows/build-containers.yml)
-[![Read the Docs](https://readthedocs.org/projects/spack/badge/?version=latest)](https://spack.readthedocs.io)
-[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
-[![Slack](https://slack.spack.io/badge.svg)](https://slack.spack.io)
-[![Matrix](https://img.shields.io/matrix/spack-space%3Amatrix.org?label=Matrix)](https://matrix.to/#/#spack-space:matrix.org)
+<h2>
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://cdn.rawgit.com/spack/spack/develop/share/spack/logo/spack-logo-white-text.svg" width="250">
+  <source media="(prefers-color-scheme: light)" srcset="https://cdn.rawgit.com/spack/spack/develop/share/spack/logo/spack-logo-text.svg" width="250">
+  <img alt="Spack" src="https://cdn.rawgit.com/spack/spack/develop/share/spack/logo/spack-logo-text.svg" width="250">
+</picture>
+
+<br>
+<br clear="all">
+
+<a href="https://github.com/spack/spack/actions/workflows/ci.yml"><img src="https://github.com/spack/spack/workflows/ci/badge.svg" alt="CI Status"></a>
+<a href="https://github.com/spack/spack/actions/workflows/bootstrapping.yml"><img src="https://github.com/spack/spack/actions/workflows/bootstrap.yml/badge.svg" alt="Bootstrap Status"></a>
+<a href="https://github.com/spack/spack/actions/workflows/build-containers.yml"><img src="https://github.com/spack/spack/actions/workflows/build-containers.yml/badge.svg" alt="Containers Status"></a>
+<a href="https://spack.readthedocs.io"><img src="https://readthedocs.org/projects/spack/badge/?version=latest" alt="Documentation Status"></a>
+<a href="https://codecov.io/gh/spack/spack"><img src="https://codecov.io/gh/spack/spack/branch/develop/graph/badge.svg" alt="Code coverage"/></a>
+<a href="https://slack.spack.io"><img src="https://slack.spack.io/badge.svg" alt="Slack"/></a>
+<a href="https://matrix.to/#/#spack-space:matrix.org"><img src="https://img.shields.io/matrix/spack-space%3Amatrix.org?label=matrix" alt="Matrix"/></a>
+
+</h2>
+
+**[Getting Started] &nbsp; • &nbsp; [Config] &nbsp; • &nbsp; [Community] &nbsp; • &nbsp; [Contributing] &nbsp; • &nbsp; [Packaging Guide]**
+
+[Getting Started]: https://spack.readthedocs.io/en/latest/getting_started.html
+[Config]: https://spack.readthedocs.io/en/latest/configuration.html
+[Community]: #community
+[Contributing]: https://spack.readthedocs.io/en/latest/contribution_guide.html
+[Packaging Guide]: https://spack.readthedocs.io/en/latest/packaging_guide.html
+
+</div>
 
 Spack is a multi-platform package manager that builds and installs
 multiple versions and configurations of software. It works on Linux,
@@ -66,10 +87,11 @@ Resources:
 * **Matrix space**: [#spack-space:matrix.org](https://matrix.to/#/#spack-space:matrix.org):
   [bridged](https://github.com/matrix-org/matrix-appservice-slack#matrix-appservice-slack) to Slack.
 * [**Github Discussions**](https://github.com/spack/spack/discussions):
-  not just for discussions, but also Q&A.
-* **Mailing list**: [groups.google.com/d/forum/spack](https://groups.google.com/d/forum/spack)
+  for Q&A and discussions. Note the pinned discussions for announcements.
 * **Twitter**: [@spackpm](https://twitter.com/spackpm). Be sure to
   `@mention` us!
+* **Mailing list**: [groups.google.com/d/forum/spack](https://groups.google.com/d/forum/spack):
+  only for announcements. Please use other venues for discussions.
 
 Contributing
 ------------------------

etc/spack/defaults/darwin/packages.yaml

@@ -50,4 +50,4 @@ packages:
     # Apple bundles libuuid in libsystem_c version 1353.100.2,
     # although the version number used here isn't critical
     - spec: apple-libuuid@1353.100.2
-      prefix: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk
+      prefix: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk

etc/spack/defaults/modules.yaml

@@ -46,12 +46,10 @@ modules:
     tcl:
       all:
         autoload: direct
-      hide_implicits: true
 
     # Default configurations if lmod is enabled
     lmod:
       all:
         autoload: direct
-      hide_implicits: true
       hierarchy:
         - mpi

lib/spack/docs/binary_caches.rst

@@ -153,7 +153,43 @@ keyring, and trusting all downloaded keys.
 List of popular build caches
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-* `Extreme-scale Scientific Software Stack (E4S) <https://e4s-project.github.io/>`_: `build cache <https://oaciss.uoregon.edu/e4s/inventory.html>`_
+* `Extreme-scale Scientific Software Stack (E4S) <https://e4s-project.github.io/>`_: `build cache <https://oaciss.uoregon.edu/e4s/inventory.html>`_'
+
+-------------------
+Build cache signing
+-------------------
+
+By default, Spack will add a cryptographic signature to each package pushed to
+a build cache, and verifies the signature when installing from a build cache.
+
+Keys for signing can be managed with the :ref:`spack gpg <cmd-spack-gpg>` command,
+as well as ``spack buildcache keys`` as mentioned above.
+
+You can disable signing when pushing with ``spack buildcache push --unsigned``,
+and disable verification when installing from any build cache with
+``spack install --no-check-signature``.
+
+Alternatively, signing and verification can be enabled or disabled on a per build cache
+basis:
+
+.. code-block:: console
+
+    $ spack mirror add --signed <name> <url>  # enable signing and verification
+    $ spack mirror add --unsigned <name> <url>  # disable signing and verification
+
+    $ spack mirror set --signed <name>  # enable signing and verification for an existing mirror
+    $ spack mirror set --unsigned <name>  # disable signing and verification for an existing mirror
+
+Or you can directly edit the ``mirrors.yaml`` configuration file:
+
+.. code-block:: yaml
+
+    mirrors:
+      <name>:
+        url: <url>
+        signed: false # disable signing and verification
+
+See also :ref:`mirrors`.
 
 ----------
 Relocation
@@ -182,6 +218,7 @@ section of the configuration:
        padded_length: 128
 
 
+.. _binary_caches_oci:
 
 -----------------------------------------
 OCI / Docker V2 registries as build cache
@@ -250,87 +287,13 @@ To significantly speed up Spack in GitHub Actions, binaries can be cached in
 GitHub Packages. This service is an OCI registry that can be linked to a GitHub
 repository.
 
-A typical workflow is to include a ``spack.yaml`` environment in your repository
-that specifies the packages to install, the target architecture, and the build
-cache to use under ``mirrors``:
-
-.. code-block:: yaml
-
-    spack:
-      specs:
-      - python@3.11
-      config:
-        install_tree:
-          root: /opt/spack
-          padded_length: 128
-      packages:
-        all:
-          require: target=x86_64_v2
-      mirrors:
-        local-buildcache: oci://ghcr.io/<organization>/<repository>
-
-A GitHub action can then be used to install the packages and push them to the
-build cache:
-
-.. code-block:: yaml
-
-    name: Install Spack packages
-
-    on: push
-
-    env:
-      SPACK_COLOR: always
-
-    jobs:
-      example:
-        runs-on: ubuntu-22.04
-        permissions:
-          packages: write
-        steps:
-        - name: Checkout
-          uses: actions/checkout@v3
-
-        - name: Checkout Spack
-          uses: actions/checkout@v3
-          with:
-            repository: spack/spack
-            path: spack
-
-        - name: Setup Spack
-          run: echo "$PWD/spack/bin" >> "$GITHUB_PATH"
-
-        - name: Concretize
-          run: spack -e . concretize
-
-        - name: Install
-          run: spack -e . install --no-check-signature
-
-        - name: Run tests
-            run: ./my_view/bin/python3 -c 'print("hello world")'
-
-        - name: Push to buildcache
-          run: |
-            spack -e . mirror set --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache
-            spack -e . buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache
-          if: ${{ !cancelled() }}
-
-The first time this action runs, it will build the packages from source and
-push them to the build cache. Subsequent runs will pull the binaries from the
-build cache. The concretizer will ensure that prebuilt binaries are favored
-over source builds.
-
-The build cache entries appear in the GitHub Packages section of your repository,
-and contain instructions for pulling and running them with ``docker`` or ``podman``.
-
-
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Using Spack's public build cache for GitHub Actions
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
 Spack offers a public build cache for GitHub Actions with a set of common packages,
 which lets you get started quickly. See the following resources for more information:
 
-* `spack/github-actions-buildcache <https://github.com/spack/github-actions-buildcache>`_
+* `spack/setup-spack <https://github.com/spack/setup-spack>`_ for setting up Spack in GitHub
+  Actions
+* `spack/github-actions-buildcache <https://github.com/spack/github-actions-buildcache>`_ for
+  more details on the public build cache
 
 .. _cmd-spack-buildcache:
 

lib/spack/docs/build_settings.rst

@@ -37,7 +37,11 @@ to enable reuse for a single installation, and you can use:
    spack install --fresh <spec>
 
 to do a fresh install if ``reuse`` is enabled by default.
-``reuse: true`` is the default.
+``reuse: dependencies`` is the default.
+
+.. seealso::
+
+   FAQ: :ref:`Why does Spack pick particular versions and variants? <faq-concretizer-precedence>`
 
 ------------------------------------------
 Selection of the target microarchitectures
@@ -99,547 +103,3 @@ while `py-numpy` still needs an older version:
 
 Up to Spack v0.20 ``duplicates:strategy:none`` was the default (and only) behavior. From Spack v0.21 the
 default behavior is ``duplicates:strategy:minimal``.
-
-.. _build-settings:
-
-================================
-Package Settings (packages.yaml)
-================================
-
-Spack allows you to customize how your software is built through the
-``packages.yaml`` file.  Using it, you can make Spack prefer particular
-implementations of virtual dependencies (e.g., MPI or BLAS/LAPACK),
-or you can make it prefer to build with particular compilers.  You can
-also tell Spack to use *external* software installations already
-present on your system.
-
-At a high level, the ``packages.yaml`` file is structured like this:
-
-.. code-block:: yaml
-
-   packages:
-     package1:
-       # settings for package1
-     package2:
-       # settings for package2
-     # ...
-     all:
-       # settings that apply to all packages.
-
-So you can either set build preferences specifically for *one* package,
-or you can specify that certain settings should apply to *all* packages.
-The types of settings you can customize are described in detail below.
-
-Spack's build defaults are in the default
-``etc/spack/defaults/packages.yaml`` file.  You can override them in
-``~/.spack/packages.yaml`` or ``etc/spack/packages.yaml``. For more
-details on how this works, see :ref:`configuration-scopes`.
-
-.. _sec-external-packages:
-
------------------
-External Packages
------------------
-
-Spack can be configured to use externally-installed
-packages rather than building its own packages. This may be desirable
-if machines ship with system packages, such as a customized MPI
-that should be used instead of Spack building its own MPI.
-
-External packages are configured through the ``packages.yaml`` file.
-Here's an example of an external configuration:
-
-.. code-block:: yaml
-
-   packages:
-     openmpi:
-       externals:
-       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64"
-         prefix: /opt/openmpi-1.4.3
-       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64+debug"
-         prefix: /opt/openmpi-1.4.3-debug
-       - spec: "openmpi@1.6.5%intel@10.1 arch=linux-debian7-x86_64"
-         prefix: /opt/openmpi-1.6.5-intel
-
-This example lists three installations of OpenMPI, one built with GCC,
-one built with GCC and debug information, and another built with Intel.
-If Spack is asked to build a package that uses one of these MPIs as a
-dependency, it will use the pre-installed OpenMPI in
-the given directory. Note that the specified path is the top-level
-install prefix, not the ``bin`` subdirectory.
-
-``packages.yaml`` can also be used to specify modules to load instead
-of the installation prefixes.  The following example says that module
-``CMake/3.7.2`` provides cmake version 3.7.2.
-
-.. code-block:: yaml
-
-   cmake:
-     externals:
-     - spec: cmake@3.7.2
-       modules:
-       - CMake/3.7.2
-
-Each ``packages.yaml`` begins with a ``packages:`` attribute, followed
-by a list of package names.  To specify externals, add an ``externals:``
-attribute under the package name, which lists externals.
-Each external should specify a ``spec:`` string that should be as
-well-defined as reasonably possible.  If a
-package lacks a spec component, such as missing a compiler or
-package version, then Spack will guess the missing component based
-on its most-favored packages, and it may guess incorrectly.
-
-Each package version and compiler listed in an external should
-have entries in Spack's packages and compiler configuration, even
-though the package and compiler may not ever be built.
-
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Prevent packages from being built from sources
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Adding an external spec in ``packages.yaml`` allows Spack to use an external location,
-but it does not prevent Spack from building packages from sources. In the above example,
-Spack might choose for many valid reasons to start building and linking with the
-latest version of OpenMPI rather than continue using the pre-installed OpenMPI versions.
-
-To prevent this, the ``packages.yaml`` configuration also allows packages
-to be flagged as non-buildable.  The previous example could be modified to
-be:
-
-.. code-block:: yaml
-
-   packages:
-     openmpi:
-       externals:
-       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64"
-         prefix: /opt/openmpi-1.4.3
-       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64+debug"
-         prefix: /opt/openmpi-1.4.3-debug
-       - spec: "openmpi@1.6.5%intel@10.1 arch=linux-debian7-x86_64"
-         prefix: /opt/openmpi-1.6.5-intel
-       buildable: False
-
-The addition of the ``buildable`` flag tells Spack that it should never build
-its own version of OpenMPI from sources, and it will instead always rely on a pre-built
-OpenMPI.
-
-.. note::
-
-   If ``concretizer:reuse`` is on (see :ref:`concretizer-options` for more information on that flag)
-   pre-built specs include specs already available from a local store, an upstream store, a registered
-   buildcache or specs marked as externals in ``packages.yaml``. If ``concretizer:reuse`` is off, only
-   external specs in ``packages.yaml`` are included in the list of pre-built specs.
-
-If an external module is specified as not buildable, then Spack will load the
-external module into the build environment which can be used for linking.
-
-The ``buildable`` does not need to be paired with external packages.
-It could also be used alone to forbid packages that may be
-buggy or otherwise undesirable.
-
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Non-buildable virtual packages
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Virtual packages in Spack can also be specified as not buildable, and
-external implementations can be provided. In the example above,
-OpenMPI is configured as not buildable, but Spack will often prefer
-other MPI implementations over the externally available OpenMPI. Spack
-can be configured with every MPI provider not buildable individually,
-but more conveniently:
-
-.. code-block:: yaml
-
-   packages:
-     mpi:
-       buildable: False
-     openmpi:
-       externals:
-       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64"
-         prefix: /opt/openmpi-1.4.3
-       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64+debug"
-         prefix: /opt/openmpi-1.4.3-debug
-       - spec: "openmpi@1.6.5%intel@10.1 arch=linux-debian7-x86_64"
-         prefix: /opt/openmpi-1.6.5-intel
-
-Spack can then use any of the listed external implementations of MPI
-to satisfy a dependency, and will choose depending on the compiler and
-architecture.
-
-In cases where the concretizer is configured to reuse specs, and other ``mpi`` providers
-(available via stores or buildcaches) are not wanted, Spack can be configured to require
-specs matching only the available externals:
-
-.. code-block:: yaml
-
-   packages:
-     mpi:
-       buildable: False
-       require:
-       - one_of: [
-           "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64",
-           "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64+debug",
-           "openmpi@1.6.5%intel@10.1 arch=linux-debian7-x86_64"
-         ]
-     openmpi:
-       externals:
-       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64"
-         prefix: /opt/openmpi-1.4.3
-       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64+debug"
-         prefix: /opt/openmpi-1.4.3-debug
-       - spec: "openmpi@1.6.5%intel@10.1 arch=linux-debian7-x86_64"
-         prefix: /opt/openmpi-1.6.5-intel
-
-This configuration prevents any spec using MPI and originating from stores or buildcaches to be reused,
-unless it matches the requirements under ``packages:mpi:require``. For more information on requirements see
-:ref:`package-requirements`.
-
-.. _cmd-spack-external-find:
-
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Automatically Find External Packages
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-You can run the :ref:`spack external find <spack-external-find>` command
-to search for system-provided packages and add them to ``packages.yaml``.
-After running this command your ``packages.yaml`` may include new entries:
-
-.. code-block:: yaml
-
-   packages:
-     cmake:
-       externals:
-       - spec: cmake@3.17.2
-         prefix: /usr
-
-Generally this is useful for detecting a small set of commonly-used packages;
-for now this is generally limited to finding build-only dependencies.
-Specific limitations include:
-
-* Packages are not discoverable by default: For a package to be
-  discoverable with ``spack external find``, it needs to add special
-  logic. See :ref:`here <make-package-findable>` for more details.
-* The logic does not search through module files, it can only detect
-  packages with executables defined in ``PATH``; you can help Spack locate
-  externals which use module files by loading any associated modules for
-  packages that you want Spack to know about before running
-  ``spack external find``.
-* Spack does not overwrite existing entries in the package configuration:
-  If there is an external defined for a spec at any configuration scope,
-  then Spack will not add a new external entry (``spack config blame packages``
-  can help locate all external entries).
-
-.. _package-requirements:
-
---------------------
-Package Requirements
---------------------
-
-Spack can be configured to always use certain compilers, package
-versions, and variants during concretization through package
-requirements.
-
-Package requirements are useful when you find yourself repeatedly
-specifying the same constraints on the command line, and wish that
-Spack respects these constraints whether you mention them explicitly
-or not. Another use case is specifying constraints that should apply
-to all root specs in an environment, without having to repeat the
-constraint everywhere.
-
-Apart from that, requirements config is more flexible than constraints
-on the command line, because it can specify constraints on packages
-*when they occur* as a dependency. In contrast, on the command line it
-is not possible to specify constraints on dependencies while also keeping
-those dependencies optional.
-
-^^^^^^^^^^^^^^^^^^^
-Requirements syntax
-^^^^^^^^^^^^^^^^^^^
-
-The package requirements configuration is specified in ``packages.yaml``,
-keyed by package name and expressed using the Spec syntax. In the simplest
-case you can specify attributes that you always want the package to have
-by providing a single spec string to ``require``:
-
-.. code-block:: yaml
-
-   packages:
-     libfabric:
-       require: "@1.13.2"
-
-In the above example, ``libfabric`` will always build with version 1.13.2. If you
-need to compose multiple configuration scopes ``require`` accepts a list of
-strings:
-
-.. code-block:: yaml
-
-   packages:
-     libfabric:
-       require:
-       - "@1.13.2"
-       - "%gcc"
-
-In this case ``libfabric`` will always build with version 1.13.2 **and** using GCC
-as a compiler.
-
-For more complex use cases, require accepts also a list of objects. These objects
-must have either a ``any_of`` or a ``one_of`` field, containing a list of spec strings,
-and they can optionally have a ``when`` and a ``message`` attribute:
-
-.. code-block:: yaml
-
-   packages:
-     openmpi:
-       require:
-       - any_of: ["@4.1.5", "%gcc"]
-         message: "in this example only 4.1.5 can build with other compilers"
-
-``any_of`` is a list of specs. One of those specs must be satisfied
-and it is also allowed for the concretized spec to match more than one.
-In the above example, that means you could build ``openmpi@4.1.5%gcc``,
-``openmpi@4.1.5%clang`` or ``openmpi@3.9%gcc``, but
-not ``openmpi@3.9%clang``.
-
-If a custom message is provided, and the requirement is not satisfiable,
-Spack will print the custom error message:
-
-.. code-block:: console
-
-   $ spack spec openmpi@3.9%clang
-   ==> Error: in this example only 4.1.5 can build with other compilers
-
-We could express a similar requirement using the ``when`` attribute:
-
-.. code-block:: yaml
-
-   packages:
-     openmpi:
-       require:
-       - any_of: ["%gcc"]
-         when: "@:4.1.4"
-         message: "in this example only 4.1.5 can build with other compilers"
-
-In the example above, if the version turns out to be 4.1.4 or less, we require the compiler to be GCC.
-For readability, Spack also allows a ``spec`` key accepting a string when there is only a single
-constraint:
-
-.. code-block:: yaml
-
-   packages:
-     openmpi:
-       require:
-       - spec: "%gcc"
-         when: "@:4.1.4"
-         message: "in this example only 4.1.5 can build with other compilers"
-
-This code snippet and the one before it are semantically equivalent.
-
-Finally, instead of ``any_of`` you can use ``one_of`` which also takes a list of specs. The final
-concretized spec must match one and only one of them:
-
-.. code-block:: yaml
-
-   packages:
-     mpich:
-       require:
-       - one_of: ["+cuda", "+rocm"]
-
-In the example above, that means you could build ``mpich+cuda`` or ``mpich+rocm`` but not ``mpich+cuda+rocm``.
-
-.. note::
-
-   For ``any_of`` and ``one_of``, the order of specs indicates a
-   preference: items that appear earlier in the list are preferred
-   (note that these preferences can be ignored in favor of others).
-
-.. note::
-
-   When using a conditional requirement, Spack is allowed to actively avoid the triggering
-   condition (the ``when=...`` spec) if that leads to a concrete spec with better scores in
-   the optimization criteria. To check the current optimization criteria and their
-   priorities you can run ``spack solve zlib``.
-
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Setting default requirements
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-You can also set default requirements for all packages under ``all``
-like this:
-
-.. code-block:: yaml
-
-   packages:
-     all:
-       require: '%clang'
-
-which means every spec will be required to use ``clang`` as a compiler.
-
-Note that in this case ``all`` represents a *default set of requirements* -
-if there are specific package requirements, then the default requirements
-under ``all`` are disregarded. For example, with a configuration like this:
-
-.. code-block:: yaml
-
-   packages:
-     all:
-       require: '%clang'
-     cmake:
-       require: '%gcc'
-
-Spack requires ``cmake`` to use ``gcc`` and all other nodes (including ``cmake``
-dependencies) to use ``clang``.
-
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Setting requirements on virtual specs
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-A requirement on a virtual spec applies whenever that virtual is present in the DAG.
-This can be useful for fixing which virtual provider you want to use:
-
-.. code-block:: yaml
-
-   packages:
-     mpi:
-       require: 'mvapich2 %gcc'
-
-With the configuration above the only allowed ``mpi`` provider is ``mvapich2 %gcc``.
-
-Requirements on the virtual spec and on the specific provider are both applied, if
-present. For instance with a configuration like:
-
-.. code-block:: yaml
-
-   packages:
-     mpi:
-       require: 'mvapich2 %gcc'
-     mvapich2:
-       require: '~cuda'
-
-you will use ``mvapich2~cuda %gcc`` as an ``mpi`` provider.
-
-.. _package-preferences:
-
--------------------
-Package Preferences
--------------------
-
-In some cases package requirements can be too strong, and package
-preferences are the better option. Package preferences do not impose
-constraints on packages for particular versions or variants values,
-they rather only set defaults. The concretizer is free to change
-them if it must, due to other constraints, and also prefers reusing
-installed packages over building new ones that are a better match for
-preferences.
-
-Most package preferences (``compilers``, ``target`` and ``providers``)
-can only be set globally under the ``all`` section of ``packages.yaml``:
-
-.. code-block:: yaml
-
-   packages:
-     all:
-       compiler: [gcc@12.2.0, clang@12:, oneapi@2023:]
-       target: [x86_64_v3]
-       providers:
-         mpi: [mvapich2, mpich, openmpi]
-
-These preferences override Spack's default and effectively reorder priorities
-when looking for the best compiler, target or virtual package provider. Each
-preference takes an ordered list of spec constraints, with earlier entries in
-the list being preferred over later entries.
-
-In the example above all packages prefer to be compiled with ``gcc@12.2.0``,
-to target the ``x86_64_v3`` microarchitecture and to use ``mvapich2`` if they
-depend on ``mpi``.
-
-The ``variants`` and ``version`` preferences can be set under
-package specific sections of the ``packages.yaml`` file:
-
-.. code-block:: yaml
-
-   packages:
-     opencv:
-       variants: +debug
-     gperftools:
-       version: [2.2, 2.4, 2.3]
-
-In this case, the preference for ``opencv`` is to build with debug options, while
-``gperftools`` prefers version 2.2 over 2.4.
-
-Any preference can be overwritten on the command line if explicitly requested.
-
-Preferences cannot overcome explicit constraints, as they only set a preferred
-ordering among homogeneous attribute values. Going back to the example, if
-``gperftools@2.3:`` was requested, then Spack will install version 2.4
-since the most preferred version 2.2 is prohibited by the version constraint.
-
-.. _package_permissions:
-
--------------------
-Package Permissions
--------------------
-
-Spack can be configured to assign permissions to the files installed
-by a package.
-
-In the ``packages.yaml`` file under ``permissions``, the attributes
-``read``, ``write``, and ``group`` control the package
-permissions. These attributes can be set per-package, or for all
-packages under ``all``. If permissions are set under ``all`` and for a
-specific package, the package-specific settings take precedence.
-
-The ``read`` and ``write`` attributes take one of ``user``, ``group``,
-and ``world``.
-
-.. code-block:: yaml
-
-  packages:
-    all:
-      permissions:
-        write: group
-        group: spack
-    my_app:
-      permissions:
-        read: group
-        group: my_team
-
-The permissions settings describe the broadest level of access to
-installations of the specified packages. The execute permissions of
-the file are set to the same level as read permissions for those files
-that are executable. The default setting for ``read`` is ``world``,
-and for ``write`` is ``user``. In the example above, installations of
-``my_app`` will be installed with user and group permissions but no
-world permissions, and owned by the group ``my_team``. All other
-packages will be installed with user and group write privileges, and
-world read privileges. Those packages will be owned by the group
-``spack``.
-
-The ``group`` attribute assigns a Unix-style group to a package. All
-files installed by the package will be owned by the assigned group,
-and the sticky group bit will be set on the install prefix and all
-directories inside the install prefix. This will ensure that even
-manually placed files within the install prefix are owned by the
-assigned group. If no group is assigned, Spack will allow the OS
-default behavior to go as expected.
-
-----------------------------
-Assigning Package Attributes
-----------------------------
-
-You can assign class-level attributes in the configuration:
-
-.. code-block:: yaml
-
-  packages:
-    mpileaks:
-      # Override existing attributes
-      url: http://www.somewhereelse.com/mpileaks-1.0.tar.gz
-      # ... or add new ones
-      x: 1
-
-Attributes set this way will be accessible to any method executed
-in the package.py file (e.g. the ``install()`` method). Values for these
-attributes may be any value parseable by yaml.
-
-These can only be applied to specific packages, not "all" or
-virtual packages.

lib/spack/docs/build_systems/cmakepackage.rst

@@ -82,7 +82,7 @@ class already contains:
 
 .. code-block:: python
 
-   depends_on('cmake', type='build')
+   depends_on("cmake", type="build")
 
 
 If you need to specify a particular version requirement, you can
@@ -90,7 +90,7 @@ override this in your package:
 
 .. code-block:: python
 
-   depends_on('cmake@2.8.12:', type='build')
+   depends_on("cmake@2.8.12:", type="build")
 
 
 ^^^^^^^^^^^^^^^^^^^
@@ -137,10 +137,10 @@ and without the :meth:`~spack.build_systems.cmake.CMakeBuilder.define` and
 
    def cmake_args(self):
        args = [
-           '-DWHATEVER:STRING=somevalue',
-           self.define('ENABLE_BROKEN_FEATURE', False),
-           self.define_from_variant('DETECT_HDF5', 'hdf5'),
-           self.define_from_variant('THREADS'), # True if +threads
+           "-DWHATEVER:STRING=somevalue",
+           self.define("ENABLE_BROKEN_FEATURE", False),
+           self.define_from_variant("DETECT_HDF5", "hdf5"),
+           self.define_from_variant("THREADS"), # True if +threads
        ]
 
        return args
@@ -151,10 +151,10 @@ and CMake simply ignores the empty command line argument. For example the follow
 
 .. code-block:: python
 
-   variant('example', default=True, when='@2.0:')
+   variant("example", default=True, when="@2.0:")
 
    def cmake_args(self):
-      return [self.define_from_variant('EXAMPLE', 'example')]
+      return [self.define_from_variant("EXAMPLE", "example")]
 
 will generate ``'cmake' '-DEXAMPLE=ON' ...`` when `@2.0: +example` is met, but will
 result in ``'cmake' '' ...`` when the spec version is below ``2.0``.
@@ -193,9 +193,9 @@ a variant to control this:
 
 .. code-block:: python
 
-   variant('build_type', default='RelWithDebInfo',
-           description='CMake build type',
-           values=('Debug', 'Release', 'RelWithDebInfo', 'MinSizeRel'))
+   variant("build_type", default="RelWithDebInfo",
+           description="CMake build type",
+           values=("Debug", "Release", "RelWithDebInfo", "MinSizeRel"))
 
 However, not every CMake package accepts all four of these options.
 Grep the ``CMakeLists.txt`` file to see if the default values are
@@ -205,9 +205,9 @@ package overrides the default variant with:
 
 .. code-block:: python
 
-   variant('build_type', default='DebugRelease',
-           description='The build type to build',
-           values=('Debug', 'Release', 'DebugRelease'))
+   variant("build_type", default="DebugRelease",
+           description="The build type to build",
+           values=("Debug", "Release", "DebugRelease"))
 
 For more information on ``CMAKE_BUILD_TYPE``, see:
 https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html
@@ -250,7 +250,7 @@ generator is Ninja. To switch to the Ninja generator, simply add:
 
 .. code-block:: python
 
-   generator = 'Ninja'
+   generator = "Ninja"
 
 
 ``CMakePackage`` defaults to "Unix Makefiles". If you switch to the
@@ -258,7 +258,7 @@ Ninja generator, make sure to add:
 
 .. code-block:: python
 
-   depends_on('ninja', type='build')
+   depends_on("ninja", type="build")
 
 to the package as well. Aside from that, you shouldn't need to do
 anything else. Spack will automatically detect that you are using
@@ -288,7 +288,7 @@ like so:
 
 .. code-block:: python
 
-   root_cmakelists_dir = 'src'
+   root_cmakelists_dir = "src"
 
 
 Note that this path is relative to the root of the extracted tarball,
@@ -304,7 +304,7 @@ different sub-directory, simply override ``build_directory`` like so:
 
 .. code-block:: python
 
-   build_directory = 'my-build'
+   build_directory = "my-build"
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 Build and install targets
@@ -324,8 +324,8 @@ library or build the documentation, you can add these like so:
 
 .. code-block:: python
 
-   build_targets = ['all', 'docs']
-   install_targets = ['install', 'docs']
+   build_targets = ["all", "docs"]
+   install_targets = ["install", "docs"]
 
 ^^^^^^^
 Testing

lib/spack/docs/build_systems/inteloneapipackage.rst

@@ -53,18 +53,24 @@ Install the oneAPI compilers::
 
 Add the compilers to your ``compilers.yaml`` so spack can use them::
 
-  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64
-  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/bin
 
 Verify that the compilers are available::
 
   spack compiler list
 
+Note that 2024 and later releases do not include ``icc``. Before 2024,
+the package layout was different::
+  
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin
+
 The ``intel-oneapi-compilers`` package includes 2 families of
 compilers:
 
 * ``intel``: ``icc``, ``icpc``, ``ifort``. Intel's *classic*
-  compilers.
+  compilers. 2024 and later releases contain ``ifort``, but not
+  ``icc`` and ``icpc``.
 * ``oneapi``: ``icx``, ``icpx``, ``ifx``. Intel's new generation of
   compilers based on LLVM.
 
@@ -89,8 +95,8 @@ Install the oneAPI compilers::
 
 Add the compilers to your ``compilers.yaml`` so Spack can use them::
 
-  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64
-  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/bin
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/bin
 
 Verify that the compilers are available::
 
@@ -146,8 +152,7 @@ Compilers
 To use the compilers, add some information about the installation to
 ``compilers.yaml``. For most users, it is sufficient to do::
 
-  spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin/intel64
-  spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin
+  spack compiler add /opt/intel/oneapi/compiler/latest/bin
 
 Adapt the paths above if you did not install the tools in the default
 location. After adding the compilers, using them is the same
@@ -156,6 +161,12 @@ Another option is to manually add the configuration to
 ``compilers.yaml`` as described in :ref:`Compiler configuration
 <compiler-config>`.
 
+Before 2024, the directory structure was different::
+  
+  spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin/intel64
+  spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin
+
+
 Libraries
 ---------
 

lib/spack/docs/build_systems/intelpackage.rst

@@ -90,7 +90,7 @@ and optimizers do require a paid license.  In Spack, they are packaged as:
     TODO: Confirm and possible change(!) the scope of MPI components (runtime
     vs. devel) in current (and previous?) *cluster/professional/composer*
     editions, i.e., presence in downloads, possibly subject to license
-    coverage(!); see `disussion in PR #4300
+    coverage(!); see `discussion in PR #4300
     <https://github.com/spack/spack/pull/4300#issuecomment-305582898>`_.  [NB:
     An "mpi" subdirectory is not indicative of the full MPI SDK being present
     (i.e., ``mpicc``, ..., and header files).  The directory may just as well
@@ -392,7 +392,7 @@ See section
 :ref:`Configuration Scopes <configuration-scopes>`
 for an explanation about the different files
 and section
-:ref:`Build customization <build-settings>`
+:ref:`Build customization <packages-config>`
 for specifics and examples for ``packages.yaml`` files.
 
 .. If your system administrator did not provide modules for pre-installed Intel
@@ -934,9 +934,9 @@ a *virtual* ``mkl`` package is declared in Spack.
   .. code-block:: python
 
      # Examples for absolute and conditional dependencies:
-     depends_on('mkl')
-     depends_on('mkl', when='+mkl')
-     depends_on('mkl', when='fftw=mkl')
+     depends_on("mkl")
+     depends_on("mkl", when="+mkl")
+     depends_on("mkl", when="fftw=mkl")
 
   The ``MKLROOT`` environment variable (part of the documented API) will be set
   during all stages of client package installation, and is available to both
@@ -972,8 +972,8 @@ a *virtual* ``mkl`` package is declared in Spack.
       def configure_args(self):
           args = []
           ...
-          args.append('--with-blas=%s' % self.spec['blas'].libs.ld_flags)
-          args.append('--with-lapack=%s' % self.spec['lapack'].libs.ld_flags)
+          args.append("--with-blas=%s" % self.spec["blas"].libs.ld_flags)
+          args.append("--with-lapack=%s" % self.spec["lapack"].libs.ld_flags)
           ...
 
   .. tip::
@@ -989,13 +989,13 @@ a *virtual* ``mkl`` package is declared in Spack.
 
   .. code-block:: python
 
-    self.spec['blas'].headers.include_flags
+    self.spec["blas"].headers.include_flags
 
   and to generate linker options (``-L<dir> -llibname ...``), use the same as above,
 
   .. code-block:: python
 
-    self.spec['blas'].libs.ld_flags
+    self.spec["blas"].libs.ld_flags
 
   See
   :ref:`MakefilePackage <makefilepackage>`

lib/spack/docs/build_systems/luapackage.rst

@@ -88,7 +88,7 @@ override the ``luarocks_args`` method like so:
 .. code-block:: python
 
     def luarocks_args(self):
-        return ['flag1', 'flag2']
+        return ["flag1", "flag2"]
 
 One common use of this is to override warnings or flags for newer compilers, as in:
 

lib/spack/docs/build_systems/mavenpackage.rst

@@ -48,8 +48,8 @@ class automatically adds the following dependencies:
 
 .. code-block:: python
 
-   depends_on('java', type=('build', 'run'))
-   depends_on('maven', type='build')
+   depends_on("java", type=("build", "run"))
+   depends_on("maven", type="build")
 
 
 In the ``pom.xml`` file, you may see sections like:
@@ -72,8 +72,8 @@ should add:
 
 .. code-block:: python
 
-   depends_on('java@7:', type='build')
-   depends_on('maven@3.5.4:', type='build')
+   depends_on("java@7:", type="build")
+   depends_on("maven@3.5.4:", type="build")
 
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -88,9 +88,9 @@ the build phase. For example:
 
    def build_args(self):
        return [
-           '-Pdist,native',
-           '-Dtar',
-           '-Dmaven.javadoc.skip=true'
+           "-Pdist,native",
+           "-Dtar",
+           "-Dmaven.javadoc.skip=true"
        ]
 
 

lib/spack/docs/build_systems/mesonpackage.rst

@@ -86,8 +86,8 @@ the ``MesonPackage`` base class already contains:
 
 .. code-block:: python
 
-   depends_on('meson', type='build')
-   depends_on('ninja', type='build')
+   depends_on("meson", type="build")
+   depends_on("ninja", type="build")
 
 
 If you need to specify a particular version requirement, you can
@@ -95,8 +95,8 @@ override this in your package:
 
 .. code-block:: python
 
-   depends_on('meson@0.43.0:', type='build')
-   depends_on('ninja', type='build')
+   depends_on("meson@0.43.0:", type="build")
+   depends_on("ninja", type="build")
 
 
 ^^^^^^^^^^^^^^^^^^^
@@ -121,7 +121,7 @@ override the ``meson_args`` method like so:
 .. code-block:: python
 
    def meson_args(self):
-       return ['--warnlevel=3']
+       return ["--warnlevel=3"]
 
 
 This method can be used to pass flags as well as variables.

lib/spack/docs/build_systems/perlpackage.rst

@@ -118,7 +118,7 @@ so ``PerlPackage`` contains:
 
 .. code-block:: python
 
-   extends('perl')
+   extends("perl")
 
 
 If your package requires a specific version of Perl, you should
@@ -132,14 +132,14 @@ properly. If your package uses ``Makefile.PL`` to build, add:
 
 .. code-block:: python
 
-   depends_on('perl-extutils-makemaker', type='build')
+   depends_on("perl-extutils-makemaker", type="build")
 
 
 If your package uses ``Build.PL`` to build, add:
 
 .. code-block:: python
 
-   depends_on('perl-module-build', type='build')
+   depends_on("perl-module-build", type="build")
 
 
 ^^^^^^^^^^^^^^^^^
@@ -165,11 +165,11 @@ arguments to ``Makefile.PL`` or ``Build.PL`` by overriding
 .. code-block:: python
 
    def configure_args(self):
-       expat = self.spec['expat'].prefix
+       expat = self.spec["expat"].prefix
 
        return [
-           'EXPATLIBPATH={0}'.format(expat.lib),
-           'EXPATINCPATH={0}'.format(expat.include),
+           "EXPATLIBPATH={0}".format(expat.lib),
+           "EXPATINCPATH={0}".format(expat.include),
        ]
 
 

lib/spack/docs/build_systems/qmakepackage.rst

@@ -83,7 +83,7 @@ base class already contains:
 
 .. code-block:: python
 
-   depends_on('qt', type='build')
+   depends_on("qt", type="build")
 
 
 If you want to specify a particular version requirement, or need to
@@ -91,7 +91,7 @@ link to the ``qt`` libraries, you can override this in your package:
 
 .. code-block:: python
 
-   depends_on('qt@5.6.0:')
+   depends_on("qt@5.6.0:")
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 Passing arguments to qmake
@@ -103,7 +103,7 @@ override the ``qmake_args`` method like so:
 .. code-block:: python
 
    def qmake_args(self):
-       return ['-recursive']
+       return ["-recursive"]
 
 
 This method can be used to pass flags as well as variables.
@@ -118,7 +118,7 @@ sub-directory by adding the following to the package:
 
 .. code-block:: python
 
-   build_directory = 'src'
+   build_directory = "src"
 
 
 ^^^^^^^^^^^^^^^^^^^^^^

lib/spack/docs/build_systems/rpackage.rst

@@ -163,28 +163,28 @@ attributes that can be used to set ``homepage``, ``url``, ``list_url``, and
 
 .. code-block:: python
 
-   cran = 'caret'
+   cran = "caret"
 
 is equivalent to:
 
 .. code-block:: python
 
-   homepage = 'https://cloud.r-project.org/package=caret'
-   url      = 'https://cloud.r-project.org/src/contrib/caret_6.0-86.tar.gz'
-   list_url = 'https://cloud.r-project.org/src/contrib/Archive/caret'
+   homepage = "https://cloud.r-project.org/package=caret"
+   url      = "https://cloud.r-project.org/src/contrib/caret_6.0-86.tar.gz"
+   list_url = "https://cloud.r-project.org/src/contrib/Archive/caret"
 
 Likewise, the following ``bioc`` attribute:
 
 .. code-block:: python
 
-   bioc = 'BiocVersion'
+   bioc = "BiocVersion"
 
 is equivalent to:
 
 .. code-block:: python
 
-   homepage = 'https://bioconductor.org/packages/BiocVersion/'
-   git      = 'https://git.bioconductor.org/packages/BiocVersion'
+   homepage = "https://bioconductor.org/packages/BiocVersion/"
+   git      = "https://git.bioconductor.org/packages/BiocVersion"
 
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -200,7 +200,7 @@ base class contains:
 
 .. code-block:: python
 
-   extends('r')
+   extends("r")
 
 
 Take a close look at the homepage for ``caret``. If you look at the
@@ -209,7 +209,7 @@ You should add this to your package like so:
 
 .. code-block:: python
 
-   depends_on('r@3.2.0:', type=('build', 'run'))
+   depends_on("r@3.2.0:", type=("build", "run"))
 
 
 ^^^^^^^^^^^^^^
@@ -227,7 +227,7 @@ and list all of their dependencies in the following sections:
 * LinkingTo
 
 As far as Spack is concerned, all 3 of these dependency types
-correspond to ``type=('build', 'run')``, so you don't have to worry
+correspond to ``type=("build", "run")``, so you don't have to worry
 about the details. If you are curious what they mean,
 https://github.com/spack/spack/issues/2951 has a pretty good summary:
 
@@ -330,7 +330,7 @@ the dependency:
 
 .. code-block:: python
 
-   depends_on('r-lattice@0.20:', type=('build', 'run'))
+   depends_on("r-lattice@0.20:", type=("build", "run"))
 
 
 ^^^^^^^^^^^^^^^^^^
@@ -361,20 +361,20 @@ like so:
 .. code-block:: python
 
    def configure_args(self):
-       mpi_name = self.spec['mpi'].name
+       mpi_name = self.spec["mpi"].name
 
        # The type of MPI. Supported values are:
        # OPENMPI, LAM, MPICH, MPICH2, or CRAY
-       if mpi_name == 'openmpi':
-           Rmpi_type = 'OPENMPI'
-       elif mpi_name == 'mpich':
-           Rmpi_type = 'MPICH2'
+       if mpi_name == "openmpi":
+           Rmpi_type = "OPENMPI"
+       elif mpi_name == "mpich":
+           Rmpi_type = "MPICH2"
        else:
-           raise InstallError('Unsupported MPI type')
+           raise InstallError("Unsupported MPI type")
 
        return [
-           '--with-Rmpi-type={0}'.format(Rmpi_type),
-           '--with-mpi={0}'.format(spec['mpi'].prefix),
+           "--with-Rmpi-type={0}".format(Rmpi_type),
+           "--with-mpi={0}".format(spec["mpi"].prefix),
        ]
 
 

lib/spack/docs/build_systems/rubypackage.rst

@@ -84,8 +84,8 @@ The ``*.gemspec`` file may contain something like:
 
 .. code-block:: ruby
 
-   summary = 'An implementation of the AsciiDoc text processor and publishing toolchain'
-   description = 'A fast, open source text processor and publishing toolchain for converting AsciiDoc content to HTML 5, DocBook 5, and other formats.'
+   summary = "An implementation of the AsciiDoc text processor and publishing toolchain"
+   description = "A fast, open source text processor and publishing toolchain for converting AsciiDoc content to HTML 5, DocBook 5, and other formats."
 
 
 Either of these can be used for the description of the Spack package.
@@ -98,7 +98,7 @@ The ``*.gemspec`` file may contain something like:
 
 .. code-block:: ruby
 
-   homepage = 'https://asciidoctor.org'
+   homepage = "https://asciidoctor.org"
 
 
 This should be used as the official homepage of the Spack package.
@@ -112,21 +112,21 @@ the base class contains:
 
 .. code-block:: python
 
-   extends('ruby')
+   extends("ruby")
 
 
 The ``*.gemspec`` file may contain something like:
 
 .. code-block:: ruby
 
-   required_ruby_version = '>= 2.3.0'
+   required_ruby_version = ">= 2.3.0"
 
 
 This can be added to the Spack package using:
 
 .. code-block:: python
 
-   depends_on('ruby@2.3.0:', type=('build', 'run'))
+   depends_on("ruby@2.3.0:", type=("build", "run"))
 
 
 ^^^^^^^^^^^^^^^^^

lib/spack/docs/build_systems/sippackage.rst

@@ -124,7 +124,7 @@ are wrong, you can provide the names yourself by overriding
 
 .. code-block:: python
 
-   import_modules = ['PyQt5']
+   import_modules = ["PyQt5"]
 
 
 These tests often catch missing dependencies and non-RPATHed

lib/spack/docs/build_systems/wafpackage.rst

@@ -63,8 +63,8 @@ run package-specific unit tests.
 .. code-block:: python
 
    def installtest(self):
-       with working_dir('test'):
-           pytest = which('py.test')
+       with working_dir("test"):
+           pytest = which("py.test")
            pytest()
 
 
@@ -93,7 +93,7 @@ the following dependency automatically:
 
 .. code-block:: python
 
-   depends_on('python@2.5:', type='build')
+   depends_on("python@2.5:", type="build")
 
 
 Waf only supports Python 2.5 and up.
@@ -113,7 +113,7 @@ phase, you can use:
        args = []
 
        if self.run_tests:
-           args.append('--test')
+           args.append("--test")
 
        return args
 

lib/spack/docs/configuration.rst

@@ -17,7 +17,7 @@ case you want to skip directly to specific docs:
 * :ref:`config.yaml <config-yaml>`
 * :ref:`mirrors.yaml <mirrors>`
 * :ref:`modules.yaml <modules>`
-* :ref:`packages.yaml <build-settings>`
+* :ref:`packages.yaml <packages-config>`
 * :ref:`repos.yaml <repositories>`
 
 You can also add any of these as inline configuration in the YAML
@@ -243,9 +243,11 @@ lower-precedence settings. Completely ignoring higher-level configuration
 options is supported with the ``::`` notation for keys (see
 :ref:`config-overrides` below).
 
-There are also special notations for string concatenation and precendense override.
-Using the ``+:`` notation  can be used to force *prepending* strings or lists. For lists, this is identical
-to the default behavior. Using the ``-:`` works similarly, but for *appending* values.
+There are also special notations for string concatenation and precendense override:
+
+* ``+:`` will force *prepending* strings or lists. For lists, this is the default behavior.
+* ``-:`` works similarly, but for *appending* values.
+
 :ref:`config-prepend-append`
 
 ^^^^^^^^^^^

lib/spack/docs/containers.rst

@@ -9,24 +9,96 @@
 Container Images
 ================
 
-Spack :ref:`environments` are a great tool to create container images, but
-preparing one that is suitable for production requires some more boilerplate
-than just:
+Spack :ref:`environments` can easily be turned into container images. This page
+outlines two ways in which this can be done:
+
+1. By installing the environment on the host system, and copying the installations
+   into the container image. This approach does not require any tools like Docker
+   or Singularity to be installed.
+2. By generating a Docker or Singularity recipe that can be used to build the
+   container image. In this approach, Spack builds the software inside the
+   container runtime, not on the host system.
+
+The first approach is easiest if you already have an installed environment,
+the second approach gives more control over the container image.
+
+---------------------------
+From existing installations
+---------------------------
+
+If you already have a Spack environment installed on your system, you can
+share the binaries as an OCI compatible container image. To get started you
+just have to configure and OCI registry and run ``spack buildcache push``.
+
+.. code-block:: console
+  
+   # Create and install an environment in the current directory
+   spack env create -d .
+   spack -e . add pkg-a pkg-b
+   spack -e . install
+
+   # Configure the registry
+   spack -e . mirror add --oci-username ... --oci-password ... container-registry oci://example.com/name/image
+
+   # Push the image
+   spack -e . buildcache push --update-index --base-image ubuntu:22.04 --tag my_env container-registry
+
+The resulting container image can then be run as follows:
+
+.. code-block:: console
+
+   $ docker run -it example.com/name/image:my_env
+
+The image generated by Spack consists of the specified base image with each package from the
+environment as a separate layer on top. The image is minimal by construction, it only contains the
+environment roots and its runtime dependencies.
+
+.. note::
+
+  When using registries like GHCR and Docker Hub, the ``--oci-password`` flag is not
+  the password for your account, but a personal access token you need to generate separately.
+
+The specified ``--base-image`` should have a libc that is compatible with the host system.
+For example if your host system is Ubuntu 20.04, you can use ``ubuntu:20.04``, ``ubuntu:22.04``
+or newer: the libc in the container image must be at least the version of the host system,
+assuming ABI compatibility. It is also perfectly fine to use a completely different
+Linux distribution as long as the libc is compatible.
+
+For convenience, Spack also turns the OCI registry into a :ref:`build cache <binary_caches_oci>`,
+so that future ``spack install`` of the environment will simply pull the binaries from the
+registry instead of doing source builds. The flag ``--update-index`` is needed to make Spack
+take the build cache into account when concretizing.
+
+.. note::
+
+  When generating container images in CI, the approach above is recommended when CI jobs
+  already run in a sandboxed environment. You can simply use ``spack`` directly
+  in the CI job and push the resulting image to a registry. Subsequent CI jobs should
+  run faster because Spack can install from the same registry instead of rebuilding from
+  sources.
+
+---------------------------------------------
+Generating recipes for Docker and Singularity
+---------------------------------------------
+
+Apart from copying existing installations into container images, Spack can also
+generate recipes for container images. This is useful if you want to run Spack
+itself in a sandboxed environment instead of on the host system.
+
+Since recipes need a little bit more boilerplate than
 
 .. code-block:: docker
 
    COPY spack.yaml /environment
    RUN spack -e /environment install
 
-Additional actions may be needed to minimize the size of the
-container, or to update the system software that is installed in the base
-image, or to set up a proper entrypoint to run the image. These tasks are
-usually both necessary and repetitive, so Spack comes with a command
-to generate recipes for container images starting from a ``spack.yaml``.
+Spack provides a command to generate customizable recipes for container images. Customizations
+include minimizing the size of the image, installing packages in the base image using the system
+package manager, and setting up a proper entrypoint to run the image.
 
---------------------
+~~~~~~~~~~~~~~~~~~~~
 A Quick Introduction
---------------------
+~~~~~~~~~~~~~~~~~~~~
 
 Consider having a Spack environment like the following:
 
@@ -37,8 +109,8 @@ Consider having a Spack environment like the following:
      - gromacs+mpi
      - mpich
 
-Producing a ``Dockerfile`` from it is as simple as moving to the directory
-where the ``spack.yaml`` file is stored and giving the following command:
+Producing a ``Dockerfile`` from it is as simple as changing directories to
+where the ``spack.yaml`` file is stored and running the following command:
 
 .. code-block:: console
 
@@ -104,9 +176,9 @@ configuration are discussed in details in the sections below.
 
 .. _container_spack_images:
 
---------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 Spack Images on Docker Hub
---------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Docker images with Spack preinstalled and ready to be used are
 built when a release is tagged, or nightly on ``develop``. The images
@@ -176,9 +248,9 @@ by Spack use them as default base images for their ``build`` stage,
 even though handles to use custom base images provided by users are
 available to accommodate complex use cases.
 
----------------------------------
-Creating Images From Environments
----------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Configuring the Container Recipe
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Any Spack Environment can be used for the automatic generation of container
 recipes. Sensible defaults are provided for things like the base image or the
@@ -219,18 +291,18 @@ under the ``container`` attribute of environments:
 
 A detailed description of the options available can be found in the :ref:`container_config_options` section.
 
--------------------
+~~~~~~~~~~~~~~~~~~~
 Setting Base Images
--------------------
+~~~~~~~~~~~~~~~~~~~
 
 The ``images`` subsection is used to select both the image where
 Spack builds the software and the image where the built software
 is installed. This attribute can be set in different ways and
 which one to use depends on the use case at hand.
 
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+""""""""""""""""""""""""""""""""""""""""
 Use Official Spack Images From Dockerhub
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+""""""""""""""""""""""""""""""""""""""""
 
 To generate a recipe that uses an official Docker image from the
 Spack organization to build the software and the corresponding official OS image
@@ -435,9 +507,9 @@ responsibility to ensure that:
 Therefore we don't recommend its use in cases that can be otherwise
 covered by the simplified mode shown first.
 
-----------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Singularity Definition Files
-----------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In addition to producing recipes in ``Dockerfile`` format Spack can produce
 Singularity Definition Files by just changing the value of the ``format``
@@ -458,9 +530,9 @@ attribute:
 The minimum version of Singularity required to build a SIF (Singularity Image Format)
 image from the recipes generated by Spack is ``3.5.3``.
 
-------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Extending the Jinja2 Templates
-------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The Dockerfile and the Singularity definition file that Spack can generate are based on
 a few Jinja2 templates that are rendered according to the environment being containerized.
@@ -581,9 +653,9 @@ The recipe that gets generated contains the two extra instruction that we added
 
 .. _container_config_options:
 
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 Configuration Reference
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 
 The tables below describe all the configuration options that are currently supported
 to customize the generation of container recipes:
@@ -680,13 +752,13 @@ to customize the generation of container recipes:
      - Description string
      - No
 
---------------
+~~~~~~~~~~~~~~
 Best Practices
---------------
+~~~~~~~~~~~~~~
 
-^^^
+"""
 MPI
-^^^
+"""
 Due to the dependency on Fortran for OpenMPI, which is the spack default
 implementation, consider adding ``gfortran`` to the ``apt-get install`` list.
 
@@ -697,9 +769,9 @@ For execution on HPC clusters, it can be helpful to import the docker
 image into Singularity in order to start a program with an *external*
 MPI. Otherwise, also add ``openssh-server`` to the ``apt-get install`` list.
 
-^^^^
+""""
 CUDA
-^^^^
+""""
 Starting from CUDA 9.0, Nvidia provides minimal CUDA images based on
 Ubuntu. Please see `their instructions <https://hub.docker.com/r/nvidia/cuda/>`_.
 Avoid double-installing CUDA by adding, e.g.
@@ -718,9 +790,9 @@ to your ``spack.yaml``.
 Users will either need ``nvidia-docker`` or e.g. Singularity to *execute*
 device kernels.
 
-^^^^^^^^^^^^^^^^^^^^^^^^^
+"""""""""""""""""""""""""
 Docker on Windows and OSX
-^^^^^^^^^^^^^^^^^^^^^^^^^
+"""""""""""""""""""""""""
 
 On Mac OS and Windows, docker runs on a hypervisor that is not allocated much
 memory by default, and some spack packages may fail to build due to lack of

lib/spack/docs/extensions.rst

@@ -9,46 +9,42 @@
 Custom Extensions
 =================
 
-*Spack extensions* permit you to extend Spack capabilities by deploying your
+*Spack extensions* allow you to extend Spack capabilities by deploying your
 own custom commands or logic in an arbitrary location on your filesystem.
 This might be extremely useful e.g. to develop and maintain a command whose purpose is
 too specific to be considered for reintegration into the mainline or to
 evolve a command through its early stages before starting a discussion to merge
 it upstream.
+
 From Spack's point of view an extension is any path in your filesystem which
-respects a prescribed naming and layout for files:
+respects the following naming and layout for files:
 
 .. code-block:: console
 
   spack-scripting/ # The top level directory must match the format 'spack-{extension_name}'
   ├── pytest.ini # Optional file if the extension ships its own tests
   ├── scripting # Folder that may contain modules that are needed for the extension commands
-  │   └── cmd # Folder containing extension commands
-  │       └── filter.py # A new command that will be available
-  ├── tests # Tests for this extension
+  │   ├── cmd # Folder containing extension commands
+  │   │   └── filter.py # A new command that will be available
+  │   └── functions.py # Module with internal details
+  └── tests # Tests for this extension
   │   ├── conftest.py
   │   └── test_filter.py
   └── templates # Templates that may be needed by the extension
 
-In the example above the extension named *scripting* adds an additional command (``filter``)
-and unit tests to verify its behavior. The code for this example can be
-obtained by cloning the corresponding git repository:
+In the example above, the extension is named *scripting*. It adds an additional command
+(``spack filter``) and unit tests to verify its behavior.
 
-.. TODO: write an ad-hoc "hello world" extension and make it part of the spack organization
+The extension can import any core Spack module in its implementation. When loaded by
+the ``spack`` command, the extension itself is imported as a Python package in the
+``spack.extensions`` namespace. In the example above, since the extension is named
+"scripting", the corresponding Python module is ``spack.extensions.scripting``.
+
+The code for this example extension can be obtained by cloning the corresponding git repository:
 
 .. code-block:: console
 
-   $ cd ~/
-   $ mkdir tmp && cd tmp
-   $ git clone https://github.com/alalazo/spack-scripting.git
-   Cloning into 'spack-scripting'...
-   remote: Counting objects: 11, done.
-   remote: Compressing objects: 100% (7/7), done.
-   remote: Total 11 (delta 0), reused 11 (delta 0), pack-reused 0
-   Receiving objects: 100% (11/11), done.
-
-As you can see by inspecting the sources, Python modules that are part of the extension
-can import any core Spack module.
+   $ git -C /tmp clone https://github.com/spack/spack-scripting.git
 
 ---------------------------------
 Configure Spack to Use Extensions
@@ -61,7 +57,7 @@ paths to ``config.yaml``. In the case of our example this means ensuring that:
 
    config:
      extensions:
-     - ~/tmp/spack-scripting
+     - /tmp/spack-scripting
 
 is part of your configuration file. Once this is setup any command that the extension provides
 will be available from the command line:
@@ -86,37 +82,32 @@ will be available from the command line:
      --implicit       select specs that are not installed or were installed implicitly
      --output OUTPUT  where to dump the result
 
-The corresponding unit tests can be run giving the appropriate options
-to ``spack unit-test``:
+The corresponding unit tests can be run giving the appropriate options to ``spack unit-test``:
 
 .. code-block:: console
 
    $ spack unit-test --extension=scripting
-
-   ============================================================== test session starts ===============================================================
-   platform linux2 -- Python 2.7.15rc1, pytest-3.2.5, py-1.4.34, pluggy-0.4.0
-   rootdir: /home/mculpo/tmp/spack-scripting, inifile: pytest.ini
+   ========================================== test session starts ===========================================
+   platform linux -- Python 3.11.5, pytest-7.4.3, pluggy-1.3.0
+   rootdir: /home/culpo/github/spack-scripting
+   configfile: pytest.ini
+   testpaths: tests
+   plugins: xdist-3.5.0
    collected 5 items
 
-   tests/test_filter.py ...XX
-   ============================================================ short test summary info =============================================================
-   XPASS tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
-   XPASS tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
+   tests/test_filter.py .....                                                                         [100%]
 
-   =========================================================== slowest 20 test durations ============================================================
-   3.74s setup    tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
-   0.17s call     tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
-   0.16s call     tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
-   0.15s call     tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
-   0.13s call     tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
-   0.08s call     tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
-   0.04s teardown tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
-   0.00s setup    tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
-   0.00s setup    tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
-   0.00s setup    tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
-   0.00s setup    tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
-   0.00s teardown tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
-   0.00s teardown tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
-   0.00s teardown tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
-   0.00s teardown tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
-   ====================================================== 3 passed, 2 xpassed in 4.51 seconds =======================================================
+   ========================================== slowest 30 durations ==========================================
+   2.31s setup    tests/test_filter.py::test_filtering_specs[kwargs0-specs0-expected0]
+   0.57s call     tests/test_filter.py::test_filtering_specs[kwargs2-specs2-expected2]
+   0.56s call     tests/test_filter.py::test_filtering_specs[kwargs4-specs4-expected4]
+   0.54s call     tests/test_filter.py::test_filtering_specs[kwargs3-specs3-expected3]
+   0.54s call     tests/test_filter.py::test_filtering_specs[kwargs1-specs1-expected1]
+   0.48s call     tests/test_filter.py::test_filtering_specs[kwargs0-specs0-expected0]
+   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs4-specs4-expected4]
+   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs2-specs2-expected2]
+   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs1-specs1-expected1]
+   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs3-specs3-expected3]
+
+   (5 durations < 0.005s hidden.  Use -vv to show these durations.)
+   =========================================== 5 passed in 5.06s ============================================

lib/spack/docs/frequently_asked_questions.rst

@@ -0,0 +1,77 @@
+.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.
+
+   SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+==========================
+Frequently Asked Questions
+==========================
+
+This page contains answers to frequently asked questions about Spack.
+If you have questions that are not answered here, feel free to ask on
+`Slack <https://slack.spack.io>`_ or `GitHub Discussions
+<https://github.com/spack/spack/discussions>`_. If you've learned the
+answer to a question that you think should be here, please consider
+contributing to this page.
+
+.. _faq-concretizer-precedence:
+
+-----------------------------------------------------
+Why does Spack pick particular versions and variants?
+-----------------------------------------------------
+
+This question comes up in a variety of forms:
+
+ 1. Why does Spack seem to ignore my package preferences from ``packages.yaml`` config?
+ 2. Why does Spack toggle a variant instead of using the default from the ``package.py`` file?
+
+The short answer is that Spack always picks an optimal configuration
+based on a complex set of criteria\ [#f1]_. These criteria are more nuanced
+than always choosing the latest versions or default variants.
+
+.. note::
+
+    As a rule of thumb: requirements + constraints > reuse > preferences > defaults.
+
+The following set of criteria (from lowest to highest precedence) explain
+common cases where concretization output may seem surprising at first.
+
+1. :ref:`Package preferences <package-preferences>` configured in ``packages.yaml``
+   override variant defaults from ``package.py`` files, and influence the optimal
+   ordering of versions. Preferences are specified as follows:
+
+   .. code-block:: yaml
+
+      packages:
+        foo:
+          version: [1.0, 1.1]
+          variants: ~mpi
+
+2. :ref:`Reuse concretization <concretizer-options>` configured in ``concretizer.yaml``
+   overrides preferences, since it's typically faster to reuse an existing spec than to
+   build a preferred one from sources. When build caches are enabled, specs may be reused
+   from a remote location too. Reuse concretization is configured as follows:
+
+   .. code-block:: yaml
+
+      concretizer:
+        reuse: dependencies  # other options are 'true' and 'false'
+
+3. :ref:`Package requirements <package-requirements>` configured in ``packages.yaml``,
+   and constraints from the command line as well as ``package.py`` files override all
+   of the above. Requirements are specified as follows:
+
+   .. code-block:: yaml
+
+      packages:
+        foo:
+          require:
+          - "@1.2: +mpi"
+
+Requirements and constraints restrict the set of possible solutions, while reuse
+behavior and preferences influence what an optimal solution looks like.
+
+
+.. rubric:: Footnotes
+
+.. [#f1] The exact list of criteria can be retrieved with the ``spack solve`` command

lib/spack/docs/gpu_configuration.rst

@@ -111,3 +111,28 @@ CUDA is split into fewer components and is simpler to specify:
          prefix: /opt/cuda/cuda-11.0.2/
 
 where ``/opt/cuda/cuda-11.0.2/lib/`` contains ``libcudart.so``.
+
+
+
+-----------------------------------
+Using an External OpenGL API
+-----------------------------------
+Depending on whether we have a graphics card or not, we may choose to use OSMesa or GLX to implement the OpenGL API.
+
+If a graphics card is unavailable, OSMesa is recommended and can typically be built with Spack.
+However, if we prefer to utilize the system GLX tailored to our graphics card, we need to declare it as an external. Here's how to do it:
+
+
+.. code-block:: yaml
+
+  packages:
+    libglx:
+      require: [opengl]
+    opengl:
+      buildable: false
+      externals:
+      - prefix: /usr/
+        spec: opengl@4.6
+
+Note that prefix has to be the root of both the libraries and the headers, using is /usr not the path the the lib.
+To know which spec for opengl is available use ``cd /usr/include/GL && grep -Ri gl_version``.

lib/spack/docs/index.rst

@@ -55,6 +55,7 @@ or refer to the full manual below.
    getting_started
    basic_usage
    replace_conda_homebrew
+   frequently_asked_questions
 
 .. toctree::
    :maxdepth: 2
@@ -70,7 +71,7 @@ or refer to the full manual below.
 
    configuration
    config_yaml
-   bootstrapping
+   packages_yaml
    build_settings
    environments
    containers
@@ -78,6 +79,7 @@ or refer to the full manual below.
    module_file_support
    repositories
    binary_caches
+   bootstrapping
    command_index
    chain
    extensions

lib/spack/docs/packages_yaml.rst

@@ -0,0 +1,620 @@
+.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.
+
+   SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+.. _packages-config:
+
+================================
+Package Settings (packages.yaml)
+================================
+
+Spack allows you to customize how your software is built through the
+``packages.yaml`` file.  Using it, you can make Spack prefer particular
+implementations of virtual dependencies (e.g., MPI or BLAS/LAPACK),
+or you can make it prefer to build with particular compilers.  You can
+also tell Spack to use *external* software installations already
+present on your system.
+
+At a high level, the ``packages.yaml`` file is structured like this:
+
+.. code-block:: yaml
+
+   packages:
+     package1:
+       # settings for package1
+     package2:
+       # settings for package2
+     # ...
+     all:
+       # settings that apply to all packages.
+
+So you can either set build preferences specifically for *one* package,
+or you can specify that certain settings should apply to *all* packages.
+The types of settings you can customize are described in detail below.
+
+Spack's build defaults are in the default
+``etc/spack/defaults/packages.yaml`` file.  You can override them in
+``~/.spack/packages.yaml`` or ``etc/spack/packages.yaml``. For more
+details on how this works, see :ref:`configuration-scopes`.
+
+.. _sec-external-packages:
+
+-----------------
+External Packages
+-----------------
+
+Spack can be configured to use externally-installed
+packages rather than building its own packages. This may be desirable
+if machines ship with system packages, such as a customized MPI
+that should be used instead of Spack building its own MPI.
+
+External packages are configured through the ``packages.yaml`` file.
+Here's an example of an external configuration:
+
+.. code-block:: yaml
+
+   packages:
+     openmpi:
+       externals:
+       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64"
+         prefix: /opt/openmpi-1.4.3
+       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64+debug"
+         prefix: /opt/openmpi-1.4.3-debug
+       - spec: "openmpi@1.6.5%intel@10.1 arch=linux-debian7-x86_64"
+         prefix: /opt/openmpi-1.6.5-intel
+
+This example lists three installations of OpenMPI, one built with GCC,
+one built with GCC and debug information, and another built with Intel.
+If Spack is asked to build a package that uses one of these MPIs as a
+dependency, it will use the pre-installed OpenMPI in
+the given directory. Note that the specified path is the top-level
+install prefix, not the ``bin`` subdirectory.
+
+``packages.yaml`` can also be used to specify modules to load instead
+of the installation prefixes.  The following example says that module
+``CMake/3.7.2`` provides cmake version 3.7.2.
+
+.. code-block:: yaml
+
+   cmake:
+     externals:
+     - spec: cmake@3.7.2
+       modules:
+       - CMake/3.7.2
+
+Each ``packages.yaml`` begins with a ``packages:`` attribute, followed
+by a list of package names.  To specify externals, add an ``externals:``
+attribute under the package name, which lists externals.
+Each external should specify a ``spec:`` string that should be as
+well-defined as reasonably possible.  If a
+package lacks a spec component, such as missing a compiler or
+package version, then Spack will guess the missing component based
+on its most-favored packages, and it may guess incorrectly.
+
+Each package version and compiler listed in an external should
+have entries in Spack's packages and compiler configuration, even
+though the package and compiler may not ever be built.
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Extra attributes for external packages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Sometimes external packages require additional attributes to be used
+effectively. This information can be defined on a per-package basis
+and stored in the ``extra_attributes`` section of the external package
+configuration. In addition to per-package information, this section
+can be used to define environment modifications to be performed
+whenever the package is used. For example, if an external package is
+built without ``rpath`` support, it may require ``LD_LIBRARY_PATH``
+settings to find its dependencies. This could be configured as
+follows:
+
+.. code-block:: yaml
+
+   packages:
+     mpich:
+       externals:
+       - spec: mpich@3.3 %clang@12.0.0 +hwloc
+         prefix: /path/to/mpich
+         extra_attributes:
+           environment:
+             prepend_path:
+               LD_LIBRARY_PATH: /path/to/hwloc/lib64
+
+See :ref:`configuration_environment_variables` for more information on
+how to configure environment modifications in Spack config files.
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Prevent packages from being built from sources
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Adding an external spec in ``packages.yaml`` allows Spack to use an external location,
+but it does not prevent Spack from building packages from sources. In the above example,
+Spack might choose for many valid reasons to start building and linking with the
+latest version of OpenMPI rather than continue using the pre-installed OpenMPI versions.
+
+To prevent this, the ``packages.yaml`` configuration also allows packages
+to be flagged as non-buildable.  The previous example could be modified to
+be:
+
+.. code-block:: yaml
+
+   packages:
+     openmpi:
+       externals:
+       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64"
+         prefix: /opt/openmpi-1.4.3
+       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64+debug"
+         prefix: /opt/openmpi-1.4.3-debug
+       - spec: "openmpi@1.6.5%intel@10.1 arch=linux-debian7-x86_64"
+         prefix: /opt/openmpi-1.6.5-intel
+       buildable: False
+
+The addition of the ``buildable`` flag tells Spack that it should never build
+its own version of OpenMPI from sources, and it will instead always rely on a pre-built
+OpenMPI.
+
+.. note::
+
+   If ``concretizer:reuse`` is on (see :ref:`concretizer-options` for more information on that flag)
+   pre-built specs include specs already available from a local store, an upstream store, a registered
+   buildcache or specs marked as externals in ``packages.yaml``. If ``concretizer:reuse`` is off, only
+   external specs in ``packages.yaml`` are included in the list of pre-built specs.
+
+If an external module is specified as not buildable, then Spack will load the
+external module into the build environment which can be used for linking.
+
+The ``buildable`` does not need to be paired with external packages.
+It could also be used alone to forbid packages that may be
+buggy or otherwise undesirable.
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Non-buildable virtual packages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Virtual packages in Spack can also be specified as not buildable, and
+external implementations can be provided. In the example above,
+OpenMPI is configured as not buildable, but Spack will often prefer
+other MPI implementations over the externally available OpenMPI. Spack
+can be configured with every MPI provider not buildable individually,
+but more conveniently:
+
+.. code-block:: yaml
+
+   packages:
+     mpi:
+       buildable: False
+     openmpi:
+       externals:
+       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64"
+         prefix: /opt/openmpi-1.4.3
+       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64+debug"
+         prefix: /opt/openmpi-1.4.3-debug
+       - spec: "openmpi@1.6.5%intel@10.1 arch=linux-debian7-x86_64"
+         prefix: /opt/openmpi-1.6.5-intel
+
+Spack can then use any of the listed external implementations of MPI
+to satisfy a dependency, and will choose depending on the compiler and
+architecture.
+
+In cases where the concretizer is configured to reuse specs, and other ``mpi`` providers
+(available via stores or buildcaches) are not wanted, Spack can be configured to require
+specs matching only the available externals:
+
+.. code-block:: yaml
+
+   packages:
+     mpi:
+       buildable: False
+       require:
+       - one_of: [
+           "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64",
+           "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64+debug",
+           "openmpi@1.6.5%intel@10.1 arch=linux-debian7-x86_64"
+         ]
+     openmpi:
+       externals:
+       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64"
+         prefix: /opt/openmpi-1.4.3
+       - spec: "openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64+debug"
+         prefix: /opt/openmpi-1.4.3-debug
+       - spec: "openmpi@1.6.5%intel@10.1 arch=linux-debian7-x86_64"
+         prefix: /opt/openmpi-1.6.5-intel
+
+This configuration prevents any spec using MPI and originating from stores or buildcaches to be reused,
+unless it matches the requirements under ``packages:mpi:require``. For more information on requirements see
+:ref:`package-requirements`.
+
+.. _cmd-spack-external-find:
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Automatically Find External Packages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can run the :ref:`spack external find <spack-external-find>` command
+to search for system-provided packages and add them to ``packages.yaml``.
+After running this command your ``packages.yaml`` may include new entries:
+
+.. code-block:: yaml
+
+   packages:
+     cmake:
+       externals:
+       - spec: cmake@3.17.2
+         prefix: /usr
+
+Generally this is useful for detecting a small set of commonly-used packages;
+for now this is generally limited to finding build-only dependencies.
+Specific limitations include:
+
+* Packages are not discoverable by default: For a package to be
+  discoverable with ``spack external find``, it needs to add special
+  logic. See :ref:`here <make-package-findable>` for more details.
+* The logic does not search through module files, it can only detect
+  packages with executables defined in ``PATH``; you can help Spack locate
+  externals which use module files by loading any associated modules for
+  packages that you want Spack to know about before running
+  ``spack external find``.
+* Spack does not overwrite existing entries in the package configuration:
+  If there is an external defined for a spec at any configuration scope,
+  then Spack will not add a new external entry (``spack config blame packages``
+  can help locate all external entries).
+
+.. _package-requirements:
+
+--------------------
+Package Requirements
+--------------------
+
+Spack can be configured to always use certain compilers, package
+versions, and variants during concretization through package
+requirements.
+
+Package requirements are useful when you find yourself repeatedly
+specifying the same constraints on the command line, and wish that
+Spack respects these constraints whether you mention them explicitly
+or not. Another use case is specifying constraints that should apply
+to all root specs in an environment, without having to repeat the
+constraint everywhere.
+
+Apart from that, requirements config is more flexible than constraints
+on the command line, because it can specify constraints on packages
+*when they occur* as a dependency. In contrast, on the command line it
+is not possible to specify constraints on dependencies while also keeping
+those dependencies optional.
+
+.. seealso::
+
+   FAQ: :ref:`Why does Spack pick particular versions and variants? <faq-concretizer-precedence>`
+
+
+^^^^^^^^^^^^^^^^^^^
+Requirements syntax
+^^^^^^^^^^^^^^^^^^^
+
+The package requirements configuration is specified in ``packages.yaml``,
+keyed by package name and expressed using the Spec syntax. In the simplest
+case you can specify attributes that you always want the package to have
+by providing a single spec string to ``require``:
+
+.. code-block:: yaml
+
+   packages:
+     libfabric:
+       require: "@1.13.2"
+
+In the above example, ``libfabric`` will always build with version 1.13.2. If you
+need to compose multiple configuration scopes ``require`` accepts a list of
+strings:
+
+.. code-block:: yaml
+
+   packages:
+     libfabric:
+       require:
+       - "@1.13.2"
+       - "%gcc"
+
+In this case ``libfabric`` will always build with version 1.13.2 **and** using GCC
+as a compiler.
+
+For more complex use cases, require accepts also a list of objects. These objects
+must have either a ``any_of`` or a ``one_of`` field, containing a list of spec strings,
+and they can optionally have a ``when`` and a ``message`` attribute:
+
+.. code-block:: yaml
+
+   packages:
+     openmpi:
+       require:
+       - any_of: ["@4.1.5", "%gcc"]
+         message: "in this example only 4.1.5 can build with other compilers"
+
+``any_of`` is a list of specs. One of those specs must be satisfied
+and it is also allowed for the concretized spec to match more than one.
+In the above example, that means you could build ``openmpi@4.1.5%gcc``,
+``openmpi@4.1.5%clang`` or ``openmpi@3.9%gcc``, but
+not ``openmpi@3.9%clang``.
+
+If a custom message is provided, and the requirement is not satisfiable,
+Spack will print the custom error message:
+
+.. code-block:: console
+
+   $ spack spec openmpi@3.9%clang
+   ==> Error: in this example only 4.1.5 can build with other compilers
+
+We could express a similar requirement using the ``when`` attribute:
+
+.. code-block:: yaml
+
+   packages:
+     openmpi:
+       require:
+       - any_of: ["%gcc"]
+         when: "@:4.1.4"
+         message: "in this example only 4.1.5 can build with other compilers"
+
+In the example above, if the version turns out to be 4.1.4 or less, we require the compiler to be GCC.
+For readability, Spack also allows a ``spec`` key accepting a string when there is only a single
+constraint:
+
+.. code-block:: yaml
+
+   packages:
+     openmpi:
+       require:
+       - spec: "%gcc"
+         when: "@:4.1.4"
+         message: "in this example only 4.1.5 can build with other compilers"
+
+This code snippet and the one before it are semantically equivalent.
+
+Finally, instead of ``any_of`` you can use ``one_of`` which also takes a list of specs. The final
+concretized spec must match one and only one of them:
+
+.. code-block:: yaml
+
+   packages:
+     mpich:
+       require:
+       - one_of: ["+cuda", "+rocm"]
+
+In the example above, that means you could build ``mpich+cuda`` or ``mpich+rocm`` but not ``mpich+cuda+rocm``.
+
+.. note::
+
+   For ``any_of`` and ``one_of``, the order of specs indicates a
+   preference: items that appear earlier in the list are preferred
+   (note that these preferences can be ignored in favor of others).
+
+.. note::
+
+   When using a conditional requirement, Spack is allowed to actively avoid the triggering
+   condition (the ``when=...`` spec) if that leads to a concrete spec with better scores in
+   the optimization criteria. To check the current optimization criteria and their
+   priorities you can run ``spack solve zlib``.
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Setting default requirements
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can also set default requirements for all packages under ``all``
+like this:
+
+.. code-block:: yaml
+
+   packages:
+     all:
+       require: '%clang'
+
+which means every spec will be required to use ``clang`` as a compiler.
+
+Requirements on variants for all packages are possible too, but note that they
+are only enforced for those packages that define these variants, otherwise they
+are disregarded. For example:
+
+.. code-block:: yaml
+
+   packages:
+     all:
+       require:
+       - "+shared"
+       - "+cuda"
+
+will just enforce ``+shared`` on ``zlib``, which has a boolean ``shared`` variant but
+no ``cuda`` variant.
+
+Constraints in a single spec literal are always considered as a whole, so in a case like:
+
+.. code-block:: yaml
+
+   packages:
+     all:
+       require: "+shared +cuda"
+
+the default requirement will be enforced only if a package has both a ``cuda`` and
+a ``shared`` variant, and will never be partially enforced.
+
+Finally, ``all`` represents a *default set of requirements* -
+if there are specific package requirements, then the default requirements
+under ``all`` are disregarded. For example, with a configuration like this:
+
+.. code-block:: yaml
+
+   packages:
+     all:
+       require:
+       - 'build_type=Debug'
+       - '%clang'
+     cmake:
+       require:
+       - 'build_type=Debug'
+       - '%gcc'
+
+Spack requires ``cmake`` to use ``gcc`` and all other nodes (including ``cmake``
+dependencies) to use ``clang``. If enforcing ``build_type=Debug`` is needed also
+on ``cmake``, it must be repeated in the specific ``cmake`` requirements.
+
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Setting requirements on virtual specs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A requirement on a virtual spec applies whenever that virtual is present in the DAG.
+This can be useful for fixing which virtual provider you want to use:
+
+.. code-block:: yaml
+
+   packages:
+     mpi:
+       require: 'mvapich2 %gcc'
+
+With the configuration above the only allowed ``mpi`` provider is ``mvapich2 %gcc``.
+
+Requirements on the virtual spec and on the specific provider are both applied, if
+present. For instance with a configuration like:
+
+.. code-block:: yaml
+
+   packages:
+     mpi:
+       require: 'mvapich2 %gcc'
+     mvapich2:
+       require: '~cuda'
+
+you will use ``mvapich2~cuda %gcc`` as an ``mpi`` provider.
+
+.. _package-preferences:
+
+-------------------
+Package Preferences
+-------------------
+
+In some cases package requirements can be too strong, and package
+preferences are the better option. Package preferences do not impose
+constraints on packages for particular versions or variants values,
+they rather only set defaults. The concretizer is free to change
+them if it must, due to other constraints, and also prefers reusing
+installed packages over building new ones that are a better match for
+preferences.
+
+.. seealso::
+
+   FAQ: :ref:`Why does Spack pick particular versions and variants? <faq-concretizer-precedence>`
+
+
+Most package preferences (``compilers``, ``target`` and ``providers``)
+can only be set globally under the ``all`` section of ``packages.yaml``:
+
+.. code-block:: yaml
+
+   packages:
+     all:
+       compiler: [gcc@12.2.0, clang@12:, oneapi@2023:]
+       target: [x86_64_v3]
+       providers:
+         mpi: [mvapich2, mpich, openmpi]
+
+These preferences override Spack's default and effectively reorder priorities
+when looking for the best compiler, target or virtual package provider. Each
+preference takes an ordered list of spec constraints, with earlier entries in
+the list being preferred over later entries.
+
+In the example above all packages prefer to be compiled with ``gcc@12.2.0``,
+to target the ``x86_64_v3`` microarchitecture and to use ``mvapich2`` if they
+depend on ``mpi``.
+
+The ``variants`` and ``version`` preferences can be set under
+package specific sections of the ``packages.yaml`` file:
+
+.. code-block:: yaml
+
+   packages:
+     opencv:
+       variants: +debug
+     gperftools:
+       version: [2.2, 2.4, 2.3]
+
+In this case, the preference for ``opencv`` is to build with debug options, while
+``gperftools`` prefers version 2.2 over 2.4.
+
+Any preference can be overwritten on the command line if explicitly requested.
+
+Preferences cannot overcome explicit constraints, as they only set a preferred
+ordering among homogeneous attribute values. Going back to the example, if
+``gperftools@2.3:`` was requested, then Spack will install version 2.4
+since the most preferred version 2.2 is prohibited by the version constraint.
+
+.. _package_permissions:
+
+-------------------
+Package Permissions
+-------------------
+
+Spack can be configured to assign permissions to the files installed
+by a package.
+
+In the ``packages.yaml`` file under ``permissions``, the attributes
+``read``, ``write``, and ``group`` control the package
+permissions. These attributes can be set per-package, or for all
+packages under ``all``. If permissions are set under ``all`` and for a
+specific package, the package-specific settings take precedence.
+
+The ``read`` and ``write`` attributes take one of ``user``, ``group``,
+and ``world``.
+
+.. code-block:: yaml
+
+  packages:
+    all:
+      permissions:
+        write: group
+        group: spack
+    my_app:
+      permissions:
+        read: group
+        group: my_team
+
+The permissions settings describe the broadest level of access to
+installations of the specified packages. The execute permissions of
+the file are set to the same level as read permissions for those files
+that are executable. The default setting for ``read`` is ``world``,
+and for ``write`` is ``user``. In the example above, installations of
+``my_app`` will be installed with user and group permissions but no
+world permissions, and owned by the group ``my_team``. All other
+packages will be installed with user and group write privileges, and
+world read privileges. Those packages will be owned by the group
+``spack``.
+
+The ``group`` attribute assigns a Unix-style group to a package. All
+files installed by the package will be owned by the assigned group,
+and the sticky group bit will be set on the install prefix and all
+directories inside the install prefix. This will ensure that even
+manually placed files within the install prefix are owned by the
+assigned group. If no group is assigned, Spack will allow the OS
+default behavior to go as expected.
+
+----------------------------
+Assigning Package Attributes
+----------------------------
+
+You can assign class-level attributes in the configuration:
+
+.. code-block:: yaml
+
+  packages:
+    mpileaks:
+      # Override existing attributes
+      url: http://www.somewhereelse.com/mpileaks-1.0.tar.gz
+      # ... or add new ones
+      x: 1
+
+Attributes set this way will be accessible to any method executed
+in the package.py file (e.g. the ``install()`` method). Values for these
+attributes may be any value parseable by yaml.
+
+These can only be applied to specific packages, not "all" or
+virtual packages.

lib/spack/docs/packaging_guide.rst

@@ -2337,7 +2337,7 @@ window while a batch job is running ``spack install`` on the same or
 overlapping dependencies without any process trying to re-do the work of
 another.
 
-For example, if you are using SLURM, you could launch an installation
+For example, if you are using Slurm, you could launch an installation
 of ``mpich`` using the following command:
 
 .. code-block:: console
@@ -5284,7 +5284,7 @@ installed example.
            example = which(self.prefix.bin.example)
            example()
 
-Output showing the identification of each test part after runnig the tests
+Output showing the identification of each test part after running the tests
 is illustrated below.
 
 .. code-block:: console
@@ -5781,7 +5781,7 @@ with those implemented in the package itself.
    * - `Cxx
        <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/cxx>`_
      - Compiles and runs several ``hello`` programs
-   * - `Fortan
+   * - `Fortran
        <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/fortran>`_
      - Compiles and runs ``hello`` programs (``F`` and ``f90``)
    * - `Mpi

lib/spack/docs/requirements.txt

@@ -1,13 +1,13 @@
 sphinx==7.2.6
 sphinxcontrib-programoutput==0.17
 sphinx_design==0.5.0
-sphinx-rtd-theme==1.3.0
+sphinx-rtd-theme==2.0.0
 python-levenshtein==0.23.0
-docutils==0.18.1
-pygments==2.16.1
-urllib3==2.0.7
+docutils==0.20.1
+pygments==2.17.2
+urllib3==2.1.0
 pytest==7.4.3
-isort==5.12.0
-black==23.10.1
+isort==5.13.2
+black==23.12.0
 flake8==6.1.0
-mypy==1.6.1
+mypy==1.7.1

lib/spack/docs/signing.rst

@@ -142,7 +142,7 @@ Reputational Key
 ----------------
 
 The Reputational Key is the public facing key used to sign complete groups of
-development and release packages. Only one key pair exsits in this class of
+development and release packages. Only one key pair exists in this class of
 keys. In contrast to the Intermediate CI Key the Reputational Key *should* be
 used to verify package integrity. At the end of develop and release pipeline a
 final pipeline job pulls down all signed package metadata built by the pipeline,
@@ -272,7 +272,7 @@ Internal Implementation
 
 The technical implementation of the pipeline signing process includes components
 defined in Amazon Web Services, the Kubernetes cluster, at affilicated
-institutions, and the GitLab/GitLab Runner deployment. We present the techincal
+institutions, and the GitLab/GitLab Runner deployment. We present the technical
 implementation in two interdependent sections. The first addresses how secrets
 are managed through the lifecycle of a develop or release pipeline. The second
 section describes how Gitlab Runner and pipelines are configured and managed to
@@ -295,7 +295,7 @@ infrastructure.
 -----------------------
 
 Multiple intermediate CI signing keys exist, one Intermediate CI Key for jobs
-run in AWS, and one key for each affiliated institution (e.g. Univerity of
+run in AWS, and one key for each affiliated institution (e.g. University of
 Oregon). Here we describe how the Intermediate CI Key is managed in AWS:
 
 The Intermediate CI Key (including the Signing Intermediate CI Private Key is
@@ -305,7 +305,7 @@ contains an ASCII-armored export of just the *public* components of the
 Reputational Key. This secret also contains the *public* components of each of
 the affiliated institutions' Intermediate CI Key. These are potentially needed
 to verify dependent packages which may have been found in the public mirror or
-built by a protected job running on an affiliated institution's infrastrcuture
+built by a protected job running on an affiliated institution's infrastructure
 in an earlier stage of the pipeline.
 
 Procedurally the ``spack-intermediate-ci-signing-key`` secret is used in

lib/spack/llnl/util/lang.py

@@ -1047,9 +1047,9 @@ def __bool__(self):
         """Whether any exceptions were handled."""
         return bool(self.exceptions)
 
-    def forward(self, context: str) -> "GroupedExceptionForwarder":
+    def forward(self, context: str, base: type = BaseException) -> "GroupedExceptionForwarder":
         """Return a contextmanager which extracts tracebacks and prefixes a message."""
-        return GroupedExceptionForwarder(context, self)
+        return GroupedExceptionForwarder(context, self, base)
 
     def _receive_forwarded(self, context: str, exc: Exception, tb: List[str]):
         self.exceptions.append((context, exc, tb))
@@ -1072,15 +1072,18 @@ class GroupedExceptionForwarder:
     """A contextmanager to capture exceptions and forward them to a
     GroupedExceptionHandler."""
 
-    def __init__(self, context: str, handler: GroupedExceptionHandler):
+    def __init__(self, context: str, handler: GroupedExceptionHandler, base: type):
         self._context = context
         self._handler = handler
+        self._base = base
 
     def __enter__(self):
         return None
 
     def __exit__(self, exc_type, exc_value, tb):
         if exc_value is not None:
+            if not issubclass(exc_type, self._base):
+                return False
             self._handler._receive_forwarded(self._context, exc_value, traceback.format_tb(tb))
 
         # Suppress any exception from being re-raised:

lib/spack/spack/__init__.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 #: PEP440 canonical <major>.<minor>.<micro>.<devN> string
-__version__ = "0.21.0.dev0"
+__version__ = "0.22.0.dev0"
 spack_version = __version__
 
 

lib/spack/spack/audit.py

@@ -40,6 +40,7 @@ def _search_duplicate_compilers(error_cls):
 import collections.abc
 import glob
 import inspect
+import io
 import itertools
 import pathlib
 import pickle
@@ -54,6 +55,7 @@ def _search_duplicate_compilers(error_cls):
 import spack.repo
 import spack.spec
 import spack.util.crypto
+import spack.util.spack_yaml as syaml
 import spack.variant
 
 #: Map an audit tag to a list of callables implementing checks
@@ -250,6 +252,88 @@ def _search_duplicate_specs_in_externals(error_cls):
     return errors
 
 
+@config_packages
+def _deprecated_preferences(error_cls):
+    """Search package preferences deprecated in v0.21 (and slated for removal in v0.22)"""
+    # TODO (v0.22): remove this audit as the attributes will not be allowed in config
+    errors = []
+    packages_yaml = spack.config.CONFIG.get_config("packages")
+
+    def make_error(attribute_name, config_data, summary):
+        s = io.StringIO()
+        s.write("Occurring in the following file:\n")
+        dict_view = syaml.syaml_dict((k, v) for k, v in config_data.items() if k == attribute_name)
+        syaml.dump_config(dict_view, stream=s, blame=True)
+        return error_cls(summary=summary, details=[s.getvalue()])
+
+    if "all" in packages_yaml and "version" in packages_yaml["all"]:
+        summary = "Using the deprecated 'version' attribute under 'packages:all'"
+        errors.append(make_error("version", packages_yaml["all"], summary))
+
+    for package_name in packages_yaml:
+        if package_name == "all":
+            continue
+
+        package_conf = packages_yaml[package_name]
+        for attribute in ("compiler", "providers", "target"):
+            if attribute not in package_conf:
+                continue
+            summary = (
+                f"Using the deprecated '{attribute}' attribute " f"under 'packages:{package_name}'"
+            )
+            errors.append(make_error(attribute, package_conf, summary))
+
+    return errors
+
+
+@config_packages
+def _avoid_mismatched_variants(error_cls):
+    """Warns if variant preferences have mismatched types or names."""
+    errors = []
+    packages_yaml = spack.config.CONFIG.get_config("packages")
+
+    def make_error(config_data, summary):
+        s = io.StringIO()
+        s.write("Occurring in the following file:\n")
+        syaml.dump_config(config_data, stream=s, blame=True)
+        return error_cls(summary=summary, details=[s.getvalue()])
+
+    for pkg_name in packages_yaml:
+        # 'all:' must be more forgiving, since it is setting defaults for everything
+        if pkg_name == "all" or "variants" not in packages_yaml[pkg_name]:
+            continue
+
+        preferences = packages_yaml[pkg_name]["variants"]
+        if not isinstance(preferences, list):
+            preferences = [preferences]
+
+        for variants in preferences:
+            current_spec = spack.spec.Spec(variants)
+            pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
+            for variant in current_spec.variants.values():
+                # Variant does not exist at all
+                if variant.name not in pkg_cls.variants:
+                    summary = (
+                        f"Setting a preference for the '{pkg_name}' package to the "
+                        f"non-existing variant '{variant.name}'"
+                    )
+                    errors.append(make_error(preferences, summary))
+                    continue
+
+                # Variant cannot accept this value
+                s = spack.spec.Spec(pkg_name)
+                try:
+                    s.update_variant_validate(variant.name, variant.value)
+                except Exception:
+                    summary = (
+                        f"Setting the variant '{variant.name}' of the '{pkg_name}' package "
+                        f"to the invalid value '{str(variant)}'"
+                    )
+                    errors.append(make_error(preferences, summary))
+
+    return errors
+
+
 #: Sanity checks on package directives
 package_directives = AuditClass(
     group="packages",
@@ -642,13 +726,46 @@ def _unknown_variants_in_directives(pkgs, error_cls):
 
 
 @package_directives
-def _unknown_variants_in_dependencies(pkgs, error_cls):
-    """Report unknown dependencies and wrong variants for dependencies"""
+def _issues_in_depends_on_directive(pkgs, error_cls):
+    """Reports issues with 'depends_on' directives.
+
+    Issues might be unknown dependencies, unknown variants or variant values, or declaration
+    of nested dependencies.
+    """
     errors = []
     for pkg_name in pkgs:
         pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
         filename = spack.repo.PATH.filename_for_package_name(pkg_name)
         for dependency_name, dependency_data in pkg_cls.dependencies.items():
+            # Check if there are nested dependencies declared. We don't want directives like:
+            #
+            #     depends_on('foo+bar ^fee+baz')
+            #
+            # but we'd like to have two dependencies listed instead.
+            for when, dependency_edge in dependency_data.items():
+                dependency_spec = dependency_edge.spec
+                nested_dependencies = dependency_spec.dependencies()
+                if nested_dependencies:
+                    summary = (
+                        f"{pkg_name}: invalid nested dependency "
+                        f"declaration '{str(dependency_spec)}'"
+                    )
+                    details = [
+                        f"split depends_on('{str(dependency_spec)}', when='{str(when)}') "
+                        f"into {len(nested_dependencies) + 1} directives",
+                        f"in {filename}",
+                    ]
+                    errors.append(error_cls(summary=summary, details=details))
+
+                for s in (dependency_spec, when):
+                    if s.virtual and s.variants:
+                        summary = f"{pkg_name}: virtual dependency cannot have variants"
+                        details = [
+                            f"remove variants from '{str(s)}' in depends_on directive",
+                            f"in {filename}",
+                        ]
+                        errors.append(error_cls(summary=summary, details=details))
+
             # No need to analyze virtual packages
             if spack.repo.PATH.is_virtual(dependency_name):
                 continue
@@ -776,7 +893,7 @@ def _version_constraints_are_satisfiable_by_some_version_in_repo(pkgs, error_cls
                 )
             except Exception:
                 summary = (
-                    "{0}: dependency on {1} cannot be satisfied " "by known versions of {1.name}"
+                    "{0}: dependency on {1} cannot be satisfied by known versions of {1.name}"
                 ).format(pkg_name, s)
                 details = ["happening in " + filename]
                 if dependency_pkg_cls is not None:
@@ -818,6 +935,53 @@ def _analyze_variants_in_directive(pkg, constraint, directive, error_cls):
     return errors
 
 
+@package_directives
+def _named_specs_in_when_arguments(pkgs, error_cls):
+    """Reports named specs in the 'when=' attribute of a directive.
+
+    Note that 'conflicts' is the only directive allowing that.
+    """
+    errors = []
+    for pkg_name in pkgs:
+        pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
+
+        def _extracts_errors(triggers, summary):
+            _errors = []
+            for trigger in list(triggers):
+                when_spec = spack.spec.Spec(trigger)
+                if when_spec.name is not None and when_spec.name != pkg_name:
+                    details = [f"using '{trigger}', should be '^{trigger}'"]
+                    _errors.append(error_cls(summary=summary, details=details))
+            return _errors
+
+        for dname, triggers in pkg_cls.dependencies.items():
+            summary = f"{pkg_name}: wrong 'when=' condition for the '{dname}' dependency"
+            errors.extend(_extracts_errors(triggers, summary))
+
+        for vname, (variant, triggers) in pkg_cls.variants.items():
+            summary = f"{pkg_name}: wrong 'when=' condition for the '{vname}' variant"
+            errors.extend(_extracts_errors(triggers, summary))
+
+        for provided, triggers in pkg_cls.provided.items():
+            summary = f"{pkg_name}: wrong 'when=' condition for the '{provided}' virtual"
+            errors.extend(_extracts_errors(triggers, summary))
+
+        for _, triggers in pkg_cls.requirements.items():
+            triggers = [when_spec for when_spec, _, _ in triggers]
+            summary = f"{pkg_name}: wrong 'when=' condition in 'requires' directive"
+            errors.extend(_extracts_errors(triggers, summary))
+
+        triggers = list(pkg_cls.patches)
+        summary = f"{pkg_name}: wrong 'when=' condition in 'patch' directives"
+        errors.extend(_extracts_errors(triggers, summary))
+
+        triggers = list(pkg_cls.resources)
+        summary = f"{pkg_name}: wrong 'when=' condition in 'resource' directives"
+        errors.extend(_extracts_errors(triggers, summary))
+
+    return llnl.util.lang.dedupe(errors)
+
+
 #: Sanity checks on package directives
 external_detection = AuditClass(
     group="externals",

lib/spack/spack/binary_distribution.py

@@ -25,7 +25,7 @@
 import warnings
 from contextlib import closing, contextmanager
 from gzip import GzipFile
-from typing import Dict, List, NamedTuple, Optional, Set, Tuple
+from typing import Dict, Iterable, List, NamedTuple, Optional, Set, Tuple
 from urllib.error import HTTPError, URLError
 
 import llnl.util.filesystem as fsys
@@ -66,8 +66,9 @@
 from spack.stage import Stage
 from spack.util.executable import which
 
-_build_cache_relative_path = "build_cache"
-_build_cache_keys_relative_path = "_pgp"
+BUILD_CACHE_RELATIVE_PATH = "build_cache"
+BUILD_CACHE_KEYS_RELATIVE_PATH = "_pgp"
+CURRENT_BUILD_CACHE_LAYOUT_VERSION = 1
 
 
 class BuildCacheDatabase(spack_db.Database):
@@ -229,7 +230,11 @@ def _associate_built_specs_with_mirror(self, cache_key, mirror_url):
                 )
                 return
 
-            spec_list = db.query_local(installed=False, in_buildcache=True)
+            spec_list = [
+                s
+                for s in db.query_local(installed=any, in_buildcache=any)
+                if s.external or db.query_local_by_spec_hash(s.dag_hash()).in_buildcache
+            ]
 
             for indexed_spec in spec_list:
                 dag_hash = indexed_spec.dag_hash()
@@ -481,7 +486,7 @@ def _fetch_and_cache_index(self, mirror_url, cache_entry={}):
         scheme = urllib.parse.urlparse(mirror_url).scheme
 
         if scheme != "oci" and not web_util.url_exists(
-            url_util.join(mirror_url, _build_cache_relative_path, "index.json")
+            url_util.join(mirror_url, BUILD_CACHE_RELATIVE_PATH, "index.json")
         ):
             return False
 
@@ -600,6 +605,10 @@ def __init__(self, msg):
         super().__init__(msg)
 
 
+class InvalidMetadataFile(spack.error.SpackError):
+    pass
+
+
 class UnsignedPackageException(spack.error.SpackError):
     """
     Raised if installation of unsigned package is attempted without
@@ -614,11 +623,11 @@ def compute_hash(data):
 
 
 def build_cache_relative_path():
-    return _build_cache_relative_path
+    return BUILD_CACHE_RELATIVE_PATH
 
 
 def build_cache_keys_relative_path():
-    return _build_cache_keys_relative_path
+    return BUILD_CACHE_KEYS_RELATIVE_PATH
 
 
 def build_cache_prefix(prefix):
@@ -1401,7 +1410,7 @@ def _build_tarball_in_stage_dir(spec: Spec, out_url: str, stage_dir: str, option
             spec_dict = sjson.load(content)
         else:
             raise ValueError("{0} not a valid spec file type".format(spec_file))
-    spec_dict["buildcache_layout_version"] = 1
+    spec_dict["buildcache_layout_version"] = CURRENT_BUILD_CACHE_LAYOUT_VERSION
     spec_dict["binary_cache_checksum"] = {"hash_algorithm": "sha256", "hash": checksum}
 
     with open(specfile_path, "w") as outfile:
@@ -1560,14 +1569,50 @@ def _delete_staged_downloads(download_result):
     download_result["specfile_stage"].destroy()
 
 
-def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
+def _get_valid_spec_file(path: str, max_supported_layout: int) -> Tuple[Dict, int]:
+    """Read and validate a spec file, returning the spec dict with its layout version, or raising
+    InvalidMetadataFile if invalid."""
+    try:
+        with open(path, "rb") as f:
+            binary_content = f.read()
+    except OSError:
+        raise InvalidMetadataFile(f"No such file: {path}")
+
+    # In the future we may support transparently decompressing compressed spec files.
+    if binary_content[:2] == b"\x1f\x8b":
+        raise InvalidMetadataFile("Compressed spec files are not supported")
+
+    try:
+        as_string = binary_content.decode("utf-8")
+        if path.endswith(".json.sig"):
+            spec_dict = Spec.extract_json_from_clearsig(as_string)
+        else:
+            spec_dict = json.loads(as_string)
+    except Exception as e:
+        raise InvalidMetadataFile(f"Could not parse {path} due to: {e}") from e
+
+    # Ensure this version is not too new.
+    try:
+        layout_version = int(spec_dict.get("buildcache_layout_version", 0))
+    except ValueError as e:
+        raise InvalidMetadataFile("Could not parse layout version") from e
+
+    if layout_version > max_supported_layout:
+        raise InvalidMetadataFile(
+            f"Layout version {layout_version} is too new for this version of Spack"
+        )
+
+    return spec_dict, layout_version
+
+
+def download_tarball(spec, unsigned: Optional[bool] = False, mirrors_for_spec=None):
     """
     Download binary tarball for given package into stage area, returning
     path to downloaded tarball if successful, None otherwise.
 
     Args:
         spec (spack.spec.Spec): Concrete spec
-        unsigned (bool): Whether or not to require signed binaries
+        unsigned: if ``True`` or ``False`` override the mirror signature verification defaults
         mirrors_for_spec (list): Optional list of concrete specs and mirrors
             obtained by calling binary_distribution.get_mirrors_for_spec().
             These will be checked in order first before looking in other
@@ -1588,7 +1633,9 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
            "signature_verified": "true-if-binary-pkg-was-already-verified"
        }
     """
-    configured_mirrors = spack.mirror.MirrorCollection(binary=True).values()
+    configured_mirrors: Iterable[spack.mirror.Mirror] = spack.mirror.MirrorCollection(
+        binary=True
+    ).values()
     if not configured_mirrors:
         tty.die("Please add a spack mirror to allow download of pre-compiled packages.")
 
@@ -1606,8 +1653,16 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
     # mirror for the spec twice though.
     try_first = [i["mirror_url"] for i in mirrors_for_spec] if mirrors_for_spec else []
     try_next = [i.fetch_url for i in configured_mirrors if i.fetch_url not in try_first]
+    mirror_urls = try_first + try_next
 
-    mirrors = try_first + try_next
+    # TODO: turn `mirrors_for_spec` into a list of Mirror instances, instead of doing that here.
+    def fetch_url_to_mirror(url):
+        for mirror in configured_mirrors:
+            if mirror.fetch_url == url:
+                return mirror
+        return spack.mirror.Mirror(url)
+
+    mirrors = [fetch_url_to_mirror(url) for url in mirror_urls]
 
     tried_to_verify_sigs = []
 
@@ -1616,14 +1671,17 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
     # we remove support for deprecated spec formats and buildcache layouts.
     for try_signed in (True, False):
         for mirror in mirrors:
+            # Override mirror's default if
+            currently_unsigned = unsigned if unsigned is not None else not mirror.signed
+
             # If it's an OCI index, do things differently, since we cannot compose URLs.
-            parsed = urllib.parse.urlparse(mirror)
+            fetch_url = mirror.fetch_url
 
             # TODO: refactor this to some "nice" place.
-            if parsed.scheme == "oci":
-                ref = spack.oci.image.ImageReference.from_string(mirror[len("oci://") :]).with_tag(
-                    spack.oci.image.default_tag(spec)
-                )
+            if fetch_url.startswith("oci://"):
+                ref = spack.oci.image.ImageReference.from_string(
+                    fetch_url[len("oci://") :]
+                ).with_tag(spack.oci.image.default_tag(spec))
 
                 # Fetch the manifest
                 try:
@@ -1652,6 +1710,18 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
                     try:
                         local_specfile_stage.fetch()
                         local_specfile_stage.check()
+                        try:
+                            _get_valid_spec_file(
+                                local_specfile_stage.save_filename,
+                                CURRENT_BUILD_CACHE_LAYOUT_VERSION,
+                            )
+                        except InvalidMetadataFile as e:
+                            tty.warn(
+                                f"Ignoring binary package for {spec.name}/{spec.dag_hash()[:7]} "
+                                f"from {fetch_url} due to invalid metadata file: {e}"
+                            )
+                            local_specfile_stage.destroy()
+                            continue
                     except Exception:
                         continue
                     local_specfile_stage.cache_local()
@@ -1670,28 +1740,43 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
                     "tarball_stage": tarball_stage,
                     "specfile_stage": local_specfile_stage,
                     "signature_verified": False,
+                    "signature_required": not currently_unsigned,
                 }
 
             else:
                 ext = "json.sig" if try_signed else "json"
-                specfile_path = url_util.join(mirror, _build_cache_relative_path, specfile_prefix)
+                specfile_path = url_util.join(
+                    fetch_url, BUILD_CACHE_RELATIVE_PATH, specfile_prefix
+                )
                 specfile_url = f"{specfile_path}.{ext}"
-                spackfile_url = url_util.join(mirror, _build_cache_relative_path, tarball)
+                spackfile_url = url_util.join(fetch_url, BUILD_CACHE_RELATIVE_PATH, tarball)
                 local_specfile_stage = try_fetch(specfile_url)
                 if local_specfile_stage:
                     local_specfile_path = local_specfile_stage.save_filename
                     signature_verified = False
 
-                    if try_signed and not unsigned:
+                    try:
+                        _get_valid_spec_file(
+                            local_specfile_path, CURRENT_BUILD_CACHE_LAYOUT_VERSION
+                        )
+                    except InvalidMetadataFile as e:
+                        tty.warn(
+                            f"Ignoring binary package for {spec.name}/{spec.dag_hash()[:7]} "
+                            f"from {fetch_url} due to invalid metadata file: {e}"
+                        )
+                        local_specfile_stage.destroy()
+                        continue
+
+                    if try_signed and not currently_unsigned:
                         # If we found a signed specfile at the root, try to verify
                         # the signature immediately.  We will not download the
                         # tarball if we could not verify the signature.
                         tried_to_verify_sigs.append(specfile_url)
                         signature_verified = try_verify(local_specfile_path)
                         if not signature_verified:
-                            tty.warn("Failed to verify: {0}".format(specfile_url))
+                            tty.warn(f"Failed to verify: {specfile_url}")
 
-                    if unsigned or signature_verified or not try_signed:
+                    if currently_unsigned or signature_verified or not try_signed:
                         # We will download the tarball in one of three cases:
                         #     1. user asked for --no-check-signature
                         #     2. user didn't ask for --no-check-signature, but we
@@ -1714,6 +1799,7 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
                                 "tarball_stage": tarball_stage,
                                 "specfile_stage": local_specfile_stage,
                                 "signature_verified": signature_verified,
+                                "signature_required": not currently_unsigned,
                             }
 
                     local_specfile_stage.destroy()
@@ -1912,7 +1998,7 @@ def is_backup_file(file):
             relocate.relocate_text(text_names, prefix_to_prefix_text)
 
 
-def _extract_inner_tarball(spec, filename, extract_to, unsigned, remote_checksum):
+def _extract_inner_tarball(spec, filename, extract_to, signature_required: bool, remote_checksum):
     stagepath = os.path.dirname(filename)
     spackfile_name = tarball_name(spec, ".spack")
     spackfile_path = os.path.join(stagepath, spackfile_name)
@@ -1932,7 +2018,7 @@ def _extract_inner_tarball(spec, filename, extract_to, unsigned, remote_checksum
     else:
         raise ValueError("Cannot find spec file for {0}.".format(extract_to))
 
-    if not unsigned:
+    if signature_required:
         if os.path.exists("%s.asc" % specfile_path):
             suppress = config.get("config:suppress_gpg_warnings", False)
             try:
@@ -1981,7 +2067,7 @@ def _tar_strip_component(tar: tarfile.TarFile, prefix: str):
                 m.linkname = m.linkname[result.end() :]
 
 
-def extract_tarball(spec, download_result, unsigned=False, force=False, timer=timer.NULL_TIMER):
+def extract_tarball(spec, download_result, force=False, timer=timer.NULL_TIMER):
     """
     extract binary tarball for given package into install area
     """
@@ -2001,35 +2087,30 @@ def extract_tarball(spec, download_result, unsigned=False, force=False, timer=ti
     )
 
     specfile_path = download_result["specfile_stage"].save_filename
-
-    with open(specfile_path, "r") as inputfile:
-        content = inputfile.read()
-        if specfile_path.endswith(".json.sig"):
-            spec_dict = Spec.extract_json_from_clearsig(content)
-        else:
-            spec_dict = sjson.load(content)
-
+    spec_dict, layout_version = _get_valid_spec_file(
+        specfile_path, CURRENT_BUILD_CACHE_LAYOUT_VERSION
+    )
     bchecksum = spec_dict["binary_cache_checksum"]
 
     filename = download_result["tarball_stage"].save_filename
-    signature_verified = download_result["signature_verified"]
+    signature_verified: bool = download_result["signature_verified"]
+    signature_required: bool = download_result["signature_required"]
     tmpdir = None
 
-    if (
-        "buildcache_layout_version" not in spec_dict
-        or int(spec_dict["buildcache_layout_version"]) < 1
-    ):
+    if layout_version == 0:
         # Handle the older buildcache layout where the .spack file
         # contains a spec json, maybe an .asc file (signature),
         # and another tarball containing the actual install tree.
         tmpdir = tempfile.mkdtemp()
         try:
-            tarfile_path = _extract_inner_tarball(spec, filename, tmpdir, unsigned, bchecksum)
+            tarfile_path = _extract_inner_tarball(
+                spec, filename, tmpdir, signature_required, bchecksum
+            )
         except Exception as e:
             _delete_staged_downloads(download_result)
             shutil.rmtree(tmpdir)
             raise e
-    else:
+    elif layout_version == 1:
         # Newer buildcache layout: the .spack file contains just
         # in the install tree, the signature, if it exists, is
         # wrapped around the spec.json at the root.  If sig verify
@@ -2037,9 +2118,10 @@ def extract_tarball(spec, download_result, unsigned=False, force=False, timer=ti
         # the tarball.
         tarfile_path = filename
 
-        if not unsigned and not signature_verified:
+        if signature_required and not signature_verified:
             raise UnsignedPackageException(
-                "To install unsigned packages, use the --no-check-signature option."
+                "To install unsigned packages, use the --no-check-signature option, "
+                "or configure the mirror with signed: false."
             )
 
         # compute the sha256 checksum of the tarball
@@ -2053,7 +2135,6 @@ def extract_tarball(spec, download_result, unsigned=False, force=False, timer=ti
             raise NoChecksumException(
                 tarfile_path, size, contents, "sha256", expected, local_checksum
             )
-
     try:
         with closing(tarfile.open(tarfile_path, "r")) as tar:
             # Remove install prefix from tarfil to extract directly into spec.prefix
@@ -2153,7 +2234,7 @@ def install_root_node(spec, unsigned=False, force=False, sha256=None):
     # don't print long padded paths while extracting/relocating binaries
     with spack.util.path.filter_padding():
         tty.msg('Installing "{0}" from a buildcache'.format(spec.format()))
-        extract_tarball(spec, download_result, unsigned, force)
+        extract_tarball(spec, download_result, force)
         spack.hooks.post_install(spec, False)
         spack.store.STORE.db.add(spec, spack.store.STORE.layout)
 
@@ -2184,10 +2265,10 @@ def try_direct_fetch(spec, mirrors=None):
 
     for mirror in binary_mirrors:
         buildcache_fetch_url_json = url_util.join(
-            mirror.fetch_url, _build_cache_relative_path, specfile_name
+            mirror.fetch_url, BUILD_CACHE_RELATIVE_PATH, specfile_name
         )
         buildcache_fetch_url_signed_json = url_util.join(
-            mirror.fetch_url, _build_cache_relative_path, signed_specfile_name
+            mirror.fetch_url, BUILD_CACHE_RELATIVE_PATH, signed_specfile_name
         )
         try:
             _, _, fs = web_util.read_from_url(buildcache_fetch_url_signed_json)
@@ -2292,7 +2373,7 @@ def get_keys(install=False, trust=False, force=False, mirrors=None):
     for mirror in mirror_collection.values():
         fetch_url = mirror.fetch_url
         keys_url = url_util.join(
-            fetch_url, _build_cache_relative_path, _build_cache_keys_relative_path
+            fetch_url, BUILD_CACHE_RELATIVE_PATH, BUILD_CACHE_KEYS_RELATIVE_PATH
         )
         keys_index = url_util.join(keys_url, "index.json")
 
@@ -2357,7 +2438,7 @@ def push_keys(*mirrors, **kwargs):
         for mirror in mirrors:
             push_url = getattr(mirror, "push_url", mirror)
             keys_url = url_util.join(
-                push_url, _build_cache_relative_path, _build_cache_keys_relative_path
+                push_url, BUILD_CACHE_RELATIVE_PATH, BUILD_CACHE_KEYS_RELATIVE_PATH
             )
             keys_local = url_util.local_file_path(keys_url)
 
@@ -2495,11 +2576,11 @@ def download_buildcache_entry(file_descriptions, mirror_url=None):
         )
 
     if mirror_url:
-        mirror_root = os.path.join(mirror_url, _build_cache_relative_path)
+        mirror_root = os.path.join(mirror_url, BUILD_CACHE_RELATIVE_PATH)
         return _download_buildcache_entry(mirror_root, file_descriptions)
 
     for mirror in spack.mirror.MirrorCollection(binary=True).values():
-        mirror_root = os.path.join(mirror.fetch_url, _build_cache_relative_path)
+        mirror_root = os.path.join(mirror.fetch_url, BUILD_CACHE_RELATIVE_PATH)
 
         if _download_buildcache_entry(mirror_root, file_descriptions):
             return True
@@ -2590,7 +2671,7 @@ def __init__(self, url, local_hash, urlopen=web_util.urlopen):
 
     def get_remote_hash(self):
         # Failure to fetch index.json.hash is not fatal
-        url_index_hash = url_util.join(self.url, _build_cache_relative_path, "index.json.hash")
+        url_index_hash = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json.hash")
         try:
             response = self.urlopen(urllib.request.Request(url_index_hash, headers=self.headers))
         except urllib.error.URLError:
@@ -2611,7 +2692,7 @@ def conditional_fetch(self) -> FetchIndexResult:
             return FetchIndexResult(etag=None, hash=None, data=None, fresh=True)
 
         # Otherwise, download index.json
-        url_index = url_util.join(self.url, _build_cache_relative_path, "index.json")
+        url_index = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json")
 
         try:
             response = self.urlopen(urllib.request.Request(url_index, headers=self.headers))
@@ -2655,7 +2736,7 @@ def __init__(self, url, etag, urlopen=web_util.urlopen):
 
     def conditional_fetch(self) -> FetchIndexResult:
         # Just do a conditional fetch immediately
-        url = url_util.join(self.url, _build_cache_relative_path, "index.json")
+        url = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json")
         headers = {
             "User-Agent": web_util.SPACK_USER_AGENT,
             "If-None-Match": '"{}"'.format(self.etag),

lib/spack/spack/bootstrap/_common.py

@@ -16,6 +16,7 @@
 import llnl.util.filesystem as fs
 from llnl.util import tty
 
+import spack.platforms
 import spack.store
 import spack.util.environment
 import spack.util.executable
@@ -206,16 +207,19 @@ def _root_spec(spec_str: str) -> str:
     """Add a proper compiler and target to a spec used during bootstrapping.
 
     Args:
-        spec_str (str): spec to be bootstrapped. Must be without compiler and target.
+        spec_str: spec to be bootstrapped. Must be without compiler and target.
     """
-    # Add a proper compiler hint to the root spec. We use GCC for
-    # everything but MacOS and Windows.
-    if str(spack.platforms.host()) == "darwin":
+    # Add a compiler requirement to the root spec.
+    platform = str(spack.platforms.host())
+    if platform == "darwin":
         spec_str += " %apple-clang"
-    elif str(spack.platforms.host()) == "windows":
-        spec_str += " %msvc"
-    else:
+    elif platform == "windows":
+        # TODO (johnwparent): Remove version constraint when clingo patch is up
+        spec_str += " %msvc@:19.37"
+    elif platform == "linux":
         spec_str += " %gcc"
+    elif platform == "freebsd":
+        spec_str += " %clang"
 
     target = archspec.cpu.host().family
     spec_str += f" target={target}"

lib/spack/spack/bootstrap/core.py

@@ -386,7 +386,7 @@ def ensure_module_importable_or_raise(module: str, abstract_spec: Optional[str]
     exception_handler = GroupedExceptionHandler()
 
     for current_config in bootstrapping_sources():
-        with exception_handler.forward(current_config["name"]):
+        with exception_handler.forward(current_config["name"], Exception):
             source_is_enabled_or_raise(current_config)
             current_bootstrapper = create_bootstrapper(current_config)
             if current_bootstrapper.try_import(module, abstract_spec):
@@ -441,7 +441,7 @@ def ensure_executables_in_path_or_raise(
     exception_handler = GroupedExceptionHandler()
 
     for current_config in bootstrapping_sources():
-        with exception_handler.forward(current_config["name"]):
+        with exception_handler.forward(current_config["name"], Exception):
             source_is_enabled_or_raise(current_config)
             current_bootstrapper = create_bootstrapper(current_config)
             if current_bootstrapper.try_search_path(executables, abstract_spec):

lib/spack/spack/bootstrap/environment.py

@@ -19,7 +19,6 @@
 import spack.tengine
 import spack.util.cpus
 import spack.util.executable
-from spack.environment import depfile
 
 from ._common import _root_spec
 from .config import root_path, spec_for_current_python, store_path
@@ -86,12 +85,9 @@ def __init__(self) -> None:
         super().__init__(self.environment_root())
 
     def update_installations(self) -> None:
-        """Update the installations of this environment.
-
-        The update is done using a depfile on Linux and macOS, and using the ``install_all``
-        method of environments on Windows.
-        """
-        with tty.SuppressOutput(msg_enabled=False, warn_enabled=False):
+        """Update the installations of this environment."""
+        log_enabled = tty.is_debug() or tty.is_verbose()
+        with tty.SuppressOutput(msg_enabled=log_enabled, warn_enabled=log_enabled):
             specs = self.concretize()
         if specs:
             colorized_specs = [
@@ -100,11 +96,9 @@ def update_installations(self) -> None:
             ]
             tty.msg(f"[BOOTSTRAPPING] Installing dependencies ({', '.join(colorized_specs)})")
             self.write(regenerate=False)
-            if sys.platform == "win32":
+            with tty.SuppressOutput(msg_enabled=log_enabled, warn_enabled=log_enabled):
                 self.install_all()
-            else:
-                self._install_with_depfile()
-            self.write(regenerate=True)
+                self.write(regenerate=True)
 
     def update_syspath_and_environ(self) -> None:
         """Update ``sys.path`` and the PATH, PYTHONPATH environment variables to point to
@@ -122,25 +116,6 @@ def update_syspath_and_environ(self) -> None:
             + [str(x) for x in self.pythonpaths()]
         )
 
-    def _install_with_depfile(self) -> None:
-        model = depfile.MakefileModel.from_env(self)
-        template = spack.tengine.make_environment().get_template(
-            os.path.join("depfile", "Makefile")
-        )
-        makefile = self.environment_root() / "Makefile"
-        makefile.write_text(template.render(model.to_dict()))
-        make = spack.util.executable.which("make")
-        kwargs = {}
-        if not tty.is_debug():
-            kwargs = {"output": os.devnull, "error": os.devnull}
-        make(
-            "-C",
-            str(self.environment_root()),
-            "-j",
-            str(spack.util.cpus.determine_number_of_jobs(parallel=True)),
-            **kwargs,
-        )
-
     def _write_spack_yaml_file(self) -> None:
         tty.msg(
             "[BOOTSTRAPPING] Spack has missing dependencies, creating a bootstrapping environment"
@@ -161,7 +136,7 @@ def _write_spack_yaml_file(self) -> None:
 
 def isort_root_spec() -> str:
     """Return the root spec used to bootstrap isort"""
-    return _root_spec("py-isort@4.3.5:")
+    return _root_spec("py-isort@5")
 
 
 def mypy_root_spec() -> str:

lib/spack/spack/bootstrap/status.py

@@ -66,7 +66,6 @@ def _core_requirements() -> List[RequiredResponseType]:
     _core_system_exes = {
         "make": _missing("make", "required to build software from sources"),
         "patch": _missing("patch", "required to patch source code before building"),
-        "bash": _missing("bash", "required for Spack compiler wrapper"),
         "tar": _missing("tar", "required to manage code archives"),
         "gzip": _missing("gzip", "required to compress/decompress code archives"),
         "unzip": _missing("unzip", "required to compress/decompress code archives"),

lib/spack/spack/build_environment.py

@@ -324,19 +324,29 @@ def set_compiler_environment_variables(pkg, env):
     # ttyout, ttyerr, etc.
     link_dir = spack.paths.build_env_path
 
-    # Set SPACK compiler variables so that our wrapper knows what to call
+    # Set SPACK compiler variables so that our wrapper knows what to
+    # call.  If there is no compiler configured then use a default
+    # wrapper which will emit an error if it is used.
     if compiler.cc:
         env.set("SPACK_CC", compiler.cc)
         env.set("CC", os.path.join(link_dir, compiler.link_paths["cc"]))
+    else:
+        env.set("CC", os.path.join(link_dir, "cc"))
     if compiler.cxx:
         env.set("SPACK_CXX", compiler.cxx)
         env.set("CXX", os.path.join(link_dir, compiler.link_paths["cxx"]))
+    else:
+        env.set("CC", os.path.join(link_dir, "c++"))
     if compiler.f77:
         env.set("SPACK_F77", compiler.f77)
         env.set("F77", os.path.join(link_dir, compiler.link_paths["f77"]))
+    else:
+        env.set("F77", os.path.join(link_dir, "f77"))
     if compiler.fc:
         env.set("SPACK_FC", compiler.fc)
         env.set("FC", os.path.join(link_dir, compiler.link_paths["fc"]))
+    else:
+        env.set("FC", os.path.join(link_dir, "fc"))
 
     # Set SPACK compiler rpath flags so that our wrapper knows what to use
     env.set("SPACK_CC_RPATH_ARG", compiler.cc_rpath_arg)
@@ -743,15 +753,16 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD):
         set_compiler_environment_variables(pkg, env_mods)
         set_wrapper_variables(pkg, env_mods)
 
-    tty.debug("setup_package: grabbing modifications from dependencies")
-    env_mods.extend(setup_context.get_env_modifications())
-    tty.debug("setup_package: collected all modifications from dependencies")
-
-    # architecture specific setup
+    # Platform specific setup goes before package specific setup. This is for setting
+    # defaults like MACOSX_DEPLOYMENT_TARGET on macOS.
     platform = spack.platforms.by_name(pkg.spec.architecture.platform)
     target = platform.target(pkg.spec.architecture.target)
     platform.setup_platform_environment(pkg, env_mods)
 
+    tty.debug("setup_package: grabbing modifications from dependencies")
+    env_mods.extend(setup_context.get_env_modifications())
+    tty.debug("setup_package: collected all modifications from dependencies")
+
     if context == Context.TEST:
         env_mods.prepend_path("PATH", ".")
     elif context == Context.BUILD and not dirty and not env_mods.is_unset("CPATH"):
@@ -1021,6 +1032,11 @@ def get_env_modifications(self) -> EnvironmentModifications:
                     if id(spec) in self.nodes_in_subdag:
                         pkg.setup_dependent_run_environment(run_env_mods, spec)
                 pkg.setup_run_environment(run_env_mods)
+
+                external_env = (dspec.extra_attributes or {}).get("environment", {})
+                if external_env:
+                    run_env_mods.extend(spack.schema.environment.parse(external_env))
+
                 if self.context == Context.BUILD:
                     # Don't let the runtime environment of comiler like dependencies leak into the
                     # build env

lib/spack/spack/build_systems/cached_cmake.py

@@ -9,6 +9,7 @@
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 
+import spack.build_environment
 import spack.builder
 
 from .cmake import CMakeBuilder, CMakePackage
@@ -34,6 +35,11 @@ def cmake_cache_option(name, boolean_value, comment="", force=False):
     return 'set({0} {1} CACHE BOOL "{2}"{3})\n'.format(name, value, comment, force_str)
 
 
+def cmake_cache_filepath(name, value, comment=""):
+    """Generate a string for a cmake cache variable of type FILEPATH"""
+    return 'set({0} "{1}" CACHE FILEPATH "{2}")\n'.format(name, value, comment)
+
+
 class CachedCMakeBuilder(CMakeBuilder):
     #: Phases of a Cached CMake package
     #: Note: the initconfig phase is used for developer builds as a final phase to stop on
@@ -257,6 +263,15 @@ def initconfig_hardware_entries(self):
             entries.append(
                 cmake_cache_path("HIP_CXX_COMPILER", "{0}".format(self.spec["hip"].hipcc))
             )
+            llvm_bin = spec["llvm-amdgpu"].prefix.bin
+            llvm_prefix = spec["llvm-amdgpu"].prefix
+            # Some ROCm systems seem to point to /<path>/rocm-<ver>/ and
+            # others point to /<path>/rocm-<ver>/llvm
+            if os.path.basename(os.path.normpath(llvm_prefix)) != "llvm":
+                llvm_bin = os.path.join(llvm_prefix, "llvm/bin/")
+            entries.append(
+                cmake_cache_filepath("CMAKE_HIP_COMPILER", os.path.join(llvm_bin, "clang++"))
+            )
             archs = self.spec.variants["amdgpu_target"].value
             if archs[0] != "none":
                 arch_str = ";".join(archs)
@@ -271,13 +286,29 @@ def initconfig_hardware_entries(self):
     def std_initconfig_entries(self):
         cmake_prefix_path_env = os.environ["CMAKE_PREFIX_PATH"]
         cmake_prefix_path = cmake_prefix_path_env.replace(os.pathsep, ";")
+        cmake_rpaths_env = spack.build_environment.get_rpaths(self.pkg)
+        cmake_rpaths_path = ";".join(cmake_rpaths_env)
+        complete_rpath_list = cmake_rpaths_path
+        if "SPACK_COMPILER_EXTRA_RPATHS" in os.environ:
+            spack_extra_rpaths_env = os.environ["SPACK_COMPILER_EXTRA_RPATHS"]
+            spack_extra_rpaths_path = spack_extra_rpaths_env.replace(os.pathsep, ";")
+            complete_rpath_list = "{0};{1}".format(complete_rpath_list, spack_extra_rpaths_path)
+
+        if "SPACK_COMPILER_IMPLICIT_RPATHS" in os.environ:
+            spack_implicit_rpaths_env = os.environ["SPACK_COMPILER_IMPLICIT_RPATHS"]
+            spack_implicit_rpaths_path = spack_implicit_rpaths_env.replace(os.pathsep, ";")
+            complete_rpath_list = "{0};{1}".format(complete_rpath_list, spack_implicit_rpaths_path)
+
         return [
             "#------------------{0}".format("-" * 60),
             "# !!!! This is a generated file, edit at own risk !!!!",
             "#------------------{0}".format("-" * 60),
             "# CMake executable path: {0}".format(self.pkg.spec["cmake"].command.path),
             "#------------------{0}\n".format("-" * 60),
-            cmake_cache_path("CMAKE_PREFIX_PATH", cmake_prefix_path),
+            cmake_cache_string("CMAKE_PREFIX_PATH", cmake_prefix_path),
+            cmake_cache_string("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON"),
+            cmake_cache_string("CMAKE_BUILD_RPATH", complete_rpath_list),
+            cmake_cache_string("CMAKE_INSTALL_RPATH", complete_rpath_list),
             self.define_cmake_cache_from_variant("CMAKE_BUILD_TYPE", "build_type"),
         ]
 

lib/spack/spack/build_systems/cargo.py

@@ -0,0 +1,89 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import inspect
+
+import llnl.util.filesystem as fs
+
+import spack.builder
+import spack.package_base
+from spack.directives import build_system, depends_on
+from spack.multimethod import when
+
+from ._checks import BaseBuilder, execute_install_time_tests
+
+
+class CargoPackage(spack.package_base.PackageBase):
+    """Specialized class for packages built using a Makefiles."""
+
+    #: This attribute is used in UI queries that need to know the build
+    #: system base class
+    build_system_class = "CargoPackage"
+
+    build_system("cargo")
+
+    with when("build_system=cargo"):
+        depends_on("rust", type="build")
+
+
+@spack.builder.builder("cargo")
+class CargoBuilder(BaseBuilder):
+    """The Cargo builder encodes the most common way of building software with
+    a rust Cargo.toml file. It has two phases that can be overridden, if need be:
+
+            1. :py:meth:`~.CargoBuilder.build`
+            2. :py:meth:`~.CargoBuilder.install`
+
+    For a finer tuning you may override:
+
+        +-----------------------------------------------+----------------------+
+        | **Method**                                    | **Purpose**          |
+        +===============================================+======================+
+        | :py:meth:`~.CargoBuilder.build_args`          | Specify arguments    |
+        |                                               | to ``cargo install`` |
+        +-----------------------------------------------+----------------------+
+        | :py:meth:`~.CargoBuilder.check_args`          | Specify arguments    |
+        |                                               | to ``cargo test``    |
+        +-----------------------------------------------+----------------------+
+    """
+
+    phases = ("build", "install")
+
+    #: Callback names for install-time test
+    install_time_test_callbacks = ["check"]
+
+    @property
+    def build_directory(self):
+        """Return the directory containing the main Cargo.toml."""
+        return self.pkg.stage.source_path
+
+    @property
+    def build_args(self):
+        """Arguments for ``cargo build``."""
+        return []
+
+    @property
+    def check_args(self):
+        """Argument for ``cargo test`` during check phase"""
+        return []
+
+    def build(self, pkg, spec, prefix):
+        """Runs ``cargo install`` in the source directory"""
+        with fs.working_dir(self.build_directory):
+            inspect.getmodule(pkg).cargo(
+                "install", "--root", "out", "--path", ".", *self.build_args
+            )
+
+    def install(self, pkg, spec, prefix):
+        """Copy build files into package prefix."""
+        with fs.working_dir(self.build_directory):
+            fs.install_tree("out", prefix)
+
+    spack.builder.run_after("install")(execute_install_time_tests)
+
+    def check(self):
+        """Run "cargo test"."""
+        with fs.working_dir(self.build_directory):
+            inspect.getmodule(self.pkg).cargo("test", *self.check_args)

lib/spack/spack/build_systems/cuda.py

@@ -136,12 +136,12 @@ def cuda_flags(arch_list):
         conflicts("%gcc@11:", when="+cuda ^cuda@:11.4.0")
         conflicts("%gcc@11.2:", when="+cuda ^cuda@:11.5")
         conflicts("%gcc@12:", when="+cuda ^cuda@:11.8")
-        conflicts("%gcc@13:", when="+cuda ^cuda@:12.1")
+        conflicts("%gcc@13:", when="+cuda ^cuda@:12.3")
         conflicts("%clang@12:", when="+cuda ^cuda@:11.4.0")
         conflicts("%clang@13:", when="+cuda ^cuda@:11.5")
         conflicts("%clang@14:", when="+cuda ^cuda@:11.7")
         conflicts("%clang@15:", when="+cuda ^cuda@:12.0")
-        conflicts("%clang@16:", when="+cuda ^cuda@:12.1")
+        conflicts("%clang@16:", when="+cuda ^cuda@:12.3")
 
         # https://gist.github.com/ax3l/9489132#gistcomment-3860114
         conflicts("%gcc@10", when="+cuda ^cuda@:11.4.0")

lib/spack/spack/build_systems/go.py

@@ -0,0 +1,98 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import inspect
+
+import llnl.util.filesystem as fs
+
+import spack.builder
+import spack.package_base
+from spack.directives import build_system, extends
+from spack.multimethod import when
+
+from ._checks import BaseBuilder, execute_install_time_tests
+
+
+class GoPackage(spack.package_base.PackageBase):
+    """Specialized class for packages built using the Go toolchain."""
+
+    #: This attribute is used in UI queries that need to know the build
+    #: system base class
+    build_system_class = "GoPackage"
+
+    #: Legacy buildsystem attribute used to deserialize and install old specs
+    legacy_buildsystem = "go"
+
+    build_system("go")
+
+    with when("build_system=go"):
+        # TODO: this seems like it should be depends_on, see
+        # setup_dependent_build_environment in go for why I kept it like this
+        extends("go@1.14:", type="build")
+
+
+@spack.builder.builder("go")
+class GoBuilder(BaseBuilder):
+    """The Go builder encodes the most common way of building software with
+    a golang go.mod file. It has two phases that can be overridden, if need be:
+
+            1. :py:meth:`~.GoBuilder.build`
+            2. :py:meth:`~.GoBuilder.install`
+
+    For a finer tuning you may override:
+
+        +-----------------------------------------------+--------------------+
+        | **Method**                                    | **Purpose**        |
+        +===============================================+====================+
+        | :py:meth:`~.GoBuilder.build_args`             | Specify arguments  |
+        |                                               | to ``go build``    |
+        +-----------------------------------------------+--------------------+
+        | :py:meth:`~.GoBuilder.check_args`             | Specify arguments  |
+        |                                               | to ``go test``     |
+        +-----------------------------------------------+--------------------+
+    """
+
+    phases = ("build", "install")
+
+    #: Callback names for install-time test
+    install_time_test_callbacks = ["check"]
+
+    def setup_build_environment(self, env):
+        env.set("GO111MODULE", "on")
+        env.set("GOTOOLCHAIN", "local")
+
+    @property
+    def build_directory(self):
+        """Return the directory containing the main go.mod."""
+        return self.pkg.stage.source_path
+
+    @property
+    def build_args(self):
+        """Arguments for ``go build``."""
+        # Pass ldflags -s = --strip-all and -w = --no-warnings by default
+        return ["-ldflags", "-s -w", "-o", f"{self.pkg.name}"]
+
+    @property
+    def check_args(self):
+        """Argument for ``go test`` during check phase"""
+        return []
+
+    def build(self, pkg, spec, prefix):
+        """Runs ``go build`` in the source directory"""
+        with fs.working_dir(self.build_directory):
+            inspect.getmodule(pkg).go("build", *self.build_args)
+
+    def install(self, pkg, spec, prefix):
+        """Install built binaries into prefix bin."""
+        with fs.working_dir(self.build_directory):
+            fs.mkdirp(prefix.bin)
+            fs.install(pkg.name, prefix.bin)
+
+    spack.builder.run_after("install")(execute_install_time_tests)
+
+    def check(self):
+        """Run ``go test .`` in the source directory"""
+        with fs.working_dir(self.build_directory):
+            inspect.getmodule(self.pkg).go("test", *self.check_args)

lib/spack/spack/build_systems/oneapi.py

@@ -7,13 +7,12 @@
 import os
 import platform
 import shutil
-from os.path import basename, dirname, isdir
+from os.path import basename, isdir
 
-from llnl.util.filesystem import find_headers, find_libraries, join_path
+from llnl.util.filesystem import HeaderList, find_libraries, join_path, mkdirp
 from llnl.util.link_tree import LinkTree
 
 from spack.directives import conflicts, variant
-from spack.package import mkdirp
 from spack.util.environment import EnvironmentModifications
 from spack.util.executable import Executable
 
@@ -56,10 +55,21 @@ def component_dir(self):
         """Subdirectory for this component in the install prefix."""
         raise NotImplementedError
 
+    @property
+    def v2_layout_versions(self):
+        """Version that implements the v2 directory layout."""
+        raise NotImplementedError
+
+    @property
+    def v2_layout(self):
+        """Returns true if this version implements the v2 directory layout."""
+        return self.spec.satisfies(self.v2_layout_versions)
+
     @property
     def component_prefix(self):
         """Path to component <prefix>/<component>/<version>."""
-        return self.prefix.join(join_path(self.component_dir, self.spec.version))
+        v = self.spec.version.up_to(2) if self.v2_layout else self.spec.version
+        return self.prefix.join(self.component_dir).join(str(v))
 
     @property
     def env_script_args(self):
@@ -113,8 +123,9 @@ def install_component(self, installer_path):
                 shutil.rmtree("/var/intel/installercache", ignore_errors=True)
 
         # Some installers have a bug and do not return an error code when failing
-        if not isdir(join_path(self.prefix, self.component_dir)):
-            raise RuntimeError("install failed")
+        install_dir = self.component_prefix
+        if not isdir(install_dir):
+            raise RuntimeError("install failed to directory: {0}".format(install_dir))
 
     def setup_run_environment(self, env):
         """Adds environment variables to the generated module file.
@@ -129,7 +140,7 @@ def setup_run_environment(self, env):
         if "~envmods" not in self.spec:
             env.extend(
                 EnvironmentModifications.from_sourcing_file(
-                    join_path(self.component_prefix, "env", "vars.sh"), *self.env_script_args
+                    self.component_prefix.env.join("vars.sh"), *self.env_script_args
                 )
             )
 
@@ -168,16 +179,40 @@ class IntelOneApiLibraryPackage(IntelOneApiPackage):
 
     """
 
+    def header_directories(self, dirs):
+        h = HeaderList([])
+        h.directories = dirs
+        return h
+
     @property
     def headers(self):
-        include_path = join_path(self.component_prefix, "include")
-        return find_headers("*", include_path, recursive=True)
+        return self.header_directories(
+            [self.component_prefix.include, self.component_prefix.include.join(self.component_dir)]
+        )
 
     @property
     def libs(self):
-        lib_path = join_path(self.component_prefix, "lib", "intel64")
-        lib_path = lib_path if isdir(lib_path) else dirname(lib_path)
-        return find_libraries("*", root=lib_path, shared=True, recursive=True)
+        # for v2_layout all libraries are in the top level, v1 sometimes put them in intel64
+        return find_libraries("*", root=self.component_prefix.lib, recursive=not self.v2_layout)
+
+
+class IntelOneApiLibraryPackageWithSdk(IntelOneApiPackage):
+    """Base class for Intel oneAPI library packages with SDK components.
+
+    Contains some convenient default implementations for libraries
+    that expose functionality in sdk subdirectories.
+    Implement the method directly in the package if something
+    different is needed.
+
+    """
+
+    @property
+    def headers(self):
+        return self.header_directories([self.component_prefix.sdk.include])
+
+    @property
+    def libs(self):
+        return find_libraries("*", self.component_prefix.sdk.lib64)
 
 
 class IntelOneApiStaticLibraryList:
@@ -212,3 +247,7 @@ def link_flags(self):
     @property
     def ld_flags(self):
         return "{0} {1}".format(self.search_flags, self.link_flags)
+
+
+#: Tuple of Intel math libraries, exported to packages
+INTEL_MATH_LIBRARIES = ("intel-mkl", "intel-oneapi-mkl", "intel-parallel-studio")

lib/spack/spack/build_systems/perl.py

@@ -10,13 +10,12 @@
 import spack.builder
 import spack.package_base
 from spack.directives import build_system, extends
-from spack.package_base import PackageBase
 from spack.util.executable import Executable
 
 from ._checks import BaseBuilder, execute_build_time_tests
 
 
-class PerlPackage(PackageBase):
+class PerlPackage(spack.package_base.PackageBase):
     """Specialized class for packages that are built using Perl."""
 
     #: This attribute is used in UI queries that need to know the build
@@ -61,6 +60,30 @@ class PerlBuilder(BaseBuilder):
     #: Callback names for build-time test
     build_time_test_callbacks = ["check"]
 
+    @property
+    def build_method(self):
+        """Searches the package for either a Makefile.PL or Build.PL.
+
+        Raises:
+            RuntimeError: if neither Makefile.PL nor Build.PL exist
+        """
+        if os.path.isfile("Makefile.PL"):
+            build_method = "Makefile.PL"
+        elif os.path.isfile("Build.PL"):
+            build_method = "Build.PL"
+        else:
+            raise RuntimeError("Unknown build_method for perl package")
+        return build_method
+
+    @property
+    def build_executable(self):
+        """Returns the executable method to build the perl package"""
+        if self.build_method == "Makefile.PL":
+            build_executable = inspect.getmodule(self.pkg).make
+        elif self.build_method == "Build.PL":
+            build_executable = Executable(os.path.join(self.pkg.stage.source_path, "Build"))
+        return build_executable
+
     def configure_args(self):
         """List of arguments passed to :py:meth:`~.PerlBuilder.configure`.
 
@@ -73,19 +96,7 @@ def configure(self, pkg, spec, prefix):
         """Run Makefile.PL or Build.PL with arguments consisting of
         an appropriate installation base directory followed by the
         list returned by :py:meth:`~.PerlBuilder.configure_args`.
-
-        Raises:
-            RuntimeError: if neither Makefile.PL nor Build.PL exist
         """
-        if os.path.isfile("Makefile.PL"):
-            self.build_method = "Makefile.PL"
-            self.build_executable = inspect.getmodule(self.pkg).make
-        elif os.path.isfile("Build.PL"):
-            self.build_method = "Build.PL"
-            self.build_executable = Executable(os.path.join(self.pkg.stage.source_path, "Build"))
-        else:
-            raise RuntimeError("Unknown build_method for perl package")
-
         if self.build_method == "Makefile.PL":
             options = ["Makefile.PL", "INSTALL_BASE={0}".format(prefix)]
         elif self.build_method == "Build.PL":

lib/spack/spack/build_systems/python.py

@@ -6,13 +6,14 @@
 import os
 import re
 import shutil
-from typing import Optional
+from typing import Iterable, List, Mapping, Optional
 
 import archspec
 
 import llnl.util.filesystem as fs
 import llnl.util.lang as lang
 import llnl.util.tty as tty
+from llnl.util.filesystem import HeaderList, LibraryList
 
 import spack.builder
 import spack.config
@@ -25,14 +26,18 @@
 from spack.directives import build_system, depends_on, extends, maintainers
 from spack.error import NoHeadersError, NoLibrariesError
 from spack.install_test import test_part
+from spack.spec import Spec
+from spack.util.prefix import Prefix
 
 from ._checks import BaseBuilder, execute_install_time_tests
 
 
-def _flatten_dict(dictionary):
+def _flatten_dict(dictionary: Mapping[str, object]) -> Iterable[str]:
     """Iterable that yields KEY=VALUE paths through a dictionary.
+
     Args:
         dictionary: Possibly nested dictionary of arbitrary keys and values.
+
     Yields:
         A single path through the dictionary.
     """
@@ -50,7 +55,7 @@ class PythonExtension(spack.package_base.PackageBase):
     maintainers("adamjstewart")
 
     @property
-    def import_modules(self):
+    def import_modules(self) -> Iterable[str]:
         """Names of modules that the Python package provides.
 
         These are used to test whether or not the installation succeeded.
@@ -65,7 +70,7 @@ def import_modules(self):
         detected, this property can be overridden by the package.
 
         Returns:
-            list: list of strings of module names
+            List of strings of module names.
         """
         modules = []
         pkg = self.spec["python"].package
@@ -102,14 +107,14 @@ def import_modules(self):
         return modules
 
     @property
-    def skip_modules(self):
+    def skip_modules(self) -> Iterable[str]:
         """Names of modules that should be skipped when running tests.
 
         These are a subset of import_modules. If a module has submodules,
         they are skipped as well (meaning a.b is skipped if a is contained).
 
         Returns:
-            list: list of strings of module names
+            List of strings of module names.
         """
         return []
 
@@ -185,12 +190,12 @@ def remove_files_from_view(self, view, merge_map):
 
         view.remove_files(to_remove)
 
-    def test_imports(self):
+    def test_imports(self) -> None:
         """Attempts to import modules of the installed package."""
 
         # Make sure we are importing the installed modules,
         # not the ones in the source directory
-        python = inspect.getmodule(self).python
+        python = inspect.getmodule(self).python  # type: ignore[union-attr]
         for module in self.import_modules:
             with test_part(
                 self,
@@ -315,24 +320,27 @@ class PythonPackage(PythonExtension):
     py_namespace: Optional[str] = None
 
     @lang.classproperty
-    def homepage(cls):
+    def homepage(cls) -> Optional[str]:  # type: ignore[override]
         if cls.pypi:
             name = cls.pypi.split("/")[0]
-            return "https://pypi.org/project/" + name + "/"
+            return f"https://pypi.org/project/{name}/"
+        return None
 
     @lang.classproperty
-    def url(cls):
+    def url(cls) -> Optional[str]:
         if cls.pypi:
-            return "https://files.pythonhosted.org/packages/source/" + cls.pypi[0] + "/" + cls.pypi
+            return f"https://files.pythonhosted.org/packages/source/{cls.pypi[0]}/{cls.pypi}"
+        return None
 
     @lang.classproperty
-    def list_url(cls):
+    def list_url(cls) -> Optional[str]:  # type: ignore[override]
         if cls.pypi:
             name = cls.pypi.split("/")[0]
-            return "https://pypi.org/simple/" + name + "/"
+            return f"https://pypi.org/simple/{name}/"
+        return None
 
     @property
-    def headers(self):
+    def headers(self) -> HeaderList:
         """Discover header files in platlib."""
 
         # Remove py- prefix in package name
@@ -350,7 +358,7 @@ def headers(self):
         raise NoHeadersError(msg.format(self.spec.name, include, platlib))
 
     @property
-    def libs(self):
+    def libs(self) -> LibraryList:
         """Discover libraries in platlib."""
 
         # Remove py- prefix in package name
@@ -384,7 +392,7 @@ class PythonPipBuilder(BaseBuilder):
     install_time_test_callbacks = ["test"]
 
     @staticmethod
-    def std_args(cls):
+    def std_args(cls) -> List[str]:
         return [
             # Verbose
             "-vvv",
@@ -409,7 +417,7 @@ def std_args(cls):
         ]
 
     @property
-    def build_directory(self):
+    def build_directory(self) -> str:
         """The root directory of the Python package.
 
         This is usually the directory containing one of the following files:
@@ -420,51 +428,51 @@ def build_directory(self):
         """
         return self.pkg.stage.source_path
 
-    def config_settings(self, spec, prefix):
+    def config_settings(self, spec: Spec, prefix: Prefix) -> Mapping[str, object]:
         """Configuration settings to be passed to the PEP 517 build backend.
 
         Requires pip 22.1 or newer for keys that appear only a single time,
         or pip 23.1 or newer if the same key appears multiple times.
 
         Args:
-            spec (spack.spec.Spec): build spec
-            prefix (spack.util.prefix.Prefix): installation prefix
+            spec: Build spec.
+            prefix: Installation prefix.
 
         Returns:
-            dict: Possibly nested dictionary of KEY, VALUE settings
+            Possibly nested dictionary of KEY, VALUE settings.
         """
         return {}
 
-    def install_options(self, spec, prefix):
+    def install_options(self, spec: Spec, prefix: Prefix) -> Iterable[str]:
         """Extra arguments to be supplied to the setup.py install command.
 
         Requires pip 23.0 or older.
 
         Args:
-            spec (spack.spec.Spec): build spec
-            prefix (spack.util.prefix.Prefix): installation prefix
+            spec: Build spec.
+            prefix: Installation prefix.
 
         Returns:
-            list: list of options
+            List of options.
         """
         return []
 
-    def global_options(self, spec, prefix):
+    def global_options(self, spec: Spec, prefix: Prefix) -> Iterable[str]:
         """Extra global options to be supplied to the setup.py call before the install
         or bdist_wheel command.
 
         Deprecated in pip 23.1.
 
         Args:
-            spec (spack.spec.Spec): build spec
-            prefix (spack.util.prefix.Prefix): installation prefix
+            spec: Build spec.
+            prefix: Installation prefix.
 
         Returns:
-            list: list of options
+            List of options.
         """
         return []
 
-    def install(self, pkg, spec, prefix):
+    def install(self, pkg: PythonPackage, spec: Spec, prefix: Prefix) -> None:
         """Install everything from build directory."""
 
         args = PythonPipBuilder.std_args(pkg) + [f"--prefix={prefix}"]

lib/spack/spack/build_systems/rocm.py

@@ -108,6 +108,8 @@ class ROCmPackage(PackageBase):
         "gfx90a:xnack+",
         "gfx90c",
         "gfx940",
+        "gfx941",
+        "gfx942",
         "gfx1010",
         "gfx1011",
         "gfx1012",
@@ -168,6 +170,8 @@ def hip_flags(amdgpu_target):
     depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx90a:xnack-")
     depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx90a:xnack+")
     depends_on("llvm-amdgpu@5.2.0:", when="amdgpu_target=gfx940")
+    depends_on("llvm-amdgpu@5.7.0:", when="amdgpu_target=gfx941")
+    depends_on("llvm-amdgpu@5.7.0:", when="amdgpu_target=gfx942")
     depends_on("llvm-amdgpu@4.5.0:", when="amdgpu_target=gfx1013")
     depends_on("llvm-amdgpu@3.8.0:", when="amdgpu_target=gfx1030")
     depends_on("llvm-amdgpu@3.9.0:", when="amdgpu_target=gfx1031")

lib/spack/spack/ci.py

@@ -46,7 +46,22 @@
 from spack.reporters import CDash, CDashConfiguration
 from spack.reporters.cdash import build_stamp as cdash_build_stamp
 
-JOB_RETRY_CONDITIONS = ["always"]
+# See https://docs.gitlab.com/ee/ci/yaml/#retry for descriptions of conditions
+JOB_RETRY_CONDITIONS = [
+    # "always",
+    "unknown_failure",
+    "script_failure",
+    "api_failure",
+    "stuck_or_timeout_failure",
+    "runner_system_failure",
+    "runner_unsupported",
+    "stale_schedule",
+    # "job_execution_timeout",
+    "archived_failure",
+    "unmet_prerequisites",
+    "scheduler_failure",
+    "data_integrity_failure",
+]
 
 TEMP_STORAGE_MIRROR_NAME = "ci_temporary_mirror"
 SPACK_RESERVED_TAGS = ["public", "protected", "notary"]
@@ -1238,6 +1253,7 @@ def main_script_replacements(cmd):
                 op=lambda cmd: cmd.replace("mirror_prefix", temp_storage_url_prefix),
             )
 
+            cleanup_job["dependencies"] = []
             output_object["cleanup"] = cleanup_job
 
         if (
@@ -1261,6 +1277,7 @@ def main_script_replacements(cmd):
                 if buildcache_destination
                 else remote_mirror_override or remote_mirror_url
             )
+            signing_job["dependencies"] = []
 
             output_object["sign-pkgs"] = signing_job
 
@@ -1281,6 +1298,7 @@ def main_script_replacements(cmd):
             final_job["when"] = "always"
             final_job["retry"] = service_job_retries
             final_job["interruptible"] = True
+            final_job["dependencies"] = []
 
             output_object["rebuild-index"] = final_job
 

lib/spack/spack/cmd/__init__.py

@@ -6,10 +6,8 @@
 import argparse
 import os
 import re
-import shlex
 import sys
-from textwrap import dedent
-from typing import List, Match, Tuple
+from typing import List, Union
 
 import llnl.string
 import llnl.util.tty as tty
@@ -147,89 +145,37 @@ def get_command(cmd_name):
     return getattr(get_module(cmd_name), pname)
 
 
-class _UnquotedFlags:
-    """Use a heuristic in `.extract()` to detect whether the user is trying to set
-    multiple flags like the docker ENV attribute allows (e.g. 'cflags=-Os -pipe').
+def quote_kvp(string: str) -> str:
+    """For strings like ``name=value`` or ``name==value``, quote and escape the value if needed.
 
-    If the heuristic finds a match (which can be checked with `__bool__()`), a warning
-    message explaining how to quote multiple flags correctly can be generated with
-    `.report()`.
+    This is a compromise to respect quoting of key-value pairs on the CLI. The shell
+    strips quotes from quoted arguments, so we cannot know *exactly* how CLI arguments
+    were quoted. To compensate, we re-add quotes around anything staritng with ``name=``
+    or ``name==``, and we assume the rest of the argument is the value. This covers the
+    common cases of passign flags, e.g., ``cflags="-O2 -g"`` on the command line.
     """
+    match = spack.parser.SPLIT_KVP.match(string)
+    if not match:
+        return string
 
-    flags_arg_pattern = re.compile(
-        r'^({0})=([^\'"].*)$'.format("|".join(spack.spec.FlagMap.valid_compiler_flags()))
-    )
-
-    def __init__(self, all_unquoted_flag_pairs: List[Tuple[Match[str], str]]):
-        self._flag_pairs = all_unquoted_flag_pairs
-
-    def __bool__(self) -> bool:
-        return bool(self._flag_pairs)
-
-    @classmethod
-    def extract(cls, sargs: str) -> "_UnquotedFlags":
-        all_unquoted_flag_pairs: List[Tuple[Match[str], str]] = []
-        prev_flags_arg = None
-        for arg in shlex.split(sargs):
-            if prev_flags_arg is not None:
-                all_unquoted_flag_pairs.append((prev_flags_arg, arg))
-            prev_flags_arg = cls.flags_arg_pattern.match(arg)
-        return cls(all_unquoted_flag_pairs)
-
-    def report(self) -> str:
-        single_errors = [
-            "({0}) {1} {2} => {3}".format(
-                i + 1,
-                match.group(0),
-                next_arg,
-                '{0}="{1} {2}"'.format(match.group(1), match.group(2), next_arg),
-            )
-            for i, (match, next_arg) in enumerate(self._flag_pairs)
-        ]
-        return dedent(
-            """\
-        Some compiler or linker flags were provided without quoting their arguments,
-        which now causes spack to try to parse the *next* argument as a spec component
-        such as a variant instead of an additional compiler or linker flag. If the
-        intent was to set multiple flags, try quoting them together as described below.
-
-        Possible flag quotation errors (with the correctly-quoted version after the =>):
-        {0}"""
-        ).format("\n".join(single_errors))
+    key, delim, value = match.groups()
+    return f"{key}{delim}{spack.parser.quote_if_needed(value)}"
 
 
-def parse_specs(args, **kwargs):
+def parse_specs(
+    args: Union[str, List[str]], concretize: bool = False, tests: bool = False
+) -> List[spack.spec.Spec]:
     """Convenience function for parsing arguments from specs.  Handles common
     exceptions and dies if there are errors.
     """
-    concretize = kwargs.get("concretize", False)
-    normalize = kwargs.get("normalize", False)
-    tests = kwargs.get("tests", False)
+    args = [args] if isinstance(args, str) else args
+    arg_string = " ".join([quote_kvp(arg) for arg in args])
 
-    sargs = args
-    if not isinstance(args, str):
-        sargs = " ".join(args)
-    unquoted_flags = _UnquotedFlags.extract(sargs)
-
-    try:
-        specs = spack.parser.parse(sargs)
-        for spec in specs:
-            if concretize:
-                spec.concretize(tests=tests)  # implies normalize
-            elif normalize:
-                spec.normalize(tests=tests)
-        return specs
-
-    except spack.error.SpecError as e:
-        msg = e.message
-        if e.long_message:
-            msg += e.long_message
-        # Unquoted flags will be read as a variant or hash
-        if unquoted_flags and ("variant" in msg or "hash" in msg):
-            msg += "\n\n"
-            msg += unquoted_flags.report()
-
-        raise spack.error.SpackError(msg) from e
+    specs = spack.parser.parse(arg_string)
+    for spec in specs:
+        if concretize:
+            spec.concretize(tests=tests)
+    return specs
 
 
 def matching_spec_from_env(spec):

lib/spack/spack/cmd/add.py

@@ -6,7 +6,7 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
+from spack.cmd.common import arguments
 
 description = "add a spec to an environment"
 section = "environments"

lib/spack/spack/cmd/audit.py

@@ -2,6 +2,8 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import warnings
+
 import llnl.util.tty as tty
 import llnl.util.tty.colify
 import llnl.util.tty.color as cl
@@ -52,8 +54,10 @@ def setup_parser(subparser):
 
 
 def configs(parser, args):
-    reports = spack.audit.run_group(args.subcommand)
-    _process_reports(reports)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        reports = spack.audit.run_group(args.subcommand)
+        _process_reports(reports)
 
 
 def packages(parser, args):

lib/spack/spack/cmd/bootstrap.py

@@ -15,13 +15,13 @@
 import spack.bootstrap
 import spack.bootstrap.config
 import spack.bootstrap.core
-import spack.cmd.common.arguments
 import spack.config
 import spack.main
 import spack.mirror
 import spack.spec
 import spack.stage
 import spack.util.path
+from spack.cmd.common import arguments
 
 description = "manage bootstrap configuration"
 section = "system"
@@ -68,12 +68,8 @@
 
 
 def _add_scope_option(parser):
-    scopes = spack.config.scopes()
     parser.add_argument(
-        "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        help="configuration scope to read/modify",
+        "--scope", action=arguments.ConfigScope, help="configuration scope to read/modify"
     )
 
 
@@ -106,7 +102,7 @@ def setup_parser(subparser):
     disable.add_argument("name", help="name of the source to be disabled", nargs="?", default=None)
 
     reset = sp.add_parser("reset", help="reset bootstrapping configuration to Spack defaults")
-    spack.cmd.common.arguments.add_common_arguments(reset, ["yes_to_all"])
+    arguments.add_common_arguments(reset, ["yes_to_all"])
 
     root = sp.add_parser("root", help="get/set the root bootstrap directory")
     _add_scope_option(root)

lib/spack/spack/cmd/buildcache.py

@@ -21,7 +21,6 @@
 
 import spack.binary_distribution as bindist
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
 import spack.error
@@ -38,8 +37,10 @@
 import spack.util.crypto
 import spack.util.url as url_util
 import spack.util.web as web_util
+from spack import traverse
 from spack.build_environment import determine_number_of_jobs
 from spack.cmd import display_specs
+from spack.cmd.common import arguments
 from spack.oci.image import (
     Digest,
     ImageReference,
@@ -76,7 +77,19 @@ def setup_parser(subparser: argparse.ArgumentParser):
     )
     push_sign = push.add_mutually_exclusive_group(required=False)
     push_sign.add_argument(
-        "--unsigned", "-u", action="store_true", help="push unsigned buildcache tarballs"
+        "--unsigned",
+        "-u",
+        action="store_false",
+        dest="signed",
+        default=None,
+        help="push unsigned buildcache tarballs",
+    )
+    push_sign.add_argument(
+        "--signed",
+        action="store_true",
+        dest="signed",
+        default=None,
+        help="push signed buildcache tarballs",
     )
     push_sign.add_argument(
         "--key", "-k", metavar="key", type=str, default=None, help="key for signing"
@@ -110,7 +123,14 @@ def setup_parser(subparser: argparse.ArgumentParser):
         help="stop pushing on first failure (default is best effort)",
     )
     push.add_argument(
-        "--base-image", default=None, help="specify the base image for the buildcache. "
+        "--base-image", default=None, help="specify the base image for the buildcache"
+    )
+    push.add_argument(
+        "--tag",
+        "-t",
+        default=None,
+        help="when pushing to an OCI registry, tag an image containing all root specs and their "
+        "runtime dependencies",
     )
     arguments.add_common_arguments(push, ["specs", "jobs"])
     push.set_defaults(func=push_fn)
@@ -182,23 +202,22 @@ def setup_parser(subparser: argparse.ArgumentParser):
     )
 
     # used to construct scope arguments below
-    scopes = spack.config.scopes()
-
     check.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope containing mirrors to check",
     )
-    check_spec_or_specfile = check.add_mutually_exclusive_group(required=True)
-    check_spec_or_specfile.add_argument(
+    # Unfortunately there are 3 ways to do the same thing here:
+    check_specs = check.add_mutually_exclusive_group()
+    check_specs.add_argument(
         "-s", "--spec", help="check single spec instead of release specs file"
     )
-    check_spec_or_specfile.add_argument(
+    check_specs.add_argument(
         "--spec-file",
         help="check single spec from json or yaml file instead of release specs file",
     )
+    arguments.add_common_arguments(check, ["specs"])
 
     check.set_defaults(func=check_fn)
 
@@ -320,26 +339,36 @@ def push_fn(args):
         )
 
     if args.specs or args.spec_file:
-        specs = _matching_specs(spack.cmd.parse_specs(args.specs or args.spec_file))
+        roots = _matching_specs(spack.cmd.parse_specs(args.specs or args.spec_file))
     else:
-        specs = spack.cmd.require_active_env("buildcache push").all_specs()
+        roots = spack.cmd.require_active_env(cmd_name="buildcache push").concrete_roots()
 
     if args.allow_root:
         tty.warn(
             "The flag `--allow-root` is the default in Spack 0.21, will be removed in Spack 0.22"
         )
 
+    mirror: spack.mirror.Mirror = args.mirror
+
     # Check if this is an OCI image.
     try:
-        image_ref = spack.oci.oci.image_from_mirror(args.mirror)
+        target_image = spack.oci.oci.image_from_mirror(mirror)
     except ValueError:
-        image_ref = None
+        target_image = None
+
+    push_url = mirror.push_url
+
+    # When neither --signed, --unsigned nor --key are specified, use the mirror's default.
+    if args.signed is None and not args.key:
+        unsigned = not mirror.signed
+    else:
+        unsigned = not (args.key or args.signed)
 
     # For OCI images, we require dependencies to be pushed for now.
-    if image_ref:
+    if target_image:
         if "dependencies" not in args.things_to_install:
             tty.die("Dependencies must be pushed for OCI images.")
-        if not args.unsigned:
+        if not unsigned:
             tty.warn(
                 "Code signing is currently not supported for OCI images. "
                 "Use --unsigned to silence this warning."
@@ -347,26 +376,48 @@ def push_fn(args):
 
     # This is a list of installed, non-external specs.
     specs = bindist.specs_to_be_packaged(
-        specs,
+        roots,
         root="package" in args.things_to_install,
         dependencies="dependencies" in args.things_to_install,
     )
 
-    url = args.mirror.push_url
-
     # When pushing multiple specs, print the url once ahead of time, as well as how
     # many specs are being pushed.
     if len(specs) > 1:
-        tty.info(f"Selected {len(specs)} specs to push to {url}")
+        tty.info(f"Selected {len(specs)} specs to push to {push_url}")
 
     failed = []
 
     # TODO: unify this logic in the future.
-    if image_ref:
+    if target_image:
+        base_image = ImageReference.from_string(args.base_image) if args.base_image else None
         with tempfile.TemporaryDirectory(
             dir=spack.stage.get_stage_root()
         ) as tmpdir, _make_pool() as pool:
-            skipped = _push_oci(args, image_ref, specs, tmpdir, pool)
+            skipped, base_images, checksums = _push_oci(
+                target_image=target_image,
+                base_image=base_image,
+                installed_specs_with_deps=specs,
+                force=args.force,
+                tmpdir=tmpdir,
+                pool=pool,
+            )
+
+            # Apart from creating manifests for each individual spec, we allow users to create a
+            # separate image tag for all root specs and their runtime dependencies.
+            if args.tag:
+                tagged_image = target_image.with_tag(args.tag)
+                # _push_oci may not populate base_images if binaries were already in the registry
+                for spec in roots:
+                    _update_base_images(
+                        base_image=base_image,
+                        target_image=target_image,
+                        spec=spec,
+                        base_image_cache=base_images,
+                    )
+                _put_manifest(base_images, checksums, tagged_image, tmpdir, None, None, *roots)
+                tty.info(f"Tagged {tagged_image}")
+
     else:
         skipped = []
 
@@ -374,10 +425,10 @@ def push_fn(args):
             try:
                 bindist.push_or_raise(
                     spec,
-                    url,
+                    push_url,
                     bindist.PushOptions(
                         force=args.force,
-                        unsigned=args.unsigned,
+                        unsigned=unsigned,
                         key=args.key,
                         regenerate_index=args.update_index,
                     ),
@@ -385,7 +436,7 @@ def push_fn(args):
 
                 msg = f"{_progress(i, len(specs))}Pushed {_format_spec(spec)}"
                 if len(specs) == 1:
-                    msg += f" to {url}"
+                    msg += f" to {push_url}"
                 tty.info(msg)
 
             except bindist.NoOverwriteException:
@@ -427,11 +478,11 @@ def push_fn(args):
     # Update the index if requested
     # TODO: remove update index logic out of bindist; should be once after all specs are pushed
     # not once per spec.
-    if image_ref and len(skipped) < len(specs) and args.update_index:
+    if target_image and len(skipped) < len(specs) and args.update_index:
         with tempfile.TemporaryDirectory(
             dir=spack.stage.get_stage_root()
         ) as tmpdir, _make_pool() as pool:
-            _update_index_oci(image_ref, tmpdir, pool)
+            _update_index_oci(target_image, tmpdir, pool)
 
 
 def _get_spack_binary_blob(image_ref: ImageReference) -> Optional[spack.oci.oci.Blob]:
@@ -497,17 +548,21 @@ def _archspec_to_gooarch(spec: spack.spec.Spec) -> str:
 def _put_manifest(
     base_images: Dict[str, Tuple[dict, dict]],
     checksums: Dict[str, spack.oci.oci.Blob],
-    spec: spack.spec.Spec,
     image_ref: ImageReference,
     tmpdir: str,
+    extra_config: Optional[dict],
+    annotations: Optional[dict],
+    *specs: spack.spec.Spec,
 ):
-    architecture = _archspec_to_gooarch(spec)
+    architecture = _archspec_to_gooarch(specs[0])
 
     dependencies = list(
         reversed(
             list(
                 s
-                for s in spec.traverse(order="topo", deptype=("link", "run"), root=True)
+                for s in traverse.traverse_nodes(
+                    specs, order="topo", deptype=("link", "run"), root=True
+                )
                 if not s.external
             )
         )
@@ -516,7 +571,7 @@ def _put_manifest(
     base_manifest, base_config = base_images[architecture]
     env = _retrieve_env_dict_from_config(base_config)
 
-    spack.user_environment.environment_modifications_for_specs(spec).apply_modifications(env)
+    spack.user_environment.environment_modifications_for_specs(*specs).apply_modifications(env)
 
     # Create an oci.image.config file
     config = copy.deepcopy(base_config)
@@ -528,20 +583,14 @@ def _put_manifest(
     # Set the environment variables
     config["config"]["Env"] = [f"{k}={v}" for k, v in env.items()]
 
-    # From the OCI v1.0 spec:
-    # > Any extra fields in the Image JSON struct are considered implementation
-    # > specific and MUST be ignored by any implementations which are unable to
-    # > interpret them.
-    # We use this to store the Spack spec, so we can use it to create an index.
-    spec_dict = spec.to_dict(hash=ht.dag_hash)
-    spec_dict["buildcache_layout_version"] = 1
-    spec_dict["binary_cache_checksum"] = {
-        "hash_algorithm": "sha256",
-        "hash": checksums[spec.dag_hash()].compressed_digest.digest,
-    }
-    config.update(spec_dict)
+    if extra_config:
+        # From the OCI v1.0 spec:
+        # > Any extra fields in the Image JSON struct are considered implementation
+        # > specific and MUST be ignored by any implementations which are unable to
+        # > interpret them.
+        config.update(extra_config)
 
-    config_file = os.path.join(tmpdir, f"{spec.dag_hash()}.config.json")
+    config_file = os.path.join(tmpdir, f"{specs[0].dag_hash()}.config.json")
 
     with open(config_file, "w") as f:
         json.dump(config, f, separators=(",", ":"))
@@ -572,48 +621,69 @@ def _put_manifest(
                 for s in dependencies
             ),
         ],
-        "annotations": {"org.opencontainers.image.description": spec.format()},
     }
 
-    image_ref_for_spec = image_ref.with_tag(default_tag(spec))
+    if annotations:
+        oci_manifest["annotations"] = annotations
 
     # Finally upload the manifest
-    upload_manifest_with_retry(image_ref_for_spec, oci_manifest=oci_manifest)
+    upload_manifest_with_retry(image_ref, oci_manifest=oci_manifest)
 
     # delete the config file
     os.unlink(config_file)
 
-    return image_ref_for_spec
+
+def _update_base_images(
+    *,
+    base_image: Optional[ImageReference],
+    target_image: ImageReference,
+    spec: spack.spec.Spec,
+    base_image_cache: Dict[str, Tuple[dict, dict]],
+):
+    """For a given spec and base image, copy the missing layers of the base image with matching
+    arch to the registry of the target image. If no base image is specified, create a dummy
+    manifest and config file."""
+    architecture = _archspec_to_gooarch(spec)
+    if architecture in base_image_cache:
+        return
+    if base_image is None:
+        base_image_cache[architecture] = (
+            default_manifest(),
+            default_config(architecture, "linux"),
+        )
+    else:
+        base_image_cache[architecture] = copy_missing_layers_with_retry(
+            base_image, target_image, architecture
+        )
 
 
 def _push_oci(
-    args,
-    image_ref: ImageReference,
+    *,
+    target_image: ImageReference,
+    base_image: Optional[ImageReference],
     installed_specs_with_deps: List[Spec],
     tmpdir: str,
     pool: multiprocessing.pool.Pool,
-) -> List[str]:
+    force: bool = False,
+) -> Tuple[List[str], Dict[str, Tuple[dict, dict]], Dict[str, spack.oci.oci.Blob]]:
     """Push specs to an OCI registry
 
     Args:
-        args: The command line arguments.
-        image_ref: The image reference.
+        image_ref: The target OCI image
+        base_image: Optional base image, which will be copied to the target registry.
         installed_specs_with_deps: The installed specs to push, excluding externals,
             including deps, ordered from roots to leaves.
+        force: Whether to overwrite existing layers and manifests in the buildcache.
 
     Returns:
-        List[str]: The list of skipped specs (already in the buildcache).
+        A tuple consisting of the list of skipped specs already in the build cache,
+        a dictionary mapping architectures to base image manifests and configs,
+        and a dictionary mapping each spec's dag hash to a blob.
     """
 
     # Reverse the order
     installed_specs_with_deps = list(reversed(installed_specs_with_deps))
 
-    # The base image to use for the package. When not set, we use
-    # the OCI registry only for storage, and do not use any base image.
-    base_image_ref: Optional[ImageReference] = (
-        ImageReference.from_string(args.base_image) if args.base_image else None
-    )
-
     # Spec dag hash -> blob
     checksums: Dict[str, spack.oci.oci.Blob] = {}
 
@@ -623,11 +693,11 @@ def _push_oci(
     # Specs not uploaded because they already exist
     skipped = []
 
-    if not args.force:
+    if not force:
         tty.info("Checking for existing specs in the buildcache")
         to_be_uploaded = []
 
-        tags_to_check = (image_ref.with_tag(default_tag(s)) for s in installed_specs_with_deps)
+        tags_to_check = (target_image.with_tag(default_tag(s)) for s in installed_specs_with_deps)
         available_blobs = pool.map(_get_spack_binary_blob, tags_to_check)
 
         for spec, maybe_blob in zip(installed_specs_with_deps, available_blobs):
@@ -640,46 +710,63 @@ def _push_oci(
         to_be_uploaded = installed_specs_with_deps
 
     if not to_be_uploaded:
-        return skipped
+        return skipped, base_images, checksums
 
     tty.info(
-        f"{len(to_be_uploaded)} specs need to be pushed to {image_ref.domain}/{image_ref.name}"
+        f"{len(to_be_uploaded)} specs need to be pushed to "
+        f"{target_image.domain}/{target_image.name}"
     )
 
     # Upload blobs
     new_blobs = pool.starmap(
-        _push_single_spack_binary_blob, ((image_ref, spec, tmpdir) for spec in to_be_uploaded)
+        _push_single_spack_binary_blob, ((target_image, spec, tmpdir) for spec in to_be_uploaded)
     )
 
     # And update the spec to blob mapping
     for spec, blob in zip(to_be_uploaded, new_blobs):
         checksums[spec.dag_hash()] = blob
 
-    # Copy base image layers, probably fine to do sequentially.
+    # Copy base images if necessary
     for spec in to_be_uploaded:
-        architecture = _archspec_to_gooarch(spec)
-        # Get base image details, if we don't have them yet
-        if architecture in base_images:
-            continue
-        if base_image_ref is None:
-            base_images[architecture] = (default_manifest(), default_config(architecture, "linux"))
-        else:
-            base_images[architecture] = copy_missing_layers_with_retry(
-                base_image_ref, image_ref, architecture
-            )
+        _update_base_images(
+            base_image=base_image,
+            target_image=target_image,
+            spec=spec,
+            base_image_cache=base_images,
+        )
+
+    def extra_config(spec: Spec):
+        spec_dict = spec.to_dict(hash=ht.dag_hash)
+        spec_dict["buildcache_layout_version"] = 1
+        spec_dict["binary_cache_checksum"] = {
+            "hash_algorithm": "sha256",
+            "hash": checksums[spec.dag_hash()].compressed_digest.digest,
+        }
+        return spec_dict
 
     # Upload manifests
     tty.info("Uploading manifests")
-    pushed_image_ref = pool.starmap(
+    pool.starmap(
         _put_manifest,
-        ((base_images, checksums, spec, image_ref, tmpdir) for spec in to_be_uploaded),
+        (
+            (
+                base_images,
+                checksums,
+                target_image.with_tag(default_tag(spec)),
+                tmpdir,
+                extra_config(spec),
+                {"org.opencontainers.image.description": spec.format()},
+                spec,
+            )
+            for spec in to_be_uploaded
+        ),
     )
 
     # Print the image names of the top-level specs
-    for spec, ref in zip(to_be_uploaded, pushed_image_ref):
-        tty.info(f"Pushed {_format_spec(spec)} to {ref}")
+    for spec in to_be_uploaded:
+        tty.info(f"Pushed {_format_spec(spec)} to {target_image.with_tag(default_tag(spec))}")
 
-    return skipped
+    return skipped, base_images, checksums
 
 
 def _config_from_tag(image_ref: ImageReference, tag: str) -> Optional[dict]:
@@ -816,15 +903,24 @@ def check_fn(args: argparse.Namespace):
     exit code is non-zero, then at least one of the indicated specs needs to be rebuilt
     """
     if args.spec_file:
+        specs_arg = (
+            args.spec_file if os.path.sep in args.spec_file else os.path.join(".", args.spec_file)
+        )
         tty.warn(
             "The flag `--spec-file` is deprecated and will be removed in Spack 0.22. "
-            "Use --spec instead."
+            f"Use `spack buildcache check {specs_arg}` instead."
         )
+    elif args.spec:
+        specs_arg = args.spec
+        tty.warn(
+            "The flag `--spec` is deprecated and will be removed in Spack 0.23. "
+            f"Use `spack buildcache check {specs_arg}` instead."
+        )
+    else:
+        specs_arg = args.specs
 
-    specs = spack.cmd.parse_specs(args.spec or args.spec_file)
-
-    if specs:
-        specs = _matching_specs(specs)
+    if specs_arg:
+        specs = _matching_specs(spack.cmd.parse_specs(specs_arg))
     else:
         specs = spack.cmd.require_active_env("buildcache check").all_specs()
 

lib/spack/spack/cmd/change.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
+from spack.cmd.common import arguments
 
 description = "change an existing spec in an environment"
 section = "environments"

lib/spack/spack/cmd/ci.py

@@ -16,6 +16,7 @@
 import spack.cmd.buildcache as buildcache
 import spack.config as cfg
 import spack.environment as ev
+import spack.environment.depfile
 import spack.hash_types as ht
 import spack.mirror
 import spack.util.gpg as gpg_util
@@ -606,7 +607,9 @@ def ci_rebuild(args):
             "SPACK_INSTALL_FLAGS={}".format(args_to_string(deps_install_args)),
             "-j$(nproc)",
             "install-deps/{}".format(
-                ev.depfile.MakefileSpec(job_spec).safe_format("{name}-{version}-{hash}")
+                spack.environment.depfile.MakefileSpec(job_spec).safe_format(
+                    "{name}-{version}-{hash}"
+                )
             ),
         ],
         spack_cmd + ["install"] + root_install_args,

lib/spack/spack/cmd/clean.py

@@ -12,13 +12,13 @@
 
 import spack.bootstrap
 import spack.caches
-import spack.cmd.common.arguments as arguments
 import spack.cmd.test
 import spack.config
 import spack.repo
 import spack.stage
 import spack.store
 import spack.util.path
+from spack.cmd.common import arguments
 from spack.paths import lib_path, var_path
 
 description = "remove temporary build files and/or downloaded archives"

lib/spack/spack/cmd/common/arguments.py

@@ -67,12 +67,13 @@ class ConstraintAction(argparse.Action):
 
     def __call__(self, parser, namespace, values, option_string=None):
         # Query specs from command line
-        self.values = values
-        namespace.constraint = values
+        self.constraint = namespace.constraint = values
+        self.constraint_specs = namespace.constraint_specs = []
         namespace.specs = self._specs
 
     def _specs(self, **kwargs):
-        qspecs = spack.cmd.parse_specs(self.values)
+        # store parsed specs in spec.constraint after a call to specs()
+        self.constraint_specs[:] = spack.cmd.parse_specs(self.constraint)
 
         # If an environment is provided, we'll restrict the search to
         # only its installed packages.
@@ -81,12 +82,12 @@ def _specs(self, **kwargs):
             kwargs["hashes"] = set(env.all_hashes())
 
         # return everything for an empty query.
-        if not qspecs:
+        if not self.constraint_specs:
             return spack.store.STORE.db.query(**kwargs)
 
         # Return only matching stuff otherwise.
         specs = {}
-        for spec in qspecs:
+        for spec in self.constraint_specs:
             for s in spack.store.STORE.db.query(spec, **kwargs):
                 # This is fast for already-concrete specs
                 specs[s.dag_hash()] = s
@@ -124,6 +125,33 @@ def __call__(self, parser, namespace, values, option_string=None):
         setattr(namespace, self.dest, deptype)
 
 
+class ConfigScope(argparse.Action):
+    """Pick the currently configured config scopes."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        kwargs.setdefault("metavar", spack.config.SCOPES_METAVAR)
+        super().__init__(*args, **kwargs)
+
+    @property
+    def default(self):
+        return self._default() if callable(self._default) else self._default
+
+    @default.setter
+    def default(self, value):
+        self._default = value
+
+    @property
+    def choices(self):
+        return spack.config.scopes().keys()
+
+    @choices.setter
+    def choices(self, value):
+        pass
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, values)
+
+
 def _cdash_reporter(namespace):
     """Helper function to create a CDash reporter. This function gets an early reference to the
     argparse namespace under construction, so it can later use it to create the object.
@@ -357,10 +385,11 @@ def install_status():
         "--install-status",
         action="store_true",
         default=True,
-        help="show install status of packages\n\npackages can be: "
-        "installed [+], missing and needed by an installed package [-], "
-        "installed in an upstream instance [^], "
-        "or not installed (no annotation)",
+        help=(
+            "show install status of packages\n"
+            "[+] installed       [^] installed in an upstream\n"
+            " -  not installed   [-] missing dep of installed package\n"
+        ),
     )
 
 

lib/spack/spack/cmd/common/confirmation.py

@@ -0,0 +1,30 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import sys
+from typing import List
+
+import llnl.util.tty as tty
+
+import spack.cmd
+
+display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4}
+
+
+def confirm_action(specs: List[spack.spec.Spec], participle: str, noun: str):
+    """Display the list of specs to be acted on and ask for confirmation.
+
+    Args:
+        specs: specs to be removed
+        participle: action expressed as a participle, e.g. "uninstalled"
+        noun: action expressed as a noun, e.g. "uninstallation"
+    """
+    tty.msg(f"The following {len(specs)} packages will be {participle}:\n")
+    spack.cmd.display_specs(specs, **display_args)
+    print("")
+    answer = tty.get_yes_or_no("Do you want to proceed?", default=False)
+    if not answer:
+        tty.msg(f"Aborting {noun}")
+        sys.exit(0)

lib/spack/spack/cmd/common/env_utility.py

@@ -8,13 +8,13 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.deptypes as dt
 import spack.error
 import spack.paths
 import spack.spec
 import spack.store
 from spack import build_environment, traverse
+from spack.cmd.common import arguments
 from spack.context import Context
 from spack.util.environment import dump_environment, pickle_environment
 

lib/spack/spack/cmd/compiler.py

@@ -14,6 +14,7 @@
 import spack.compilers
 import spack.config
 import spack.spec
+from spack.cmd.common import arguments
 
 description = "manage compilers"
 section = "system"
@@ -23,8 +24,6 @@
 def setup_parser(subparser):
     sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="compiler_command")
 
-    scopes = spack.config.scopes()
-
     # Find
     find_parser = sp.add_parser(
         "find",
@@ -47,9 +46,8 @@ def setup_parser(subparser):
     find_parser.add_argument("add_paths", nargs=argparse.REMAINDER)
     find_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope("compilers"),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope("compilers"),
         help="configuration scope to modify",
     )
 
@@ -60,32 +58,20 @@ def setup_parser(subparser):
     )
     remove_parser.add_argument("compiler_spec")
     remove_parser.add_argument(
-        "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=None,
-        help="configuration scope to modify",
+        "--scope", action=arguments.ConfigScope, default=None, help="configuration scope to modify"
     )
 
     # List
     list_parser = sp.add_parser("list", help="list available compilers")
     list_parser.add_argument(
-        "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_list_scope(),
-        help="configuration scope to read from",
+        "--scope", action=arguments.ConfigScope, help="configuration scope to read from"
     )
 
     # Info
     info_parser = sp.add_parser("info", help="show compiler paths")
     info_parser.add_argument("compiler_spec")
     info_parser.add_argument(
-        "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_list_scope(),
-        help="configuration scope to read from",
+        "--scope", action=arguments.ConfigScope, help="configuration scope to read from"
     )
 
 

lib/spack/spack/cmd/compilers.py

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import spack.config
+from spack.cmd.common import arguments
 from spack.cmd.compiler import compiler_list
 
 description = "list available compilers"
@@ -12,13 +12,8 @@
 
 
 def setup_parser(subparser):
-    scopes = spack.config.scopes()
-
     subparser.add_argument(
-        "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        help="configuration scope to read/modify",
+        "--scope", action=arguments.ConfigScope, help="configuration scope to read/modify"
     )
 
 

lib/spack/spack/cmd/config.py

@@ -5,12 +5,12 @@
 import collections
 import os
 import shutil
+import sys
 from typing import List
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 
-import spack.cmd.common.arguments
 import spack.config
 import spack.environment as ev
 import spack.repo
@@ -18,6 +18,7 @@
 import spack.schema.packages
 import spack.store
 import spack.util.spack_yaml as syaml
+from spack.cmd.common import arguments
 from spack.util.editor import editor
 
 description = "get and set configuration options"
@@ -26,14 +27,9 @@
 
 
 def setup_parser(subparser):
-    scopes = spack.config.scopes()
-
     # User can only choose one
     subparser.add_argument(
-        "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        help="configuration scope to read/modify",
+        "--scope", action=arguments.ConfigScope, help="configuration scope to read/modify"
     )
 
     sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="config_command")
@@ -53,6 +49,7 @@ def setup_parser(subparser):
     blame_parser.add_argument(
         "section",
         help="configuration section to print\n\noptions: %(choices)s",
+        nargs="?",
         metavar="section",
         choices=spack.config.SECTION_SCHEMAS,
     )
@@ -101,13 +98,13 @@ def setup_parser(subparser):
     setup_parser.add_parser = add_parser
 
     update = sp.add_parser("update", help="update configuration files to the latest format")
-    spack.cmd.common.arguments.add_common_arguments(update, ["yes_to_all"])
+    arguments.add_common_arguments(update, ["yes_to_all"])
     update.add_argument("section", help="section to update")
 
     revert = sp.add_parser(
         "revert", help="revert configuration files to their state before update"
     )
-    spack.cmd.common.arguments.add_common_arguments(revert, ["yes_to_all"])
+    arguments.add_common_arguments(revert, ["yes_to_all"])
     revert.add_argument("section", help="section to update")
 
 
@@ -136,32 +133,50 @@ def _get_scope_and_section(args):
     return scope, section
 
 
+def print_configuration(args, *, blame: bool) -> None:
+    if args.scope and args.section is None:
+        tty.die(f"the argument --scope={args.scope} requires specifying a section.")
+
+    if args.section is not None:
+        spack.config.CONFIG.print_section(args.section, blame=blame, scope=args.scope)
+        return
+
+    print_flattened_configuration(blame=blame)
+
+
+def print_flattened_configuration(*, blame: bool) -> None:
+    """Prints to stdout a flattened version of the configuration.
+
+    Args:
+        blame: if True, shows file provenance for each entry in the configuration.
+    """
+    env = ev.active_environment()
+    if env is not None:
+        pristine = env.manifest.pristine_yaml_content
+        flattened = pristine.copy()
+        flattened[spack.schema.env.TOP_LEVEL_KEY] = pristine[spack.schema.env.TOP_LEVEL_KEY].copy()
+    else:
+        flattened = syaml.syaml_dict()
+        flattened[spack.schema.env.TOP_LEVEL_KEY] = syaml.syaml_dict()
+
+    for config_section in spack.config.SECTION_SCHEMAS:
+        current = spack.config.get(config_section)
+        flattened[spack.schema.env.TOP_LEVEL_KEY][config_section] = current
+    syaml.dump_config(flattened, stream=sys.stdout, default_flow_style=False, blame=blame)
+
+
 def config_get(args):
     """Dump merged YAML configuration for a specific section.
 
     With no arguments and an active environment, print the contents of
     the environment's manifest file (spack.yaml).
     """
-    scope, section = _get_scope_and_section(args)
-
-    if section is not None:
-        spack.config.CONFIG.print_section(section)
-
-    elif scope and scope.startswith("env:"):
-        config_file = spack.config.CONFIG.get_config_filename(scope, section)
-        if os.path.exists(config_file):
-            with open(config_file) as f:
-                print(f.read())
-        else:
-            tty.die("environment has no %s file" % ev.manifest_name)
-
-    else:
-        tty.die("`spack config get` requires a section argument or an active environment.")
+    print_configuration(args, blame=False)
 
 
 def config_blame(args):
     """Print out line-by-line blame of merged YAML."""
-    spack.config.CONFIG.print_section(args.section, blame=True)
+    print_configuration(args, blame=True)
 
 
 def config_edit(args):

lib/spack/spack/cmd/create.py

@@ -172,6 +172,14 @@ def configure_args(self):
         return args"""
 
 
+class CargoPackageTemplate(PackageTemplate):
+    """Provides appropriate overrides for cargo-based packages"""
+
+    base_class_name = "CargoPackage"
+
+    body_def = ""
+
+
 class CMakePackageTemplate(PackageTemplate):
     """Provides appropriate overrides for CMake-based packages"""
 
@@ -186,6 +194,14 @@ def cmake_args(self):
         return args"""
 
 
+class GoPackageTemplate(PackageTemplate):
+    """Provides appropriate overrides for Go-module-based packages"""
+
+    base_class_name = "GoPackage"
+
+    body_def = ""
+
+
 class LuaPackageTemplate(PackageTemplate):
     """Provides appropriate overrides for LuaRocks-based packages"""
 
@@ -575,28 +591,30 @@ def __init__(self, name, *args, **kwargs):
 
 
 templates = {
-    "autotools": AutotoolsPackageTemplate,
     "autoreconf": AutoreconfPackageTemplate,
-    "cmake": CMakePackageTemplate,
-    "bundle": BundlePackageTemplate,
-    "qmake": QMakePackageTemplate,
-    "maven": MavenPackageTemplate,
-    "scons": SconsPackageTemplate,
-    "waf": WafPackageTemplate,
+    "autotools": AutotoolsPackageTemplate,
     "bazel": BazelPackageTemplate,
+    "bundle": BundlePackageTemplate,
+    "cargo": CargoPackageTemplate,
+    "cmake": CMakePackageTemplate,
+    "generic": PackageTemplate,
+    "go": GoPackageTemplate,
+    "intel": IntelPackageTemplate,
+    "lua": LuaPackageTemplate,
+    "makefile": MakefilePackageTemplate,
+    "maven": MavenPackageTemplate,
+    "meson": MesonPackageTemplate,
+    "octave": OctavePackageTemplate,
+    "perlbuild": PerlbuildPackageTemplate,
+    "perlmake": PerlmakePackageTemplate,
     "python": PythonPackageTemplate,
+    "qmake": QMakePackageTemplate,
     "r": RPackageTemplate,
     "racket": RacketPackageTemplate,
-    "perlmake": PerlmakePackageTemplate,
-    "perlbuild": PerlbuildPackageTemplate,
-    "octave": OctavePackageTemplate,
     "ruby": RubyPackageTemplate,
-    "makefile": MakefilePackageTemplate,
-    "intel": IntelPackageTemplate,
-    "meson": MesonPackageTemplate,
-    "lua": LuaPackageTemplate,
+    "scons": SconsPackageTemplate,
     "sip": SIPPackageTemplate,
-    "generic": PackageTemplate,
+    "waf": WafPackageTemplate,
 }
 
 
@@ -679,6 +697,8 @@ def __call__(self, stage, url):
         clues = [
             (r"/CMakeLists\.txt$", "cmake"),
             (r"/NAMESPACE$", "r"),
+            (r"/Cargo\.toml$", "cargo"),
+            (r"/go\.mod$", "go"),
             (r"/configure$", "autotools"),
             (r"/configure\.(in|ac)$", "autoreconf"),
             (r"/Makefile\.am$", "autoreconf"),

lib/spack/spack/cmd/deconcretize.py

@@ -0,0 +1,103 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import argparse
+import sys
+from typing import List
+
+import llnl.util.tty as tty
+
+import spack.cmd
+import spack.cmd.common.confirmation as confirmation
+import spack.environment as ev
+import spack.spec
+from spack.cmd.common import arguments
+
+description = "remove specs from the concretized lockfile of an environment"
+section = "environments"
+level = "long"
+
+# Arguments for display_specs when we find ambiguity
+display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4}
+
+
+def setup_parser(subparser):
+    subparser.add_argument(
+        "--root", action="store_true", help="deconcretize only specific environment roots"
+    )
+    arguments.add_common_arguments(subparser, ["yes_to_all", "specs"])
+    subparser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        dest="all",
+        help="deconcretize ALL specs that match each supplied spec",
+    )
+
+
+def get_deconcretize_list(
+    args: argparse.Namespace, specs: List[spack.spec.Spec], env: ev.Environment
+) -> List[spack.spec.Spec]:
+    """
+    Get list of environment roots to deconcretize
+    """
+    env_specs = [s for _, s in env.concretized_specs()]
+    to_deconcretize = []
+    errors = []
+
+    for s in specs:
+        if args.root:
+            # find all roots matching given spec
+            to_deconc = [e for e in env_specs if e.satisfies(s)]
+        else:
+            # find all roots matching or depending on a matching spec
+            to_deconc = [e for e in env_specs if any(d.satisfies(s) for d in e.traverse())]
+
+        if len(to_deconc) < 1:
+            tty.warn(f"No matching specs to deconcretize for {s}")
+
+        elif len(to_deconc) > 1 and not args.all:
+            errors.append((s, to_deconc))
+
+        to_deconcretize.extend(to_deconc)
+
+    if errors:
+        for spec, matching in errors:
+            tty.error(f"{spec} matches multiple concrete specs:")
+            sys.stderr.write("\n")
+            spack.cmd.display_specs(matching, output=sys.stderr, **display_args)
+            sys.stderr.write("\n")
+            sys.stderr.flush()
+        tty.die("Use '--all' to deconcretize all matching specs, or be more specific")
+
+    return to_deconcretize
+
+
+def deconcretize_specs(args, specs):
+    env = spack.cmd.require_active_env(cmd_name="deconcretize")
+
+    if args.specs:
+        deconcretize_list = get_deconcretize_list(args, specs, env)
+    else:
+        deconcretize_list = [s for _, s in env.concretized_specs()]
+
+    if not args.yes_to_all:
+        confirmation.confirm_action(deconcretize_list, "deconcretized", "deconcretization")
+
+    with env.write_transaction():
+        for spec in deconcretize_list:
+            env.deconcretize(spec)
+        env.write()
+
+
+def deconcretize(parser, args):
+    if not args.specs and not args.all:
+        tty.die(
+            "deconcretize requires at least one spec argument.",
+            " Use `spack deconcretize --all` to deconcretize ALL specs.",
+        )
+
+    specs = spack.cmd.parse_specs(args.specs) if args.specs else [any]
+    deconcretize_specs(args, specs)

lib/spack/spack/cmd/dependencies.py

@@ -9,11 +9,11 @@
 from llnl.util.tty.colify import colify
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.package_base
 import spack.repo
 import spack.store
+from spack.cmd.common import arguments
 
 description = "show dependencies of a package"
 section = "basic"

lib/spack/spack/cmd/dependents.py

@@ -9,10 +9,10 @@
 from llnl.util.tty.colify import colify
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.repo
 import spack.store
+from spack.cmd.common import arguments
 
 description = "show packages that depend on another"
 section = "basic"

lib/spack/spack/cmd/deprecate.py

@@ -20,9 +20,9 @@
 from llnl.util.symlink import symlink
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.store
+from spack.cmd.common import arguments
 from spack.database import InstallStatuses
 from spack.error import SpackError
 

lib/spack/spack/cmd/dev_build.py

@@ -9,9 +9,9 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.repo
+from spack.cmd.common import arguments
 
 description = "developer build: build from code in current working directory"
 section = "build"

lib/spack/spack/cmd/develop.py

@@ -8,10 +8,10 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.spec
 import spack.util.path
 import spack.version
+from spack.cmd.common import arguments
 from spack.error import SpackError
 
 description = "add a spec to an environment's dev-build information"
@@ -45,10 +45,41 @@ def setup_parser(subparser):
     arguments.add_common_arguments(subparser, ["spec"])
 
 
-def develop(parser, args):
-    env = spack.cmd.require_active_env(cmd_name="develop")
+def _update_config(spec, path):
+    find_fn = lambda section: spec.name in section
 
+    entry = {"spec": str(spec)}
+    if path != spec.name:
+        entry["path"] = path
+
+    def change_fn(section):
+        section[spec.name] = entry
+
+    spack.config.change_or_add("develop", find_fn, change_fn)
+
+
+def _retrieve_develop_source(spec, abspath):
+    # "steal" the source code via staging API. We ask for a stage
+    # to be created, then copy it afterwards somewhere else. It would be
+    # better if we can create the `source_path` directly into its final
+    # destination.
+    pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)
+    # We construct a package class ourselves, rather than asking for
+    # Spec.package, since Spec only allows this when it is concrete
+    package = pkg_cls(spec)
+    if isinstance(package.stage[0].fetcher, spack.fetch_strategy.GitFetchStrategy):
+        package.stage[0].fetcher.get_full_repo = True
+        # If we retrieved this version before and cached it, we may have
+        # done so without cloning the full git repo; likewise, any
+        # mirror might store an instance with truncated history.
+        package.stage[0].disable_mirrors()
+
+    package.stage.steal_source(abspath)
+
+
+def develop(parser, args):
     if not args.spec:
+        env = spack.cmd.require_active_env(cmd_name="develop")
         if args.clone is False:
             raise SpackError("No spec provided to spack develop command")
 
@@ -66,7 +97,7 @@ def develop(parser, args):
             # Both old syntax `spack develop pkg@x` and new syntax `spack develop pkg@=x`
             # are currently supported.
             spec = spack.spec.parse_with_version_concrete(entry["spec"])
-            env.develop(spec=spec, path=path, clone=True)
+            _retrieve_develop_source(spec, abspath)
 
         if not env.dev_specs:
             tty.warn("No develop specs to download")
@@ -81,12 +112,16 @@ def develop(parser, args):
     version = spec.versions.concrete_range_as_version
     if not version:
         raise SpackError("Packages to develop must have a concrete version")
-
     spec.versions = spack.version.VersionList([version])
 
-    # default path is relative path to spec.name
+    # If user does not specify --path, we choose to create a directory in the
+    # active environment's directory, named after the spec
     path = args.path or spec.name
-    abspath = spack.util.path.canonicalize_path(path, default_wd=env.path)
+    if not os.path.isabs(path):
+        env = spack.cmd.require_active_env(cmd_name="develop")
+        abspath = spack.util.path.canonicalize_path(path, default_wd=env.path)
+    else:
+        abspath = path
 
     # clone default: only if the path doesn't exist
     clone = args.clone
@@ -96,15 +131,24 @@ def develop(parser, args):
     if not clone and not os.path.exists(abspath):
         raise SpackError("Provided path %s does not exist" % abspath)
 
-    if clone and os.path.exists(abspath):
-        if args.force:
-            shutil.rmtree(abspath)
-        else:
-            msg = "Path %s already exists and cannot be cloned to." % abspath
-            msg += " Use `spack develop -f` to overwrite."
-            raise SpackError(msg)
+    if clone:
+        if os.path.exists(abspath):
+            if args.force:
+                shutil.rmtree(abspath)
+            else:
+                msg = "Path %s already exists and cannot be cloned to." % abspath
+                msg += " Use `spack develop -f` to overwrite."
+                raise SpackError(msg)
 
+        _retrieve_develop_source(spec, abspath)
+
+    # Note: we could put develop specs in any scope, but I assume
+    # users would only ever want to do this for either (a) an active
+    # env or (b) a specified config file (e.g. that is included by
+    # an environment)
+    # TODO: when https://github.com/spack/spack/pull/35307 is merged,
+    # an active env is not required if a scope is specified
+    env = spack.cmd.require_active_env(cmd_name="develop")
+    tty.debug("Updating develop config for {0} transactionally".format(env.name))
     with env.write_transaction():
-        changed = env.develop(spec, path, clone)
-        if changed:
-            env.write()
+        _update_config(spec, path)

lib/spack/spack/cmd/diff.py

@@ -10,11 +10,11 @@
 from llnl.util.tty.color import cprint, get_color_when
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.solver.asp as asp
 import spack.util.environment
 import spack.util.spack_json as sjson
+from spack.cmd.common import arguments
 
 description = "compare two specs"
 section = "basic"
@@ -44,6 +44,9 @@ def setup_parser(subparser):
         action="append",
         help="select the attributes to show (defaults to all)",
     )
+    subparser.add_argument(
+        "--ignore", action="append", help="omit diffs related to these dependencies"
+    )
 
 
 def shift(asp_function):
@@ -54,7 +57,7 @@ def shift(asp_function):
     return asp.AspFunction(first, rest)
 
 
-def compare_specs(a, b, to_string=False, color=None):
+def compare_specs(a, b, to_string=False, color=None, ignore_packages=None):
     """
     Generate a comparison, including diffs (for each side) and an intersection.
 
@@ -73,6 +76,14 @@ def compare_specs(a, b, to_string=False, color=None):
     if color is None:
         color = get_color_when()
 
+    a = a.copy()
+    b = b.copy()
+
+    if ignore_packages:
+        for pkg_name in ignore_packages:
+            a.trim(pkg_name)
+            b.trim(pkg_name)
+
     # Prepare a solver setup to parse differences
     setup = asp.SpackSolverSetup()
 
@@ -200,6 +211,8 @@ def diff(parser, args):
 
     specs = []
     for spec in spack.cmd.parse_specs(args.specs):
+        # If the spec has a hash, check it before disambiguating
+        spec.replace_hash()
         if spec.concrete:
             specs.append(spec)
         else:
@@ -207,7 +220,7 @@ def diff(parser, args):
 
     # Calculate the comparison (c)
     color = False if args.dump_json else get_color_when()
-    c = compare_specs(specs[0], specs[1], to_string=True, color=color)
+    c = compare_specs(specs[0], specs[1], to_string=True, color=color, ignore_packages=args.ignore)
 
     # Default to all attributes
     attributes = args.attribute or ["all"]

lib/spack/spack/cmd/env.py

@@ -20,7 +20,6 @@
 import spack.cmd
 import spack.cmd.common
 import spack.cmd.common.arguments
-import spack.cmd.common.arguments as arguments
 import spack.cmd.install
 import spack.cmd.modules
 import spack.cmd.uninstall
@@ -31,6 +30,7 @@
 import spack.schema.env
 import spack.spec
 import spack.tengine
+from spack.cmd.common import arguments
 from spack.util.environment import EnvironmentModifications
 
 description = "manage virtual environments"

lib/spack/spack/cmd/extensions.py

@@ -10,10 +10,10 @@
 from llnl.util.tty.colify import colify
 
 import spack.cmd as cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.repo
 import spack.store
+from spack.cmd.common import arguments
 
 description = "list extensions for package"
 section = "extensions"

lib/spack/spack/cmd/external.py

@@ -14,12 +14,12 @@
 
 import spack
 import spack.cmd
-import spack.cmd.common.arguments
 import spack.config
 import spack.cray_manifest as cray_manifest
 import spack.detection
 import spack.error
 import spack.util.environment
+from spack.cmd.common import arguments
 
 description = "manage external packages in Spack configuration"
 section = "config"
@@ -29,8 +29,6 @@
 def setup_parser(subparser):
     sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="external_command")
 
-    scopes = spack.config.scopes()
-
     find_parser = sp.add_parser("find", help="add external packages to packages.yaml")
     find_parser.add_argument(
         "--not-buildable",
@@ -48,15 +46,14 @@ def setup_parser(subparser):
     )
     find_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope("packages"),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope("packages"),
         help="configuration scope to modify",
     )
     find_parser.add_argument(
         "--all", action="store_true", help="search for all packages that Spack knows about"
     )
-    spack.cmd.common.arguments.add_common_arguments(find_parser, ["tags", "jobs"])
+    arguments.add_common_arguments(find_parser, ["tags", "jobs"])
     find_parser.add_argument("packages", nargs=argparse.REMAINDER)
     find_parser.epilog = (
         'The search is by default on packages tagged with the "build-tools" or '

lib/spack/spack/cmd/fetch.py

@@ -6,11 +6,11 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
 import spack.repo
 import spack.traverse
+from spack.cmd.common import arguments
 
 description = "fetch archives for packages"
 section = "build"

lib/spack/spack/cmd/find.py

@@ -12,9 +12,9 @@
 
 import spack.bootstrap
 import spack.cmd as cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.repo
+from spack.cmd.common import arguments
 from spack.database import InstallStatuses
 
 description = "list and search installed packages"
@@ -261,10 +261,8 @@ def find(parser, args):
 
     # Exit early with an error code if no package matches the constraint
     if not results and args.constraint:
-        msg = "No package matches the query: {0}"
-        msg = msg.format(" ".join(args.constraint))
-        tty.msg(msg)
-        raise SystemExit(1)
+        constraint_str = " ".join(str(s) for s in args.constraint_specs)
+        tty.die(f"No package matches the query: {constraint_str}")
 
     # If tags have been specified on the command line, filter by tags
     if args.tags:

lib/spack/spack/cmd/gc.py

@@ -6,6 +6,7 @@
 import llnl.util.tty as tty
 
 import spack.cmd.common.arguments
+import spack.cmd.common.confirmation
 import spack.cmd.uninstall
 import spack.environment as ev
 import spack.store
@@ -41,6 +42,6 @@ def gc(parser, args):
         return
 
     if not args.yes_to_all:
-        spack.cmd.uninstall.confirm_removal(specs)
+        spack.cmd.common.confirmation.confirm_action(specs, "uninstalled", "uninstallation")
 
     spack.cmd.uninstall.do_uninstall(specs, force=False)

lib/spack/spack/cmd/gpg.py

@@ -7,11 +7,11 @@
 import os
 
 import spack.binary_distribution
-import spack.cmd.common.arguments as arguments
 import spack.mirror
 import spack.paths
 import spack.util.gpg
 import spack.util.url
+from spack.cmd.common import arguments
 
 description = "handle GPG actions for spack"
 section = "packaging"

lib/spack/spack/cmd/graph.py

@@ -5,10 +5,10 @@
 from llnl.util import tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
 import spack.store
+from spack.cmd.common import arguments
 from spack.graph import DAGWithDependencyTypes, SimpleDAG, graph_ascii, graph_dot, static_graph_dot
 
 description = "generate graphs of package dependency relationships"
@@ -61,7 +61,7 @@ def graph(parser, args):
         args.dot = True
         env = ev.active_environment()
         if env:
-            specs = env.all_specs()
+            specs = env.concrete_roots()
         else:
             specs = spack.store.STORE.db.query()
 

lib/spack/spack/cmd/info.py

@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import sys
 import textwrap
 from itertools import zip_longest
 
@@ -10,12 +11,13 @@
 import llnl.util.tty.color as color
 from llnl.util.tty.colify import colify
 
-import spack.cmd.common.arguments as arguments
 import spack.deptypes as dt
 import spack.fetch_strategy as fs
 import spack.install_test
 import spack.repo
 import spack.spec
+import spack.version
+from spack.cmd.common import arguments
 from spack.package_base import preferred_version
 
 description = "get detailed information on a particular package"
@@ -53,6 +55,7 @@ def setup_parser(subparser):
         ("--tags", print_tags.__doc__),
         ("--tests", print_tests.__doc__),
         ("--virtuals", print_virtuals.__doc__),
+        ("--variants-by-name", "list variants in strict name order; don't group by condition"),
     ]
     for opt, help_comment in options:
         subparser.add_argument(opt, action="store_true", help=help_comment)
@@ -77,35 +80,10 @@ def license(s):
 
 
 class VariantFormatter:
-    def __init__(self, variants):
-        self.variants = variants
+    def __init__(self, pkg):
+        self.variants = pkg.variants
         self.headers = ("Name [Default]", "When", "Allowed values", "Description")
 
-        # Formats
-        fmt_name = "{0} [{1}]"
-
-        # Initialize column widths with the length of the
-        # corresponding headers, as they cannot be shorter
-        # than that
-        self.column_widths = [len(x) for x in self.headers]
-
-        # Expand columns based on max line lengths
-        for k, e in variants.items():
-            v, w = e
-            candidate_max_widths = (
-                len(fmt_name.format(k, self.default(v))),  # Name [Default]
-                len(str(w)),
-                len(v.allowed_values),  # Allowed values
-                len(v.description),  # Description
-            )
-
-            self.column_widths = (
-                max(self.column_widths[0], candidate_max_widths[0]),
-                max(self.column_widths[1], candidate_max_widths[1]),
-                max(self.column_widths[2], candidate_max_widths[2]),
-                max(self.column_widths[3], candidate_max_widths[3]),
-            )
-
         # Don't let name or possible values be less than max widths
         _, cols = tty.terminal_size()
         max_name = min(self.column_widths[0], 30)
@@ -137,6 +115,8 @@ def default(self, v):
     def lines(self):
         if not self.variants:
             yield "    None"
+            return
+
         else:
             yield "    " + self.fmt % self.headers
             underline = tuple([w * "=" for w in self.column_widths])
@@ -159,7 +139,7 @@ def lines(self):
                     yield "    " + self.fmt % t
 
 
-def print_dependencies(pkg):
+def print_dependencies(pkg, args):
     """output build, link, and run package dependencies"""
 
     for deptype in ("build", "link", "run"):
@@ -172,7 +152,7 @@ def print_dependencies(pkg):
             color.cprint("    None")
 
 
-def print_detectable(pkg):
+def print_detectable(pkg, args):
     """output information on external detection"""
 
     color.cprint("")
@@ -200,7 +180,7 @@ def print_detectable(pkg):
         color.cprint("    False")
 
 
-def print_maintainers(pkg):
+def print_maintainers(pkg, args):
     """output package maintainers"""
 
     if len(pkg.maintainers) > 0:
@@ -209,7 +189,7 @@ def print_maintainers(pkg):
         color.cprint(section_title("Maintainers: ") + mnt)
 
 
-def print_phases(pkg):
+def print_phases(pkg, args):
     """output installation phases"""
 
     if hasattr(pkg.builder, "phases") and pkg.builder.phases:
@@ -221,7 +201,7 @@ def print_phases(pkg):
         color.cprint(phase_str)
 
 
-def print_tags(pkg):
+def print_tags(pkg, args):
     """output package tags"""
 
     color.cprint("")
@@ -233,7 +213,7 @@ def print_tags(pkg):
         color.cprint("    None")
 
 
-def print_tests(pkg):
+def print_tests(pkg, args):
     """output relevant build-time and stand-alone tests"""
 
     # Some built-in base packages (e.g., Autotools) define callback (e.g.,
@@ -271,18 +251,171 @@ def print_tests(pkg):
         color.cprint("    None")
 
 
-def print_variants(pkg):
+def _fmt_value(v):
+    if v is None or isinstance(v, bool):
+        return str(v).lower()
+    else:
+        return str(v)
+
+
+def _fmt_name_and_default(variant):
+    """Print colorized name [default] for a variant."""
+    return color.colorize(f"@c{{{variant.name}}} @C{{[{_fmt_value(variant.default)}]}}")
+
+
+def _fmt_when(when, indent):
+    return color.colorize(f"{indent * ' '}@B{{when}} {color.cescape(when)}")
+
+
+def _fmt_variant_description(variant, width, indent):
+    """Format a variant's description, preserving explicit line breaks."""
+    return "\n".join(
+        textwrap.fill(
+            line, width=width, initial_indent=indent * " ", subsequent_indent=indent * " "
+        )
+        for line in variant.description.split("\n")
+    )
+
+
+def _fmt_variant(variant, max_name_default_len, indent, when=None, out=None):
+    out = out or sys.stdout
+
+    _, cols = tty.terminal_size()
+
+    name_and_default = _fmt_name_and_default(variant)
+    name_default_len = color.clen(name_and_default)
+
+    values = variant.values
+    if not isinstance(variant.values, (tuple, list, spack.variant.DisjointSetsOfValues)):
+        values = [variant.values]
+
+    # put 'none' first, sort the rest by value
+    sorted_values = sorted(values, key=lambda v: (v != "none", v))
+
+    pad = 4  # min padding between 'name [default]' and values
+    value_indent = (indent + max_name_default_len + pad) * " "  # left edge of values
+
+    # This preserves any formatting (i.e., newlines) from how the description was
+    # written in package.py, but still wraps long lines for small terminals.
+    # This allows some packages to provide detailed help on their variants (see, e.g., gasnet).
+    formatted_values = "\n".join(
+        textwrap.wrap(
+            f"{', '.join(_fmt_value(v) for v in sorted_values)}",
+            width=cols - 2,
+            initial_indent=value_indent,
+            subsequent_indent=value_indent,
+        )
+    )
+    formatted_values = formatted_values[indent + name_default_len + pad :]
+
+    # name [default]   value1, value2, value3, ...
+    padding = pad * " "
+    color.cprint(f"{indent * ' '}{name_and_default}{padding}@c{{{formatted_values}}}", stream=out)
+
+    # when <spec>
+    description_indent = indent + 4
+    if when is not None and when != spack.spec.Spec():
+        out.write(_fmt_when(when, description_indent - 2))
+        out.write("\n")
+
+    # description, preserving explicit line breaks from the way it's written in the package file
+    out.write(_fmt_variant_description(variant, cols - 2, description_indent))
+    out.write("\n")
+
+
+def _variants_by_name_when(pkg):
+    """Adaptor to get variants keyed by { name: { when: { [Variant...] } }."""
+    # TODO: replace with pkg.variants_by_name(when=True) when unified directive dicts are merged.
+    variants = {}
+    for name, (variant, whens) in sorted(pkg.variants.items()):
+        for when in whens:
+            variants.setdefault(name, {}).setdefault(when, []).append(variant)
+    return variants
+
+
+def _variants_by_when_name(pkg):
+    """Adaptor to get variants keyed by { when: { name: Variant } }"""
+    # TODO: replace with pkg.variants when unified directive dicts are merged.
+    variants = {}
+    for name, (variant, whens) in pkg.variants.items():
+        for when in whens:
+            variants.setdefault(when, {})[name] = variant
+    return variants
+
+
+def _print_variants_header(pkg):
     """output variants"""
 
+    if not pkg.variants:
+        print("    None")
+        return
+
     color.cprint("")
     color.cprint(section_title("Variants:"))
 
-    formatter = VariantFormatter(pkg.variants)
-    for line in formatter.lines:
-        color.cprint(color.cescape(line))
+    variants_by_name = _variants_by_name_when(pkg)
+
+    # Calculate the max length of the "name [default]" part of the variant display
+    # This lets us know where to print variant values.
+    max_name_default_len = max(
+        color.clen(_fmt_name_and_default(variant))
+        for name, when_variants in variants_by_name.items()
+        for variants in when_variants.values()
+        for variant in variants
+    )
+
+    return max_name_default_len, variants_by_name
 
 
-def print_versions(pkg):
+def _unconstrained_ver_first(item):
+    """sort key that puts specs with open version ranges first"""
+    spec, _ = item
+    return (spack.version.any_version not in spec.versions, spec)
+
+
+def print_variants_grouped_by_when(pkg):
+    max_name_default_len, _ = _print_variants_header(pkg)
+
+    indent = 4
+    variants = _variants_by_when_name(pkg)
+    for when, variants_by_name in sorted(variants.items(), key=_unconstrained_ver_first):
+        padded_values = max_name_default_len + 4
+        start_indent = indent
+
+        if when != spack.spec.Spec():
+            sys.stdout.write("\n")
+            sys.stdout.write(_fmt_when(when, indent))
+            sys.stdout.write("\n")
+
+            # indent names slightly inside 'when', but line up values
+            padded_values -= 2
+            start_indent += 2
+
+        for name, variant in sorted(variants_by_name.items()):
+            _fmt_variant(variant, padded_values, start_indent, None, out=sys.stdout)
+
+
+def print_variants_by_name(pkg):
+    max_name_default_len, variants_by_name = _print_variants_header(pkg)
+    max_name_default_len += 4
+
+    indent = 4
+    for name, when_variants in variants_by_name.items():
+        for when, variants in sorted(when_variants.items(), key=_unconstrained_ver_first):
+            for variant in variants:
+                _fmt_variant(variant, max_name_default_len, indent, when, out=sys.stdout)
+                sys.stdout.write("\n")
+
+
+def print_variants(pkg, args):
+    """output variants"""
+    if args.variants_by_name:
+        print_variants_by_name(pkg)
+    else:
+        print_variants_grouped_by_when(pkg)
+
+
+def print_versions(pkg, args):
     """output versions"""
 
     color.cprint("")
@@ -300,18 +433,24 @@ def print_versions(pkg):
         pad = padder(pkg.versions, 4)
 
         preferred = preferred_version(pkg)
-        url = ""
-        if pkg.has_code:
-            url = fs.for_package_version(pkg, preferred)
 
+        def get_url(version):
+            try:
+                return fs.for_package_version(pkg, version)
+            except spack.fetch_strategy.InvalidArgsError:
+                return "No URL"
+
+        url = get_url(preferred) if pkg.has_code else ""
         line = version("    {0}".format(pad(preferred))) + color.cescape(url)
-        color.cprint(line)
+        color.cwrite(line)
+
+        print()
 
         safe = []
         deprecated = []
         for v in reversed(sorted(pkg.versions)):
             if pkg.has_code:
-                url = fs.for_package_version(pkg, v)
+                url = get_url(v)
             if pkg.versions[v].get("deprecated", False):
                 deprecated.append((v, url))
             else:
@@ -329,7 +468,7 @@ def print_versions(pkg):
                 color.cprint(line)
 
 
-def print_virtuals(pkg):
+def print_virtuals(pkg, args):
     """output virtual packages"""
 
     color.cprint("")
@@ -352,7 +491,7 @@ def print_virtuals(pkg):
         color.cprint("    None")
 
 
-def print_licenses(pkg):
+def print_licenses(pkg, args):
     """Output the licenses of the project."""
 
     color.cprint("")
@@ -384,7 +523,8 @@ def info(parser, args):
     else:
         color.cprint("    None")
 
-    color.cprint(section_title("Homepage: ") + pkg.homepage)
+    if getattr(pkg, "homepage"):
+        color.cprint(section_title("Homepage: ") + pkg.homepage)
 
     # Now output optional information in expected order
     sections = [
@@ -401,6 +541,6 @@ def info(parser, args):
     ]
     for print_it, func in sections:
         if print_it:
-            func(pkg)
+            func(pkg, args)
 
     color.cprint("")

lib/spack/spack/cmd/install.py

@@ -14,7 +14,6 @@
 
 import spack.build_environment
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
 import spack.fetch_strategy
@@ -23,6 +22,7 @@
 import spack.report
 import spack.spec
 import spack.store
+from spack.cmd.common import arguments
 from spack.error import SpackError
 from spack.installer import PackageInstaller
 
@@ -162,8 +162,8 @@ def setup_parser(subparser):
         "--no-check-signature",
         action="store_true",
         dest="unsigned",
-        default=False,
-        help="do not check signatures of binary packages",
+        default=None,
+        help="do not check signatures of binary packages (override mirror config)",
     )
     subparser.add_argument(
         "--show-log-on-error",

lib/spack/spack/cmd/list.py

@@ -15,9 +15,9 @@
 import llnl.util.tty as tty
 from llnl.util.tty.colify import colify
 
-import spack.cmd.common.arguments as arguments
 import spack.deptypes as dt
 import spack.repo
+from spack.cmd.common import arguments
 from spack.version import VersionList
 
 description = "list and search available packages"

lib/spack/spack/cmd/load.py

@@ -8,12 +8,12 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.cmd.find
 import spack.environment as ev
 import spack.store
 import spack.user_environment as uenv
 import spack.util.environment
+from spack.cmd.common import arguments
 
 description = "add package to the user environment"
 section = "user environment"
@@ -98,15 +98,15 @@ def load(parser, args):
         spack.cmd.display_specs(results)
         return
 
+    constraint_specs = spack.cmd.parse_specs(args.constraint)
     specs = [
-        spack.cmd.disambiguate_spec(spec, env, first=args.load_first)
-        for spec in spack.cmd.parse_specs(args.constraint)
+        spack.cmd.disambiguate_spec(spec, env, first=args.load_first) for spec in constraint_specs
     ]
 
     if not args.shell:
-        specs_str = " ".join(args.constraint) or "SPECS"
+        specs_str = " ".join(str(s) for s in constraint_specs) or "SPECS"
         spack.cmd.common.shell_init_instructions(
-            "spack load", "    eval `spack load {sh_arg} %s`" % specs_str
+            "spack load", f"    eval `spack load {{sh_arg}} {specs_str}`"
         )
         return 1
 

lib/spack/spack/cmd/location.py

@@ -9,11 +9,11 @@
 
 import spack.builder
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.paths
 import spack.repo
 import spack.stage
+from spack.cmd.common import arguments
 
 description = "print out locations of packages and spack directories"
 section = "basic"

lib/spack/spack/cmd/mark.py

@@ -8,11 +8,11 @@
 from llnl.util import tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.error
 import spack.package_base
 import spack.repo
 import spack.store
+from spack.cmd.common import arguments
 from spack.database import InstallStatuses
 
 description = "mark packages as explicitly or implicitly installed"

lib/spack/spack/cmd/mirror.py

@@ -11,7 +11,6 @@
 
 import spack.caches
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.concretize
 import spack.config
 import spack.environment as ev
@@ -20,6 +19,7 @@
 import spack.spec
 import spack.util.path
 import spack.util.web as web_util
+from spack.cmd.common import arguments
 from spack.error import SpackError
 
 description = "manage mirrors (source and binary)"
@@ -88,18 +88,14 @@ def setup_parser(subparser):
         "--mirror-url", metavar="mirror_url", type=str, help="find mirror to destroy by url"
     )
 
-    # used to construct scope arguments below
-    scopes = spack.config.scopes()
-
     # Add
     add_parser = sp.add_parser("add", help=mirror_add.__doc__)
     add_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror")
     add_parser.add_argument("url", help="url of mirror directory from 'spack mirror create'")
     add_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope to modify",
     )
     add_parser.add_argument(
@@ -111,15 +107,31 @@ def setup_parser(subparser):
             "and source use `--type binary --type source` (default)"
         ),
     )
+    add_parser_signed = add_parser.add_mutually_exclusive_group(required=False)
+    add_parser_signed.add_argument(
+        "--unsigned",
+        help="do not require signing and signature verification when pushing and installing from "
+        "this build cache",
+        action="store_false",
+        default=None,
+        dest="signed",
+    )
+    add_parser_signed.add_argument(
+        "--signed",
+        help="require signing and signature verification when pushing and installing from this "
+        "build cache",
+        action="store_true",
+        default=None,
+        dest="signed",
+    )
     arguments.add_connection_args(add_parser, False)
     # Remove
     remove_parser = sp.add_parser("remove", aliases=["rm"], help=mirror_remove.__doc__)
     remove_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror")
     remove_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope to modify",
     )
 
@@ -136,9 +148,8 @@ def setup_parser(subparser):
     )
     set_url_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope to modify",
     )
     arguments.add_connection_args(set_url_parser, False)
@@ -163,11 +174,27 @@ def setup_parser(subparser):
         ),
     )
     set_parser.add_argument("--url", help="url of mirror directory from 'spack mirror create'")
+    set_parser_unsigned = set_parser.add_mutually_exclusive_group(required=False)
+    set_parser_unsigned.add_argument(
+        "--unsigned",
+        help="do not require signing and signature verification when pushing and installing from "
+        "this build cache",
+        action="store_false",
+        default=None,
+        dest="signed",
+    )
+    set_parser_unsigned.add_argument(
+        "--signed",
+        help="require signing and signature verification when pushing and installing from this "
+        "build cache",
+        action="store_true",
+        default=None,
+        dest="signed",
+    )
     set_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope to modify",
     )
     arguments.add_connection_args(set_parser, False)
@@ -175,11 +202,7 @@ def setup_parser(subparser):
     # List
     list_parser = sp.add_parser("list", help=mirror_list.__doc__)
     list_parser.add_argument(
-        "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_list_scope(),
-        help="configuration scope to read from",
+        "--scope", action=arguments.ConfigScope, help="configuration scope to read from"
     )
 
 
@@ -194,6 +217,7 @@ def mirror_add(args):
         or args.type
         or args.oci_username
         or args.oci_password
+        or args.signed is not None
     ):
         connection = {"url": args.url}
         if args.s3_access_key_id and args.s3_access_key_secret:
@@ -209,6 +233,8 @@ def mirror_add(args):
         if args.type:
             connection["binary"] = "binary" in args.type
             connection["source"] = "source" in args.type
+        if args.signed is not None:
+            connection["signed"] = args.signed
         mirror = spack.mirror.Mirror(connection, name=args.name)
     else:
         mirror = spack.mirror.Mirror(args.url, name=args.name)
@@ -241,6 +267,8 @@ def _configure_mirror(args):
         changes["endpoint_url"] = args.s3_endpoint_url
     if args.oci_username and args.oci_password:
         changes["access_pair"] = [args.oci_username, args.oci_password]
+    if getattr(args, "signed", None) is not None:
+        changes["signed"] = args.signed
 
     # argparse cannot distinguish between --binary and --no-binary when same dest :(
     # notice that set-url does not have these args, so getattr

lib/spack/spack/cmd/modules/__init__.py

@@ -14,11 +14,11 @@
 from llnl.util.tty import color
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.modules
 import spack.modules.common
 import spack.repo
+from spack.cmd.common import arguments
 
 description = "manipulate module files"
 section = "environment"
@@ -388,21 +388,15 @@ def modules_cmd(parser, args, module_type, callbacks=callbacks):
         callbacks[args.subparser_name](module_type, specs, args)
 
     except MultipleSpecsMatch:
-        msg = "the constraint '{query}' matches multiple packages:\n"
+        query = " ".join(str(s) for s in args.constraint_specs)
+        msg = f"the constraint '{query}' matches multiple packages:\n"
         for s in specs:
             spec_fmt = "{hash:7} {name}{@version}{%compiler}"
             spec_fmt += "{compiler_flags}{variants}{arch=architecture}"
             msg += "\t" + s.cformat(spec_fmt) + "\n"
-        tty.error(msg.format(query=args.constraint))
-        tty.die(
-            "In this context exactly **one** match is needed: "
-            "please specify your constraints better."
-        )
+        tty.die(msg, "In this context exactly *one* match is needed.")
 
     except NoSpecMatches:
-        msg = "the constraint '{query}' matches no package."
-        tty.error(msg.format(query=args.constraint))
-        tty.die(
-            "In this context exactly **one** match is needed: "
-            "please specify your constraints better."
-        )
+        query = " ".join(str(s) for s in args.constraint_specs)
+        msg = f"the constraint '{query}' matches no package."
+        tty.die(msg, "In this context exactly *one* match is needed.")

lib/spack/spack/cmd/patch.py

@@ -6,12 +6,12 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
 import spack.package_base
 import spack.repo
 import spack.traverse
+from spack.cmd.common import arguments
 
 description = "patch expanded archive sources in preparation for install"
 section = "build"

lib/spack/spack/cmd/pkg.py

@@ -12,11 +12,11 @@
 from llnl.util.tty.colify import colify
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.paths
 import spack.repo
 import spack.util.executable as exe
 import spack.util.package_hash as ph
+from spack.cmd.common import arguments
 
 description = "query packages associated with particular git revisions"
 section = "developer"

lib/spack/spack/cmd/remove.py

@@ -6,7 +6,7 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
+from spack.cmd.common import arguments
 
 description = "remove specs from an environment"
 section = "environments"

lib/spack/spack/cmd/repo.py

@@ -11,6 +11,7 @@
 import spack.config
 import spack.repo
 import spack.util.path
+from spack.cmd.common import arguments
 
 description = "manage package source repositories"
 section = "config"
@@ -19,7 +20,6 @@
 
 def setup_parser(subparser):
     sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="repo_command")
-    scopes = spack.config.scopes()
 
     # Create
     create_parser = sp.add_parser("create", help=repo_create.__doc__)
@@ -42,11 +42,7 @@ def setup_parser(subparser):
     # List
     list_parser = sp.add_parser("list", help=repo_list.__doc__)
     list_parser.add_argument(
-        "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_list_scope(),
-        help="configuration scope to read from",
+        "--scope", action=arguments.ConfigScope, help="configuration scope to read from"
     )
 
     # Add
@@ -54,9 +50,8 @@ def setup_parser(subparser):
     add_parser.add_argument("path", help="path to a Spack package repository directory")
     add_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope to modify",
     )
 
@@ -67,9 +62,8 @@ def setup_parser(subparser):
     )
     remove_parser.add_argument(
         "--scope",
-        choices=scopes,
-        metavar=spack.config.SCOPES_METAVAR,
-        default=spack.config.default_modify_scope(),
+        action=arguments.ConfigScope,
+        default=lambda: spack.config.default_modify_scope(),
         help="configuration scope to modify",
     )
 

lib/spack/spack/cmd/restage.py

@@ -6,8 +6,8 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.repo
+from spack.cmd.common import arguments
 
 description = "revert checked out package source code"
 section = "build"

lib/spack/spack/cmd/solve.py

@@ -12,12 +12,12 @@
 
 import spack
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment
 import spack.hash_types as ht
 import spack.package_base
 import spack.solver.asp as asp
+from spack.cmd.common import arguments
 
 description = "concretize a specs using an ASP solver"
 section = "developer"

lib/spack/spack/cmd/spec.py

@@ -10,11 +10,11 @@
 
 import spack
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.environment as ev
 import spack.hash_types as ht
 import spack.spec
 import spack.store
+from spack.cmd.common import arguments
 
 description = "show what would be installed, given a spec"
 section = "build"

lib/spack/spack/cmd/stage.py

@@ -8,13 +8,13 @@
 import llnl.util.tty as tty
 
 import spack.cmd
-import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.environment as ev
 import spack.package_base
 import spack.repo
 import spack.stage
 import spack.traverse
+from spack.cmd.common import arguments
 
 description = "expand downloaded archive in preparation for install"
 section = "build"