asio: remove self-referential dependencies (#42469)

These shouldn't be an issue, but they can be expressed
in terms of variants on the package.

.github/workflows/audit.yaml

@@ -23,7 +23,7 @@ jobs:
         operating_system: ["ubuntu-latest", "macos-latest"]
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
       with:
         python-version: ${{inputs.python_version}}
     - name: Install Python packages

.github/workflows/bootstrap.yml

@@ -159,7 +159,7 @@ jobs:
           brew install cmake bison@2.7 tree
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
-      - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
+      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
         with:
           python-version: "3.12"
       - name: Bootstrap clingo

.github/workflows/build-containers.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
 
-      - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934
+      - uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c
         id: docker_meta
         with:
           images: |
@@ -113,7 +113,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build & Deploy ${{ matrix.dockerfile[0] }}
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56
         with:
           context: dockerfiles/${{ matrix.dockerfile[0] }}
           platforms: ${{ matrix.dockerfile[1] }}

.github/workflows/nightly-win-builds.yml

@@ -17,7 +17,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: 3.9
     - name: Install Python packages

.github/workflows/style/requirements.txt

@@ -1,7 +1,7 @@
-black==23.11.0
+black==23.12.1
 clingo==5.6.2
-flake8==6.1.0
-isort==5.12.0
-mypy==1.6.1
+flake8==7.0.0
+isort==5.13.2
+mypy==1.8.0
 types-six==1.16.21.9
-vermin==1.5.2
+vermin==1.6.0

.github/workflows/unit_tests.yaml

@@ -54,7 +54,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install System packages
@@ -101,7 +101,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -159,7 +159,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -194,7 +194,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Python packages
@@ -215,7 +215,7 @@ jobs:
         $(which spack) bootstrap disable spack-install
         $(which spack) solve zlib
         common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
-        $(which spack) unit-test --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
+        $(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
     - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
       with:
         flags: unittests,macos

.github/workflows/valid-style.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: '3.11'
         cache: 'pip'
@@ -38,7 +38,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: '3.11'
         cache: 'pip'

.github/workflows/windows_python.yml

@@ -18,7 +18,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: 3.9
     - name: Install Python packages
@@ -42,7 +42,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: 3.9
     - name: Install Python packages
@@ -66,7 +66,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
+    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: 3.9
     - name: Install Python packages

CHANGELOG.md

@@ -1,3 +1,290 @@
+# v0.21.0 (2023-11-11)
+
+`v0.21.0` is a major feature release.
+
+## Features in this release
+
+1. **Better error messages with condition chaining**
+
+   In v0.18, we added better error messages that could tell you what problem happened,
+   but they couldn't tell you *why* it happened. `0.21` adds *condition chaining* to the
+   solver, and Spack can now trace back through the conditions that led to an error and
+   build a tree of causes potential causes and where they came from. For example:
+
+   ```console
+   $ spack solve hdf5 ^cmake@3.0.1
+   ==> Error: concretization failed for the following reasons:
+
+      1. Cannot satisfy 'cmake@3.0.1'
+      2. Cannot satisfy 'cmake@3.0.1'
+           required because hdf5 ^cmake@3.0.1 requested from CLI
+      3. Cannot satisfy 'cmake@3.18:' and 'cmake@3.0.1
+           required because hdf5 ^cmake@3.0.1 requested from CLI
+           required because hdf5 depends on cmake@3.18: when @1.13:
+             required because hdf5 ^cmake@3.0.1 requested from CLI
+      4. Cannot satisfy 'cmake@3.12:' and 'cmake@3.0.1
+           required because hdf5 depends on cmake@3.12:
+             required because hdf5 ^cmake@3.0.1 requested from CLI
+           required because hdf5 ^cmake@3.0.1 requested from CLI
+   ```
+
+   More details in #40173.
+
+2. **OCI build caches**
+
+   You can now use an arbitrary [OCI](https://opencontainers.org) registry as a build
+   cache:
+
+   ```console
+   $ spack mirror add my_registry oci://user/image # Dockerhub
+   $ spack mirror add my_registry oci://ghcr.io/haampie/spack-test # GHCR
+   $ spack mirror set --push --oci-username ... --oci-password ... my_registry  # set login creds
+   $ spack buildcache push my_registry [specs...]
+   ```
+
+   And you can optionally add a base image to get *runnable* images:
+
+   ```console
+   $ spack buildcache push --base-image ubuntu:23.04 my_registry python
+   Pushed ... as [image]:python-3.11.2-65txfcpqbmpawclvtasuog4yzmxwaoia.spack
+
+   $ docker run --rm -it [image]:python-3.11.2-65txfcpqbmpawclvtasuog4yzmxwaoia.spack
+   ```
+
+   This creates a container image from the Spack installations on the host system,
+   without the need to run `spack install` from a `Dockerfile` or `sif` file. It also
+   addresses the inconvenience of losing binaries of dependencies when `RUN spack
+   install` fails inside `docker build`.
+
+   Further, the container image layers and build cache tarballs are the same files. This
+   means that `spack install` and `docker pull` use the exact same underlying binaries.
+   If you previously used `spack install` inside of `docker build`, this feature helps
+   you save storage by a factor two.
+
+   More details in #38358.
+
+3. **Multiple versions of build dependencies**
+
+   Increasingly, complex package builds require multiple versions of some build
+   dependencies. For example, Python packages frequently require very specific versions
+   of `setuptools`, `cython`, and sometimes different physics packages require different
+   versions of Python to build. The concretizer enforced that every solve was *unified*,
+   i.e., that there only be one version of every package. The concretizer now supports
+   "duplicate" nodes for *build dependencies*, but enforces unification through
+   transitive link and run dependencies. This will allow it to better resolve complex
+   dependency graphs in ecosystems like Python, and it also gets us very close to
+   modeling compilers as proper dependencies.
+
+   This change required a major overhaul of the concretizer, as well as a number of
+   performance optimizations. See #38447, #39621.
+
+4. **Cherry-picking virtual dependencies**
+
+   You can now select only a subset of virtual dependencies from a spec that may provide
+   more. For example, if you want `mpich` to be your `mpi` provider, you can be explicit
+   by writing:
+
+   ```
+   hdf5 ^[virtuals=mpi] mpich
+   ```
+
+   Or, if you want to use, e.g., `intel-parallel-studio` for `blas` along with an external
+   `lapack` like `openblas`, you could write:
+
+   ```
+   strumpack ^[virtuals=mpi] intel-parallel-studio+mkl ^[virtuals=lapack] openblas
+   ```
+
+   The `virtuals=mpi` is an edge attribute, and dependency edges in Spack graphs now
+   track which virtuals they satisfied. More details in #17229 and #35322.
+
+   Note for packaging: in Spack 0.21 `spec.satisfies("^virtual")` is true if and only if
+   the package specifies `depends_on("virtual")`. This is different from Spack 0.20,
+   where depending on a provider implied depending on the virtual provided. See #41002
+   for an example where `^mkl` was being used to test for several `mkl` providers in a
+   package that did not depend on `mkl`.
+
+5. **License directive**
+
+   Spack packages can now have license metadata, with the new `license()` directive:
+
+   ```python
+       license("Apache-2.0")
+   ```
+
+   Licenses use [SPDX identifiers](https://spdx.org/licenses), and you can use SPDX
+   expressions to combine them:
+
+   ```python
+       license("Apache-2.0 OR MIT")
+   ```
+
+   Like other directives in Spack, it's conditional, so you can handle complex cases like
+   Spack itself:
+
+   ```python
+      license("LGPL-2.1", when="@:0.11")
+      license("Apache-2.0 OR MIT", when="@0.12:")
+   ```
+
+   More details in #39346, #40598.
+
+6. **`spack deconcretize` command**
+
+   We are getting close to having a `spack update` command for environments, but we're
+   not quite there yet. This is the next best thing. `spack deconcretize` gives you
+   control over what you want to update in an already concrete environment. If you have
+   an environment built with, say, `meson`, and you want to update your `meson` version,
+   you can run:
+
+   ```console
+   spack deconcretize meson
+   ```
+
+   and have everything that depends on `meson` rebuilt the next time you run `spack
+   concretize`. In a future Spack version, we'll handle all of this in a single command,
+   but for now you can use this to drop bits of your lockfile and resolve your
+   dependencies again. More in #38803.
+
+7. **UI Improvements**
+
+   The venerable `spack info` command was looking shabby compared to the rest of Spack's
+   UI, so we reworked it to have a bit more flair. `spack info` now makes much better
+   use of terminal space and shows variants, their values, and their descriptions much
+   more clearly. Conditional variants are grouped separately so you can more easily
+   understand how packages are structured. More in #40998.
+
+   `spack checksum` now allows you to filter versions from your editor, or by version
+   range. It also notifies you about potential download URL changes. See #40403.
+
+8. **Environments can include definitions**
+
+   Spack did not previously support using `include:` with The
+   [definitions](https://spack.readthedocs.io/en/latest/environments.html#spec-list-references)
+   section of an environment, but now it does. You can use this to curate lists of specs
+   and more easily reuse them across environments. See #33960.
+
+9. **Aliases**
+
+   You can now add aliases to Spack commands in `config.yaml`, e.g. this might enshrine
+   your favorite args to `spack find` as `spack f`:
+
+   ```yaml
+   config:
+     aliases:
+       f: find -lv
+   ```
+
+   See #17229.
+
+10. **Improved autoloading of modules**
+
+    Spack 0.20 was the first release to enable autoloading of direct dependencies in
+    module files.
+
+    The downside of this was that `module avail` and `module load` tab completion would
+    show users too many modules to choose from, and many users disabled generating
+    modules for dependencies through `exclude_implicits: true`. Further, it was
+    necessary to keep hashes in module names to avoid file name clashes.
+
+    In this release, you can start using `hide_implicits: true` instead, which exposes
+    only explicitly installed packages to the user, while still autoloading
+    dependencies. On top of that, you can safely use `hash_length: 0`, as this config
+    now only applies to the modules exposed to the user -- you don't have to worry about
+    file name clashes for hidden dependencies.
+
+   Note: for `tcl` this feature requires Modules 4.7 or higher
+
+11. **Updated container labeling**
+
+    Nightly Docker images from the `develop` branch will now be tagged as `:develop` and
+    `:nightly`. The `:latest` tag is no longer associated with `:develop`, but with the
+    latest stable release. Releases will be tagged with `:{major}`, `:{major}.{minor}`
+    and `:{major}.{minor}.{patch}`. `ubuntu:18.04` has also been removed from the list of
+    generated Docker images, as it is no longer supported. See #40593.
+
+## Other new commands and directives
+
+* `spack env activate` without arguments now loads a `default` environment that you do
+  not have to create (#40756).
+* `spack find -H` / `--hashes`: a new shortcut for piping `spack find` output to
+  other commands (#38663)
+* Add `spack checksum --verify`, fix `--add` (#38458)
+* New `default_args` context manager factors out common args for directives (#39964)
+* `spack compiler find --[no]-mixed-toolchain` lets you easily mix `clang` and
+  `gfortran` on Linux (#40902)
+
+## Performance improvements
+
+* `spack external find` execution is now much faster (#39843)
+* `spack location -i` now much faster on success (#40898)
+* Drop redundant rpaths post install (#38976)
+* ASP-based solver: avoid cycles in clingo using hidden directive (#40720)
+* Fix multiple quadratic complexity issues in environments (#38771)
+
+## Other new features of note
+
+* archspec: update to v0.2.2, support for Sapphire Rapids, Power10, Neoverse V2 (#40917)
+* Propagate variants across nodes that don't have that variant (#38512)
+* Implement fish completion (#29549)
+* Can now distinguish between source/binary mirror; don't ping mirror.spack.io as much (#34523)
+* Improve status reporting on install (add [n/total] display) (#37903)
+
+## Windows
+
+This release has the best Windows support of any Spack release yet, with numerous
+improvements and much larger swaths of tests passing:
+
+* MSVC and SDK improvements (#37711, #37930, #38500, #39823, #39180)
+* Windows external finding: update default paths; treat .bat as executable on Windows (#39850)
+* Windows decompression: fix removal of intermediate file (#38958)
+* Windows: executable/path handling (#37762)
+* Windows build systems: use ninja and enable tests (#33589)
+* Windows testing (#36970, #36972, #36973, #36840, #36977, #36792, #36834, #34696, #36971)
+* Windows PowerShell support (#39118, #37951)
+* Windows symlinking and libraries (#39933, #38599, #34701, #38578, #34701)
+
+## Notable refactors
+* User-specified flags take precedence over others in Spack compiler wrappers (#37376)
+* Improve setup of build, run, and test environments (#35737, #40916)
+* `make` is no longer a required system dependency of Spack (#40380)
+* Support Python 3.12 (#40404, #40155, #40153)
+* docs: Replace package list with packages.spack.io (#40251)
+* Drop Python 2 constructs in Spack (#38720, #38718, #38703)
+
+## Binary cache and stack updates
+* e4s arm stack: duplicate and target neoverse v1 (#40369)
+* Add macOS ML CI stacks (#36586)
+* E4S Cray CI Stack (#37837)
+* e4s cray: expand spec list (#38947)
+* e4s cray sles ci: expand spec list (#39081)
+
+## Removals, deprecations, and syntax changes
+* ASP: targets, compilers and providers soft-preferences are only global (#31261)
+* Parser: fix ambiguity with whitespace in version ranges (#40344)
+* Module file generation is disabled by default; you'll need to enable it to use it (#37258)
+* Remove deprecated "extra_instructions" option for containers (#40365)
+* Stand-alone test feature deprecation postponed to v0.22 (#40600)
+* buildcache push: make `--allow-root` the default and deprecate the option (#38878)
+
+## Notable Bugfixes
+* Bugfix: propagation of multivalued variants (#39833)
+* Allow `/` in git versions (#39398)
+* Fetch & patch: actually acquire stage lock, and many more issues (#38903)
+* Environment/depfile: better escaping of targets with Git versions (#37560)
+* Prevent "spack external find" to error out on wrong permissions (#38755)
+* lmod: allow core compiler to be specified with a version range (#37789)
+
+## Spack community stats
+
+* 7,469 total packages, 303 new since `v0.20.0`
+    * 150 new Python packages
+    * 34 new R packages
+* 353 people contributed to this release
+    * 336 committers to packages
+    * 65 committers to core
+
+
 # v0.20.3 (2023-10-31)
 
 ## Bugfixes

CITATION.cff

@@ -31,13 +31,17 @@ type: software
 message: "If you are referencing Spack in a publication, please cite the paper below."
 title: "The Spack Package Manager: Bringing Order to HPC Software Chaos"
 abstract: >-
-  Large HPC centers spend considerable time supporting software for thousands of users, but the complexity of HPC software is quickly outpacing the capabilities of existing software management tools.
-  Scientific applications require specific versions of compilers, MPI, and other dependency libraries, so using a single, standard software stack is infeasible.
-  However, managing many configurations is difficult because the configuration space is combinatorial in size.
-  We introduce Spack, a tool used at Lawrence Livermore National Laboratory to manage this complexity.
-  Spack provides a novel, re- cursive specification syntax to invoke parametric builds of packages and dependencies.
-  It allows any number of builds to coexist on the same system, and it ensures that installed packages can find their dependencies, regardless of the environment.
-  We show through real-world use cases that Spack supports diverse and demanding applications, bringing order to HPC software chaos.
+  Large HPC centers spend considerable time supporting software for thousands of users, but the
+  complexity of HPC software is quickly outpacing the capabilities of existing software management
+  tools. Scientific applications require specific versions of compilers, MPI, and other dependency
+  libraries, so using a single, standard software stack is infeasible. However, managing many
+  configurations is difficult because the configuration space is combinatorial in size. We
+  introduce Spack, a tool used at Lawrence Livermore National Laboratory to manage this complexity.
+  Spack provides a novel, re- cursive specification syntax to invoke parametric builds of packages
+  and dependencies. It allows any number of builds to coexist on the same system, and it ensures
+  that installed packages can find their dependencies, regardless of the environment. We show
+  through real-world use cases that Spack supports diverse and demanding applications, bringing
+  order to HPC software chaos.
 preferred-citation:
   title: "The Spack Package Manager: Bringing Order to HPC Software Chaos"
   type: conference-paper
@@ -71,7 +75,7 @@ preferred-citation:
       type: doi
       value: 10.1145/2807591.2807623
     - description: "The DOE Document Release Number of the work"
-      type: other 
+      type: other
       value: "LLNL-CONF-669890"
 authors:
   - family-names: "Gamblin"

LICENSE-MIT

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2013-2023 LLNS, LLC and other Spack Project Developers.
+Copyright (c) 2013-2024 LLNS, LLC and other Spack Project Developers.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,13 +1,34 @@
-# <img src="https://cdn.rawgit.com/spack/spack/develop/share/spack/logo/spack-logo.svg" width="64" valign="middle" alt="Spack"/> Spack
+<div align="left">
 
-[![Unit Tests](https://github.com/spack/spack/workflows/linux%20tests/badge.svg)](https://github.com/spack/spack/actions)
-[![Bootstrapping](https://github.com/spack/spack/actions/workflows/bootstrap.yml/badge.svg)](https://github.com/spack/spack/actions/workflows/bootstrap.yml)
-[![codecov](https://codecov.io/gh/spack/spack/branch/develop/graph/badge.svg)](https://codecov.io/gh/spack/spack)
-[![Containers](https://github.com/spack/spack/actions/workflows/build-containers.yml/badge.svg)](https://github.com/spack/spack/actions/workflows/build-containers.yml)
-[![Read the Docs](https://readthedocs.org/projects/spack/badge/?version=latest)](https://spack.readthedocs.io)
-[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
-[![Slack](https://slack.spack.io/badge.svg)](https://slack.spack.io)
-[![Matrix](https://img.shields.io/matrix/spack-space%3Amatrix.org?label=Matrix)](https://matrix.to/#/#spack-space:matrix.org)
+<h2>
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://cdn.rawgit.com/spack/spack/develop/share/spack/logo/spack-logo-white-text.svg" width="250">
+  <source media="(prefers-color-scheme: light)" srcset="https://cdn.rawgit.com/spack/spack/develop/share/spack/logo/spack-logo-text.svg" width="250">
+  <img alt="Spack" src="https://cdn.rawgit.com/spack/spack/develop/share/spack/logo/spack-logo-text.svg" width="250">
+</picture>
+
+<br>
+<br clear="all">
+
+<a href="https://github.com/spack/spack/actions/workflows/ci.yml"><img src="https://github.com/spack/spack/workflows/ci/badge.svg" alt="CI Status"></a>
+<a href="https://github.com/spack/spack/actions/workflows/bootstrapping.yml"><img src="https://github.com/spack/spack/actions/workflows/bootstrap.yml/badge.svg" alt="Bootstrap Status"></a>
+<a href="https://github.com/spack/spack/actions/workflows/build-containers.yml"><img src="https://github.com/spack/spack/actions/workflows/build-containers.yml/badge.svg" alt="Containers Status"></a>
+<a href="https://spack.readthedocs.io"><img src="https://readthedocs.org/projects/spack/badge/?version=latest" alt="Documentation Status"></a>
+<a href="https://codecov.io/gh/spack/spack"><img src="https://codecov.io/gh/spack/spack/branch/develop/graph/badge.svg" alt="Code coverage"/></a>
+<a href="https://slack.spack.io"><img src="https://slack.spack.io/badge.svg" alt="Slack"/></a>
+<a href="https://matrix.to/#/#spack-space:matrix.org"><img src="https://img.shields.io/matrix/spack-space%3Amatrix.org?label=matrix" alt="Matrix"/></a>
+
+</h2>
+
+**[Getting Started] &nbsp; • &nbsp; [Config] &nbsp; • &nbsp; [Community] &nbsp; • &nbsp; [Contributing] &nbsp; • &nbsp; [Packaging Guide]**
+
+[Getting Started]: https://spack.readthedocs.io/en/latest/getting_started.html
+[Config]: https://spack.readthedocs.io/en/latest/configuration.html
+[Community]: #community
+[Contributing]: https://spack.readthedocs.io/en/latest/contribution_guide.html
+[Packaging Guide]: https://spack.readthedocs.io/en/latest/packaging_guide.html
+
+</div>
 
 Spack is a multi-platform package manager that builds and installs
 multiple versions and configurations of software. It works on Linux,
@@ -66,10 +87,11 @@ Resources:
 * **Matrix space**: [#spack-space:matrix.org](https://matrix.to/#/#spack-space:matrix.org):
   [bridged](https://github.com/matrix-org/matrix-appservice-slack#matrix-appservice-slack) to Slack.
 * [**Github Discussions**](https://github.com/spack/spack/discussions):
-  not just for discussions, but also Q&A.
-* **Mailing list**: [groups.google.com/d/forum/spack](https://groups.google.com/d/forum/spack)
+  for Q&A and discussions. Note the pinned discussions for announcements.
 * **Twitter**: [@spackpm](https://twitter.com/spackpm). Be sure to
   `@mention` us!
+* **Mailing list**: [groups.google.com/d/forum/spack](https://groups.google.com/d/forum/spack):
+  only for announcements. Please use other venues for discussions.
 
 Contributing
 ------------------------

bin/haspywin.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

bin/sbang

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # sbang project developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

bin/spack

@@ -1,7 +1,7 @@
 #!/bin/sh
 # -*- python -*-
 #
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

bin/spack-python

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

bin/spack.bat

@@ -1,4 +1,4 @@
-:: Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+:: Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 :: Spack Project Developers. See the top-level COPYRIGHT file for details.
 ::
 :: SPDX-License-Identifier: (Apache-2.0 OR MIT)

bin/spack.ps1

@@ -1,4 +1,4 @@
-#  Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+#  Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 #  Spack Project Developers. See the top-level COPYRIGHT file for details.
 
 #  SPDX-License-Identifier: (Apache-2.0 OR MIT)

bin/spack_pwsh.ps1

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

etc/spack/defaults/darwin/packages.yaml

@@ -50,4 +50,4 @@ packages:
     # Apple bundles libuuid in libsystem_c version 1353.100.2,
     # although the version number used here isn't critical
     - spec: apple-libuuid@1353.100.2
-      prefix: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk
+      prefix: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk

lib/spack/docs/_pygments/style.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/basic_usage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/binary_caches.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -153,7 +153,43 @@ keyring, and trusting all downloaded keys.
 List of popular build caches
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-* `Extreme-scale Scientific Software Stack (E4S) <https://e4s-project.github.io/>`_: `build cache <https://oaciss.uoregon.edu/e4s/inventory.html>`_
+* `Extreme-scale Scientific Software Stack (E4S) <https://e4s-project.github.io/>`_: `build cache <https://oaciss.uoregon.edu/e4s/inventory.html>`_'
+
+-------------------
+Build cache signing
+-------------------
+
+By default, Spack will add a cryptographic signature to each package pushed to
+a build cache, and verifies the signature when installing from a build cache.
+
+Keys for signing can be managed with the :ref:`spack gpg <cmd-spack-gpg>` command,
+as well as ``spack buildcache keys`` as mentioned above.
+
+You can disable signing when pushing with ``spack buildcache push --unsigned``,
+and disable verification when installing from any build cache with
+``spack install --no-check-signature``.
+
+Alternatively, signing and verification can be enabled or disabled on a per build cache
+basis:
+
+.. code-block:: console
+
+    $ spack mirror add --signed <name> <url>  # enable signing and verification
+    $ spack mirror add --unsigned <name> <url>  # disable signing and verification
+
+    $ spack mirror set --signed <name>  # enable signing and verification for an existing mirror
+    $ spack mirror set --unsigned <name>  # disable signing and verification for an existing mirror
+
+Or you can directly edit the ``mirrors.yaml`` configuration file:
+
+.. code-block:: yaml
+
+    mirrors:
+      <name>:
+        url: <url>
+        signed: false # disable signing and verification
+
+See also :ref:`mirrors`.
 
 ----------
 Relocation
@@ -182,6 +218,7 @@ section of the configuration:
        padded_length: 128
 
 
+.. _binary_caches_oci:
 
 -----------------------------------------
 OCI / Docker V2 registries as build cache
@@ -250,87 +287,13 @@ To significantly speed up Spack in GitHub Actions, binaries can be cached in
 GitHub Packages. This service is an OCI registry that can be linked to a GitHub
 repository.
 
-A typical workflow is to include a ``spack.yaml`` environment in your repository
-that specifies the packages to install, the target architecture, and the build
-cache to use under ``mirrors``:
-
-.. code-block:: yaml
-
-    spack:
-      specs:
-      - python@3.11
-      config:
-        install_tree:
-          root: /opt/spack
-          padded_length: 128
-      packages:
-        all:
-          require: target=x86_64_v2
-      mirrors:
-        local-buildcache: oci://ghcr.io/<organization>/<repository>
-
-A GitHub action can then be used to install the packages and push them to the
-build cache:
-
-.. code-block:: yaml
-
-    name: Install Spack packages
-
-    on: push
-
-    env:
-      SPACK_COLOR: always
-
-    jobs:
-      example:
-        runs-on: ubuntu-22.04
-        permissions:
-          packages: write
-        steps:
-        - name: Checkout
-          uses: actions/checkout@v3
-
-        - name: Checkout Spack
-          uses: actions/checkout@v3
-          with:
-            repository: spack/spack
-            path: spack
-
-        - name: Setup Spack
-          run: echo "$PWD/spack/bin" >> "$GITHUB_PATH"
-
-        - name: Concretize
-          run: spack -e . concretize
-
-        - name: Install
-          run: spack -e . install --no-check-signature
-
-        - name: Run tests
-            run: ./my_view/bin/python3 -c 'print("hello world")'
-
-        - name: Push to buildcache
-          run: |
-            spack -e . mirror set --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache
-            spack -e . buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache
-          if: ${{ !cancelled() }}
-
-The first time this action runs, it will build the packages from source and
-push them to the build cache. Subsequent runs will pull the binaries from the
-build cache. The concretizer will ensure that prebuilt binaries are favored
-over source builds.
-
-The build cache entries appear in the GitHub Packages section of your repository,
-and contain instructions for pulling and running them with ``docker`` or ``podman``.
-
-
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Using Spack's public build cache for GitHub Actions
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
 Spack offers a public build cache for GitHub Actions with a set of common packages,
 which lets you get started quickly. See the following resources for more information:
 
-* `spack/github-actions-buildcache <https://github.com/spack/github-actions-buildcache>`_
+* `spack/setup-spack <https://github.com/spack/setup-spack>`_ for setting up Spack in GitHub
+  Actions
+* `spack/github-actions-buildcache <https://github.com/spack/github-actions-buildcache>`_ for
+  more details on the public build cache
 
 .. _cmd-spack-buildcache:
 

lib/spack/docs/bootstrapping.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_settings.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -37,7 +37,11 @@ to enable reuse for a single installation, and you can use:
    spack install --fresh <spec>
 
 to do a fresh install if ``reuse`` is enabled by default.
-``reuse: true`` is the default.
+``reuse: dependencies`` is the default.
+
+.. seealso::
+
+   FAQ: :ref:`Why does Spack pick particular versions and variants? <faq-concretizer-precedence>`
 
 ------------------------------------------
 Selection of the target microarchitectures

lib/spack/docs/build_systems.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/autotoolspackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/bundlepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/cachedcmakepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/cmakepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -82,7 +82,7 @@ class already contains:
 
 .. code-block:: python
 
-   depends_on('cmake', type='build')
+   depends_on("cmake", type="build")
 
 
 If you need to specify a particular version requirement, you can
@@ -90,7 +90,7 @@ override this in your package:
 
 .. code-block:: python
 
-   depends_on('cmake@2.8.12:', type='build')
+   depends_on("cmake@2.8.12:", type="build")
 
 
 ^^^^^^^^^^^^^^^^^^^
@@ -137,10 +137,10 @@ and without the :meth:`~spack.build_systems.cmake.CMakeBuilder.define` and
 
    def cmake_args(self):
        args = [
-           '-DWHATEVER:STRING=somevalue',
-           self.define('ENABLE_BROKEN_FEATURE', False),
-           self.define_from_variant('DETECT_HDF5', 'hdf5'),
-           self.define_from_variant('THREADS'), # True if +threads
+           "-DWHATEVER:STRING=somevalue",
+           self.define("ENABLE_BROKEN_FEATURE", False),
+           self.define_from_variant("DETECT_HDF5", "hdf5"),
+           self.define_from_variant("THREADS"), # True if +threads
        ]
 
        return args
@@ -151,10 +151,10 @@ and CMake simply ignores the empty command line argument. For example the follow
 
 .. code-block:: python
 
-   variant('example', default=True, when='@2.0:')
+   variant("example", default=True, when="@2.0:")
 
    def cmake_args(self):
-      return [self.define_from_variant('EXAMPLE', 'example')]
+      return [self.define_from_variant("EXAMPLE", "example")]
 
 will generate ``'cmake' '-DEXAMPLE=ON' ...`` when `@2.0: +example` is met, but will
 result in ``'cmake' '' ...`` when the spec version is below ``2.0``.
@@ -193,9 +193,9 @@ a variant to control this:
 
 .. code-block:: python
 
-   variant('build_type', default='RelWithDebInfo',
-           description='CMake build type',
-           values=('Debug', 'Release', 'RelWithDebInfo', 'MinSizeRel'))
+   variant("build_type", default="RelWithDebInfo",
+           description="CMake build type",
+           values=("Debug", "Release", "RelWithDebInfo", "MinSizeRel"))
 
 However, not every CMake package accepts all four of these options.
 Grep the ``CMakeLists.txt`` file to see if the default values are
@@ -205,9 +205,9 @@ package overrides the default variant with:
 
 .. code-block:: python
 
-   variant('build_type', default='DebugRelease',
-           description='The build type to build',
-           values=('Debug', 'Release', 'DebugRelease'))
+   variant("build_type", default="DebugRelease",
+           description="The build type to build",
+           values=("Debug", "Release", "DebugRelease"))
 
 For more information on ``CMAKE_BUILD_TYPE``, see:
 https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html
@@ -250,7 +250,7 @@ generator is Ninja. To switch to the Ninja generator, simply add:
 
 .. code-block:: python
 
-   generator = 'Ninja'
+   generator = "Ninja"
 
 
 ``CMakePackage`` defaults to "Unix Makefiles". If you switch to the
@@ -258,7 +258,7 @@ Ninja generator, make sure to add:
 
 .. code-block:: python
 
-   depends_on('ninja', type='build')
+   depends_on("ninja", type="build")
 
 to the package as well. Aside from that, you shouldn't need to do
 anything else. Spack will automatically detect that you are using
@@ -288,7 +288,7 @@ like so:
 
 .. code-block:: python
 
-   root_cmakelists_dir = 'src'
+   root_cmakelists_dir = "src"
 
 
 Note that this path is relative to the root of the extracted tarball,
@@ -304,7 +304,7 @@ different sub-directory, simply override ``build_directory`` like so:
 
 .. code-block:: python
 
-   build_directory = 'my-build'
+   build_directory = "my-build"
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 Build and install targets
@@ -324,8 +324,8 @@ library or build the documentation, you can add these like so:
 
 .. code-block:: python
 
-   build_targets = ['all', 'docs']
-   install_targets = ['install', 'docs']
+   build_targets = ["all", "docs"]
+   install_targets = ["install", "docs"]
 
 ^^^^^^^
 Testing

lib/spack/docs/build_systems/cudapackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/custompackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/inteloneapipackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -53,18 +53,24 @@ Install the oneAPI compilers::
 
 Add the compilers to your ``compilers.yaml`` so spack can use them::
 
-  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64
-  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/bin
 
 Verify that the compilers are available::
 
   spack compiler list
 
+Note that 2024 and later releases do not include ``icc``. Before 2024,
+the package layout was different::
+  
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin
+
 The ``intel-oneapi-compilers`` package includes 2 families of
 compilers:
 
 * ``intel``: ``icc``, ``icpc``, ``ifort``. Intel's *classic*
-  compilers.
+  compilers. 2024 and later releases contain ``ifort``, but not
+  ``icc`` and ``icpc``.
 * ``oneapi``: ``icx``, ``icpx``, ``ifx``. Intel's new generation of
   compilers based on LLVM.
 
@@ -89,8 +95,8 @@ Install the oneAPI compilers::
 
 Add the compilers to your ``compilers.yaml`` so Spack can use them::
 
-  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64
-  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/bin
+  spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/bin
 
 Verify that the compilers are available::
 
@@ -146,8 +152,7 @@ Compilers
 To use the compilers, add some information about the installation to
 ``compilers.yaml``. For most users, it is sufficient to do::
 
-  spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin/intel64
-  spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin
+  spack compiler add /opt/intel/oneapi/compiler/latest/bin
 
 Adapt the paths above if you did not install the tools in the default
 location. After adding the compilers, using them is the same
@@ -156,6 +161,12 @@ Another option is to manually add the configuration to
 ``compilers.yaml`` as described in :ref:`Compiler configuration
 <compiler-config>`.
 
+Before 2024, the directory structure was different::
+  
+  spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin/intel64
+  spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin
+
+
 Libraries
 ---------
 

lib/spack/docs/build_systems/intelpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -90,7 +90,7 @@ and optimizers do require a paid license.  In Spack, they are packaged as:
     TODO: Confirm and possible change(!) the scope of MPI components (runtime
     vs. devel) in current (and previous?) *cluster/professional/composer*
     editions, i.e., presence in downloads, possibly subject to license
-    coverage(!); see `disussion in PR #4300
+    coverage(!); see `discussion in PR #4300
     <https://github.com/spack/spack/pull/4300#issuecomment-305582898>`_.  [NB:
     An "mpi" subdirectory is not indicative of the full MPI SDK being present
     (i.e., ``mpicc``, ..., and header files).  The directory may just as well
@@ -934,9 +934,9 @@ a *virtual* ``mkl`` package is declared in Spack.
   .. code-block:: python
 
      # Examples for absolute and conditional dependencies:
-     depends_on('mkl')
-     depends_on('mkl', when='+mkl')
-     depends_on('mkl', when='fftw=mkl')
+     depends_on("mkl")
+     depends_on("mkl", when="+mkl")
+     depends_on("mkl", when="fftw=mkl")
 
   The ``MKLROOT`` environment variable (part of the documented API) will be set
   during all stages of client package installation, and is available to both
@@ -972,8 +972,8 @@ a *virtual* ``mkl`` package is declared in Spack.
       def configure_args(self):
           args = []
           ...
-          args.append('--with-blas=%s' % self.spec['blas'].libs.ld_flags)
-          args.append('--with-lapack=%s' % self.spec['lapack'].libs.ld_flags)
+          args.append("--with-blas=%s" % self.spec["blas"].libs.ld_flags)
+          args.append("--with-lapack=%s" % self.spec["lapack"].libs.ld_flags)
           ...
 
   .. tip::
@@ -989,13 +989,13 @@ a *virtual* ``mkl`` package is declared in Spack.
 
   .. code-block:: python
 
-    self.spec['blas'].headers.include_flags
+    self.spec["blas"].headers.include_flags
 
   and to generate linker options (``-L<dir> -llibname ...``), use the same as above,
 
   .. code-block:: python
 
-    self.spec['blas'].libs.ld_flags
+    self.spec["blas"].libs.ld_flags
 
   See
   :ref:`MakefilePackage <makefilepackage>`

lib/spack/docs/build_systems/luapackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -88,7 +88,7 @@ override the ``luarocks_args`` method like so:
 .. code-block:: python
 
     def luarocks_args(self):
-        return ['flag1', 'flag2']
+        return ["flag1", "flag2"]
 
 One common use of this is to override warnings or flags for newer compilers, as in:
 

lib/spack/docs/build_systems/makefilepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -88,13 +88,13 @@ command-line. However, Makefiles that use ``?=`` for assignment honor
 environment variables. Since Spack already sets ``CC``, ``CXX``, ``F77``,
 and ``FC``, you won't need to worry about setting these variables. If
 there are any other variables you need to set, you can do this in the
-``edit`` method:
+``setup_build_environment`` method:
 
 .. code-block:: python
 
-   def edit(self, spec, prefix):
-       env["PREFIX"] = prefix
-       env["BLASLIB"] = spec["blas"].libs.ld_flags
+   def setup_build_environment(self, env):
+       env.set("PREFIX", prefix)
+       env.set("BLASLIB", spec["blas"].libs.ld_flags)
 
 
 `cbench <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/cbench/package.py>`_
@@ -140,7 +140,7 @@ Edit Makefile
 Some Makefiles are just plain stubborn and will ignore command-line
 variables. The only way to ensure that these packages build correctly
 is to directly edit the Makefile. Spack provides a ``FileFilter`` class
-and a ``filter_file`` method to help with this. For example:
+and a ``filter`` method to help with this. For example:
 
 .. code-block:: python
 

lib/spack/docs/build_systems/mavenpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -48,8 +48,8 @@ class automatically adds the following dependencies:
 
 .. code-block:: python
 
-   depends_on('java', type=('build', 'run'))
-   depends_on('maven', type='build')
+   depends_on("java", type=("build", "run"))
+   depends_on("maven", type="build")
 
 
 In the ``pom.xml`` file, you may see sections like:
@@ -72,8 +72,8 @@ should add:
 
 .. code-block:: python
 
-   depends_on('java@7:', type='build')
-   depends_on('maven@3.5.4:', type='build')
+   depends_on("java@7:", type="build")
+   depends_on("maven@3.5.4:", type="build")
 
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -88,9 +88,9 @@ the build phase. For example:
 
    def build_args(self):
        return [
-           '-Pdist,native',
-           '-Dtar',
-           '-Dmaven.javadoc.skip=true'
+           "-Pdist,native",
+           "-Dtar",
+           "-Dmaven.javadoc.skip=true"
        ]
 
 

lib/spack/docs/build_systems/mesonpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -86,8 +86,8 @@ the ``MesonPackage`` base class already contains:
 
 .. code-block:: python
 
-   depends_on('meson', type='build')
-   depends_on('ninja', type='build')
+   depends_on("meson", type="build")
+   depends_on("ninja", type="build")
 
 
 If you need to specify a particular version requirement, you can
@@ -95,8 +95,8 @@ override this in your package:
 
 .. code-block:: python
 
-   depends_on('meson@0.43.0:', type='build')
-   depends_on('ninja', type='build')
+   depends_on("meson@0.43.0:", type="build")
+   depends_on("ninja", type="build")
 
 
 ^^^^^^^^^^^^^^^^^^^
@@ -121,7 +121,7 @@ override the ``meson_args`` method like so:
 .. code-block:: python
 
    def meson_args(self):
-       return ['--warnlevel=3']
+       return ["--warnlevel=3"]
 
 
 This method can be used to pass flags as well as variables.

lib/spack/docs/build_systems/octavepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/perlpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -118,7 +118,7 @@ so ``PerlPackage`` contains:
 
 .. code-block:: python
 
-   extends('perl')
+   extends("perl")
 
 
 If your package requires a specific version of Perl, you should
@@ -132,14 +132,14 @@ properly. If your package uses ``Makefile.PL`` to build, add:
 
 .. code-block:: python
 
-   depends_on('perl-extutils-makemaker', type='build')
+   depends_on("perl-extutils-makemaker", type="build")
 
 
 If your package uses ``Build.PL`` to build, add:
 
 .. code-block:: python
 
-   depends_on('perl-module-build', type='build')
+   depends_on("perl-module-build", type="build")
 
 
 ^^^^^^^^^^^^^^^^^
@@ -165,11 +165,11 @@ arguments to ``Makefile.PL`` or ``Build.PL`` by overriding
 .. code-block:: python
 
    def configure_args(self):
-       expat = self.spec['expat'].prefix
+       expat = self.spec["expat"].prefix
 
        return [
-           'EXPATLIBPATH={0}'.format(expat.lib),
-           'EXPATINCPATH={0}'.format(expat.include),
+           "EXPATLIBPATH={0}".format(expat.lib),
+           "EXPATINCPATH={0}".format(expat.include),
        ]
 
 

lib/spack/docs/build_systems/pythonpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/qmakepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -83,7 +83,7 @@ base class already contains:
 
 .. code-block:: python
 
-   depends_on('qt', type='build')
+   depends_on("qt", type="build")
 
 
 If you want to specify a particular version requirement, or need to
@@ -91,7 +91,7 @@ link to the ``qt`` libraries, you can override this in your package:
 
 .. code-block:: python
 
-   depends_on('qt@5.6.0:')
+   depends_on("qt@5.6.0:")
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 Passing arguments to qmake
@@ -103,7 +103,7 @@ override the ``qmake_args`` method like so:
 .. code-block:: python
 
    def qmake_args(self):
-       return ['-recursive']
+       return ["-recursive"]
 
 
 This method can be used to pass flags as well as variables.
@@ -118,7 +118,7 @@ sub-directory by adding the following to the package:
 
 .. code-block:: python
 
-   build_directory = 'src'
+   build_directory = "src"
 
 
 ^^^^^^^^^^^^^^^^^^^^^^

lib/spack/docs/build_systems/racketpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/rocmpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/rpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -163,28 +163,28 @@ attributes that can be used to set ``homepage``, ``url``, ``list_url``, and
 
 .. code-block:: python
 
-   cran = 'caret'
+   cran = "caret"
 
 is equivalent to:
 
 .. code-block:: python
 
-   homepage = 'https://cloud.r-project.org/package=caret'
-   url      = 'https://cloud.r-project.org/src/contrib/caret_6.0-86.tar.gz'
-   list_url = 'https://cloud.r-project.org/src/contrib/Archive/caret'
+   homepage = "https://cloud.r-project.org/package=caret"
+   url      = "https://cloud.r-project.org/src/contrib/caret_6.0-86.tar.gz"
+   list_url = "https://cloud.r-project.org/src/contrib/Archive/caret"
 
 Likewise, the following ``bioc`` attribute:
 
 .. code-block:: python
 
-   bioc = 'BiocVersion'
+   bioc = "BiocVersion"
 
 is equivalent to:
 
 .. code-block:: python
 
-   homepage = 'https://bioconductor.org/packages/BiocVersion/'
-   git      = 'https://git.bioconductor.org/packages/BiocVersion'
+   homepage = "https://bioconductor.org/packages/BiocVersion/"
+   git      = "https://git.bioconductor.org/packages/BiocVersion"
 
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -200,7 +200,7 @@ base class contains:
 
 .. code-block:: python
 
-   extends('r')
+   extends("r")
 
 
 Take a close look at the homepage for ``caret``. If you look at the
@@ -209,7 +209,7 @@ You should add this to your package like so:
 
 .. code-block:: python
 
-   depends_on('r@3.2.0:', type=('build', 'run'))
+   depends_on("r@3.2.0:", type=("build", "run"))
 
 
 ^^^^^^^^^^^^^^
@@ -227,7 +227,7 @@ and list all of their dependencies in the following sections:
 * LinkingTo
 
 As far as Spack is concerned, all 3 of these dependency types
-correspond to ``type=('build', 'run')``, so you don't have to worry
+correspond to ``type=("build", "run")``, so you don't have to worry
 about the details. If you are curious what they mean,
 https://github.com/spack/spack/issues/2951 has a pretty good summary:
 
@@ -330,7 +330,7 @@ the dependency:
 
 .. code-block:: python
 
-   depends_on('r-lattice@0.20:', type=('build', 'run'))
+   depends_on("r-lattice@0.20:", type=("build", "run"))
 
 
 ^^^^^^^^^^^^^^^^^^
@@ -361,20 +361,20 @@ like so:
 .. code-block:: python
 
    def configure_args(self):
-       mpi_name = self.spec['mpi'].name
+       mpi_name = self.spec["mpi"].name
 
        # The type of MPI. Supported values are:
        # OPENMPI, LAM, MPICH, MPICH2, or CRAY
-       if mpi_name == 'openmpi':
-           Rmpi_type = 'OPENMPI'
-       elif mpi_name == 'mpich':
-           Rmpi_type = 'MPICH2'
+       if mpi_name == "openmpi":
+           Rmpi_type = "OPENMPI"
+       elif mpi_name == "mpich":
+           Rmpi_type = "MPICH2"
        else:
-           raise InstallError('Unsupported MPI type')
+           raise InstallError("Unsupported MPI type")
 
        return [
-           '--with-Rmpi-type={0}'.format(Rmpi_type),
-           '--with-mpi={0}'.format(spec['mpi'].prefix),
+           "--with-Rmpi-type={0}".format(Rmpi_type),
+           "--with-mpi={0}".format(spec["mpi"].prefix),
        ]
 
 

lib/spack/docs/build_systems/rubypackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -84,8 +84,8 @@ The ``*.gemspec`` file may contain something like:
 
 .. code-block:: ruby
 
-   summary = 'An implementation of the AsciiDoc text processor and publishing toolchain'
-   description = 'A fast, open source text processor and publishing toolchain for converting AsciiDoc content to HTML 5, DocBook 5, and other formats.'
+   summary = "An implementation of the AsciiDoc text processor and publishing toolchain"
+   description = "A fast, open source text processor and publishing toolchain for converting AsciiDoc content to HTML 5, DocBook 5, and other formats."
 
 
 Either of these can be used for the description of the Spack package.
@@ -98,7 +98,7 @@ The ``*.gemspec`` file may contain something like:
 
 .. code-block:: ruby
 
-   homepage = 'https://asciidoctor.org'
+   homepage = "https://asciidoctor.org"
 
 
 This should be used as the official homepage of the Spack package.
@@ -112,21 +112,21 @@ the base class contains:
 
 .. code-block:: python
 
-   extends('ruby')
+   extends("ruby")
 
 
 The ``*.gemspec`` file may contain something like:
 
 .. code-block:: ruby
 
-   required_ruby_version = '>= 2.3.0'
+   required_ruby_version = ">= 2.3.0"
 
 
 This can be added to the Spack package using:
 
 .. code-block:: python
 
-   depends_on('ruby@2.3.0:', type=('build', 'run'))
+   depends_on("ruby@2.3.0:", type=("build", "run"))
 
 
 ^^^^^^^^^^^^^^^^^

lib/spack/docs/build_systems/sconspackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/sippackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -124,7 +124,7 @@ are wrong, you can provide the names yourself by overriding
 
 .. code-block:: python
 
-   import_modules = ['PyQt5']
+   import_modules = ["PyQt5"]
 
 
 These tests often catch missing dependencies and non-RPATHed

lib/spack/docs/build_systems/sourceforgepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/wafpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -63,8 +63,8 @@ run package-specific unit tests.
 .. code-block:: python
 
    def installtest(self):
-       with working_dir('test'):
-           pytest = which('py.test')
+       with working_dir("test"):
+           pytest = which("py.test")
            pytest()
 
 
@@ -93,7 +93,7 @@ the following dependency automatically:
 
 .. code-block:: python
 
-   depends_on('python@2.5:', type='build')
+   depends_on("python@2.5:", type="build")
 
 
 Waf only supports Python 2.5 and up.
@@ -113,7 +113,7 @@ phase, you can use:
        args = []
 
        if self.run_tests:
-           args.append('--test')
+           args.append("--test")
 
        return args
 

lib/spack/docs/chain.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/conf.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -215,6 +215,7 @@ def setup(sphinx):
     ("py:class", "spack.spec.InstallStatus"),
     ("py:class", "spack.spec.SpecfileReaderBase"),
     ("py:class", "spack.install_test.Pb"),
+    ("py:class", "spack.filesystem_view.SimpleFilesystemView"),
 ]
 
 # The reST default role (used for this markup: `text`) to use for all documents.

lib/spack/docs/config_yaml.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/configuration.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/containers.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -9,24 +9,96 @@
 Container Images
 ================
 
-Spack :ref:`environments` are a great tool to create container images, but
-preparing one that is suitable for production requires some more boilerplate
-than just:
+Spack :ref:`environments` can easily be turned into container images. This page
+outlines two ways in which this can be done:
+
+1. By installing the environment on the host system, and copying the installations
+   into the container image. This approach does not require any tools like Docker
+   or Singularity to be installed.
+2. By generating a Docker or Singularity recipe that can be used to build the
+   container image. In this approach, Spack builds the software inside the
+   container runtime, not on the host system.
+
+The first approach is easiest if you already have an installed environment,
+the second approach gives more control over the container image.
+
+---------------------------
+From existing installations
+---------------------------
+
+If you already have a Spack environment installed on your system, you can
+share the binaries as an OCI compatible container image. To get started you
+just have to configure and OCI registry and run ``spack buildcache push``.
+
+.. code-block:: console
+  
+   # Create and install an environment in the current directory
+   spack env create -d .
+   spack -e . add pkg-a pkg-b
+   spack -e . install
+
+   # Configure the registry
+   spack -e . mirror add --oci-username ... --oci-password ... container-registry oci://example.com/name/image
+
+   # Push the image
+   spack -e . buildcache push --update-index --base-image ubuntu:22.04 --tag my_env container-registry
+
+The resulting container image can then be run as follows:
+
+.. code-block:: console
+
+   $ docker run -it example.com/name/image:my_env
+
+The image generated by Spack consists of the specified base image with each package from the
+environment as a separate layer on top. The image is minimal by construction, it only contains the
+environment roots and its runtime dependencies.
+
+.. note::
+
+  When using registries like GHCR and Docker Hub, the ``--oci-password`` flag is not
+  the password for your account, but a personal access token you need to generate separately.
+
+The specified ``--base-image`` should have a libc that is compatible with the host system.
+For example if your host system is Ubuntu 20.04, you can use ``ubuntu:20.04``, ``ubuntu:22.04``
+or newer: the libc in the container image must be at least the version of the host system,
+assuming ABI compatibility. It is also perfectly fine to use a completely different
+Linux distribution as long as the libc is compatible.
+
+For convenience, Spack also turns the OCI registry into a :ref:`build cache <binary_caches_oci>`,
+so that future ``spack install`` of the environment will simply pull the binaries from the
+registry instead of doing source builds. The flag ``--update-index`` is needed to make Spack
+take the build cache into account when concretizing.
+
+.. note::
+
+  When generating container images in CI, the approach above is recommended when CI jobs
+  already run in a sandboxed environment. You can simply use ``spack`` directly
+  in the CI job and push the resulting image to a registry. Subsequent CI jobs should
+  run faster because Spack can install from the same registry instead of rebuilding from
+  sources.
+
+---------------------------------------------
+Generating recipes for Docker and Singularity
+---------------------------------------------
+
+Apart from copying existing installations into container images, Spack can also
+generate recipes for container images. This is useful if you want to run Spack
+itself in a sandboxed environment instead of on the host system.
+
+Since recipes need a little bit more boilerplate than
 
 .. code-block:: docker
 
    COPY spack.yaml /environment
    RUN spack -e /environment install
 
-Additional actions may be needed to minimize the size of the
-container, or to update the system software that is installed in the base
-image, or to set up a proper entrypoint to run the image. These tasks are
-usually both necessary and repetitive, so Spack comes with a command
-to generate recipes for container images starting from a ``spack.yaml``.
+Spack provides a command to generate customizable recipes for container images. Customizations
+include minimizing the size of the image, installing packages in the base image using the system
+package manager, and setting up a proper entrypoint to run the image.
 
---------------------
+~~~~~~~~~~~~~~~~~~~~
 A Quick Introduction
---------------------
+~~~~~~~~~~~~~~~~~~~~
 
 Consider having a Spack environment like the following:
 
@@ -37,8 +109,8 @@ Consider having a Spack environment like the following:
      - gromacs+mpi
      - mpich
 
-Producing a ``Dockerfile`` from it is as simple as moving to the directory
-where the ``spack.yaml`` file is stored and giving the following command:
+Producing a ``Dockerfile`` from it is as simple as changing directories to
+where the ``spack.yaml`` file is stored and running the following command:
 
 .. code-block:: console
 
@@ -104,9 +176,9 @@ configuration are discussed in details in the sections below.
 
 .. _container_spack_images:
 
---------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 Spack Images on Docker Hub
---------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Docker images with Spack preinstalled and ready to be used are
 built when a release is tagged, or nightly on ``develop``. The images
@@ -176,9 +248,9 @@ by Spack use them as default base images for their ``build`` stage,
 even though handles to use custom base images provided by users are
 available to accommodate complex use cases.
 
----------------------------------
-Creating Images From Environments
----------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Configuring the Container Recipe
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Any Spack Environment can be used for the automatic generation of container
 recipes. Sensible defaults are provided for things like the base image or the
@@ -219,18 +291,18 @@ under the ``container`` attribute of environments:
 
 A detailed description of the options available can be found in the :ref:`container_config_options` section.
 
--------------------
+~~~~~~~~~~~~~~~~~~~
 Setting Base Images
--------------------
+~~~~~~~~~~~~~~~~~~~
 
 The ``images`` subsection is used to select both the image where
 Spack builds the software and the image where the built software
 is installed. This attribute can be set in different ways and
 which one to use depends on the use case at hand.
 
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+""""""""""""""""""""""""""""""""""""""""
 Use Official Spack Images From Dockerhub
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+""""""""""""""""""""""""""""""""""""""""
 
 To generate a recipe that uses an official Docker image from the
 Spack organization to build the software and the corresponding official OS image
@@ -435,9 +507,9 @@ responsibility to ensure that:
 Therefore we don't recommend its use in cases that can be otherwise
 covered by the simplified mode shown first.
 
-----------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Singularity Definition Files
-----------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In addition to producing recipes in ``Dockerfile`` format Spack can produce
 Singularity Definition Files by just changing the value of the ``format``
@@ -458,9 +530,9 @@ attribute:
 The minimum version of Singularity required to build a SIF (Singularity Image Format)
 image from the recipes generated by Spack is ``3.5.3``.
 
-------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Extending the Jinja2 Templates
-------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The Dockerfile and the Singularity definition file that Spack can generate are based on
 a few Jinja2 templates that are rendered according to the environment being containerized.
@@ -581,9 +653,9 @@ The recipe that gets generated contains the two extra instruction that we added
 
 .. _container_config_options:
 
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 Configuration Reference
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 
 The tables below describe all the configuration options that are currently supported
 to customize the generation of container recipes:
@@ -680,13 +752,13 @@ to customize the generation of container recipes:
      - Description string
      - No
 
---------------
+~~~~~~~~~~~~~~
 Best Practices
---------------
+~~~~~~~~~~~~~~
 
-^^^
+"""
 MPI
-^^^
+"""
 Due to the dependency on Fortran for OpenMPI, which is the spack default
 implementation, consider adding ``gfortran`` to the ``apt-get install`` list.
 
@@ -697,9 +769,9 @@ For execution on HPC clusters, it can be helpful to import the docker
 image into Singularity in order to start a program with an *external*
 MPI. Otherwise, also add ``openssh-server`` to the ``apt-get install`` list.
 
-^^^^
+""""
 CUDA
-^^^^
+""""
 Starting from CUDA 9.0, Nvidia provides minimal CUDA images based on
 Ubuntu. Please see `their instructions <https://hub.docker.com/r/nvidia/cuda/>`_.
 Avoid double-installing CUDA by adding, e.g.
@@ -718,9 +790,9 @@ to your ``spack.yaml``.
 Users will either need ``nvidia-docker`` or e.g. Singularity to *execute*
 device kernels.
 
-^^^^^^^^^^^^^^^^^^^^^^^^^
+"""""""""""""""""""""""""
 Docker on Windows and OSX
-^^^^^^^^^^^^^^^^^^^^^^^^^
+"""""""""""""""""""""""""
 
 On Mac OS and Windows, docker runs on a hypervisor that is not allocated much
 memory by default, and some spack packages may fail to build due to lack of

lib/spack/docs/contribution_guide.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/developer_guide.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/environments.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -142,6 +142,21 @@ user's prompt to begin with the environment name in brackets.
    $ spack env activate -p myenv
    [myenv] $ ...
 
+The ``activate`` command can also be used to create a new environment, if it is
+not already defined, by adding the ``--create`` flag. Managed and anonymous
+environments, anonymous environments are explained in the next section,
+can both be created using the same flags that `spack env create` accepts.
+If an environment already exists then spack will simply activate it and ignore the
+create specific flags.
+
+.. code-block:: console
+   
+   $ spack env activate --create -p myenv
+   # ...
+   # [creates if myenv does not exist yet]
+   # ...
+   [myenv] $ ...
+
 To deactivate an environment, use the command:
 
 .. code-block:: console

lib/spack/docs/extensions.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -9,46 +9,42 @@
 Custom Extensions
 =================
 
-*Spack extensions* permit you to extend Spack capabilities by deploying your
+*Spack extensions* allow you to extend Spack capabilities by deploying your
 own custom commands or logic in an arbitrary location on your filesystem.
 This might be extremely useful e.g. to develop and maintain a command whose purpose is
 too specific to be considered for reintegration into the mainline or to
 evolve a command through its early stages before starting a discussion to merge
 it upstream.
+
 From Spack's point of view an extension is any path in your filesystem which
-respects a prescribed naming and layout for files:
+respects the following naming and layout for files:
 
 .. code-block:: console
 
   spack-scripting/ # The top level directory must match the format 'spack-{extension_name}'
   ├── pytest.ini # Optional file if the extension ships its own tests
   ├── scripting # Folder that may contain modules that are needed for the extension commands
-  │   └── cmd # Folder containing extension commands
-  │       └── filter.py # A new command that will be available
-  ├── tests # Tests for this extension
+  │   ├── cmd # Folder containing extension commands
+  │   │   └── filter.py # A new command that will be available
+  │   └── functions.py # Module with internal details
+  └── tests # Tests for this extension
   │   ├── conftest.py
   │   └── test_filter.py
   └── templates # Templates that may be needed by the extension
 
-In the example above the extension named *scripting* adds an additional command (``filter``)
-and unit tests to verify its behavior. The code for this example can be
-obtained by cloning the corresponding git repository:
+In the example above, the extension is named *scripting*. It adds an additional command
+(``spack filter``) and unit tests to verify its behavior.
 
-.. TODO: write an ad-hoc "hello world" extension and make it part of the spack organization
+The extension can import any core Spack module in its implementation. When loaded by
+the ``spack`` command, the extension itself is imported as a Python package in the
+``spack.extensions`` namespace. In the example above, since the extension is named
+"scripting", the corresponding Python module is ``spack.extensions.scripting``.
+
+The code for this example extension can be obtained by cloning the corresponding git repository:
 
 .. code-block:: console
 
-   $ cd ~/
-   $ mkdir tmp && cd tmp
-   $ git clone https://github.com/alalazo/spack-scripting.git
-   Cloning into 'spack-scripting'...
-   remote: Counting objects: 11, done.
-   remote: Compressing objects: 100% (7/7), done.
-   remote: Total 11 (delta 0), reused 11 (delta 0), pack-reused 0
-   Receiving objects: 100% (11/11), done.
-
-As you can see by inspecting the sources, Python modules that are part of the extension
-can import any core Spack module.
+   $ git -C /tmp clone https://github.com/spack/spack-scripting.git
 
 ---------------------------------
 Configure Spack to Use Extensions
@@ -61,7 +57,7 @@ paths to ``config.yaml``. In the case of our example this means ensuring that:
 
    config:
      extensions:
-     - ~/tmp/spack-scripting
+     - /tmp/spack-scripting
 
 is part of your configuration file. Once this is setup any command that the extension provides
 will be available from the command line:
@@ -86,37 +82,32 @@ will be available from the command line:
      --implicit       select specs that are not installed or were installed implicitly
      --output OUTPUT  where to dump the result
 
-The corresponding unit tests can be run giving the appropriate options
-to ``spack unit-test``:
+The corresponding unit tests can be run giving the appropriate options to ``spack unit-test``:
 
 .. code-block:: console
 
    $ spack unit-test --extension=scripting
-
-   ============================================================== test session starts ===============================================================
-   platform linux2 -- Python 2.7.15rc1, pytest-3.2.5, py-1.4.34, pluggy-0.4.0
-   rootdir: /home/mculpo/tmp/spack-scripting, inifile: pytest.ini
+   ========================================== test session starts ===========================================
+   platform linux -- Python 3.11.5, pytest-7.4.3, pluggy-1.3.0
+   rootdir: /home/culpo/github/spack-scripting
+   configfile: pytest.ini
+   testpaths: tests
+   plugins: xdist-3.5.0
    collected 5 items
 
-   tests/test_filter.py ...XX
-   ============================================================ short test summary info =============================================================
-   XPASS tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
-   XPASS tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
+   tests/test_filter.py .....                                                                         [100%]
 
-   =========================================================== slowest 20 test durations ============================================================
-   3.74s setup    tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
-   0.17s call     tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
-   0.16s call     tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
-   0.15s call     tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
-   0.13s call     tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
-   0.08s call     tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
-   0.04s teardown tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
-   0.00s setup    tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
-   0.00s setup    tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
-   0.00s setup    tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
-   0.00s setup    tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
-   0.00s teardown tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
-   0.00s teardown tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
-   0.00s teardown tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
-   0.00s teardown tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
-   ====================================================== 3 passed, 2 xpassed in 4.51 seconds =======================================================
+   ========================================== slowest 30 durations ==========================================
+   2.31s setup    tests/test_filter.py::test_filtering_specs[kwargs0-specs0-expected0]
+   0.57s call     tests/test_filter.py::test_filtering_specs[kwargs2-specs2-expected2]
+   0.56s call     tests/test_filter.py::test_filtering_specs[kwargs4-specs4-expected4]
+   0.54s call     tests/test_filter.py::test_filtering_specs[kwargs3-specs3-expected3]
+   0.54s call     tests/test_filter.py::test_filtering_specs[kwargs1-specs1-expected1]
+   0.48s call     tests/test_filter.py::test_filtering_specs[kwargs0-specs0-expected0]
+   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs4-specs4-expected4]
+   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs2-specs2-expected2]
+   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs1-specs1-expected1]
+   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs3-specs3-expected3]
+
+   (5 durations < 0.005s hidden.  Use -vv to show these durations.)
+   =========================================== 5 passed in 5.06s ============================================

lib/spack/docs/features.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/frequently_asked_questions.rst

@@ -0,0 +1,77 @@
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.
+
+   SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+==========================
+Frequently Asked Questions
+==========================
+
+This page contains answers to frequently asked questions about Spack.
+If you have questions that are not answered here, feel free to ask on
+`Slack <https://slack.spack.io>`_ or `GitHub Discussions
+<https://github.com/spack/spack/discussions>`_. If you've learned the
+answer to a question that you think should be here, please consider
+contributing to this page.
+
+.. _faq-concretizer-precedence:
+
+-----------------------------------------------------
+Why does Spack pick particular versions and variants?
+-----------------------------------------------------
+
+This question comes up in a variety of forms:
+
+ 1. Why does Spack seem to ignore my package preferences from ``packages.yaml`` config?
+ 2. Why does Spack toggle a variant instead of using the default from the ``package.py`` file?
+
+The short answer is that Spack always picks an optimal configuration
+based on a complex set of criteria\ [#f1]_. These criteria are more nuanced
+than always choosing the latest versions or default variants.
+
+.. note::
+
+    As a rule of thumb: requirements + constraints > reuse > preferences > defaults.
+
+The following set of criteria (from lowest to highest precedence) explain
+common cases where concretization output may seem surprising at first.
+
+1. :ref:`Package preferences <package-preferences>` configured in ``packages.yaml``
+   override variant defaults from ``package.py`` files, and influence the optimal
+   ordering of versions. Preferences are specified as follows:
+
+   .. code-block:: yaml
+
+      packages:
+        foo:
+          version: [1.0, 1.1]
+          variants: ~mpi
+
+2. :ref:`Reuse concretization <concretizer-options>` configured in ``concretizer.yaml``
+   overrides preferences, since it's typically faster to reuse an existing spec than to
+   build a preferred one from sources. When build caches are enabled, specs may be reused
+   from a remote location too. Reuse concretization is configured as follows:
+
+   .. code-block:: yaml
+
+      concretizer:
+        reuse: dependencies  # other options are 'true' and 'false'
+
+3. :ref:`Package requirements <package-requirements>` configured in ``packages.yaml``,
+   and constraints from the command line as well as ``package.py`` files override all
+   of the above. Requirements are specified as follows:
+
+   .. code-block:: yaml
+
+      packages:
+        foo:
+          require:
+          - "@1.2: +mpi"
+
+Requirements and constraints restrict the set of possible solutions, while reuse
+behavior and preferences influence what an optimal solution looks like.
+
+
+.. rubric:: Footnotes
+
+.. [#f1] The exact list of criteria can be retrieved with the ``spack solve`` command

lib/spack/docs/getting_started.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/gpu_configuration.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -111,3 +111,28 @@ CUDA is split into fewer components and is simpler to specify:
          prefix: /opt/cuda/cuda-11.0.2/
 
 where ``/opt/cuda/cuda-11.0.2/lib/`` contains ``libcudart.so``.
+
+
+
+-----------------------------------
+Using an External OpenGL API
+-----------------------------------
+Depending on whether we have a graphics card or not, we may choose to use OSMesa or GLX to implement the OpenGL API.
+
+If a graphics card is unavailable, OSMesa is recommended and can typically be built with Spack.
+However, if we prefer to utilize the system GLX tailored to our graphics card, we need to declare it as an external. Here's how to do it:
+
+
+.. code-block:: yaml
+
+  packages:
+    libglx:
+      require: [opengl]
+    opengl:
+      buildable: false
+      externals:
+      - prefix: /usr/
+        spec: opengl@4.6
+
+Note that prefix has to be the root of both the libraries and the headers, using is /usr not the path the the lib.
+To know which spec for opengl is available use ``cd /usr/include/GL && grep -Ri gl_version``.

lib/spack/docs/index.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -55,6 +55,7 @@ or refer to the full manual below.
    getting_started
    basic_usage
    replace_conda_homebrew
+   frequently_asked_questions
 
 .. toctree::
    :maxdepth: 2

lib/spack/docs/mirrors.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/module_file_support.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/packages_yaml.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -97,6 +97,35 @@ Each package version and compiler listed in an external should
 have entries in Spack's packages and compiler configuration, even
 though the package and compiler may not ever be built.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Extra attributes for external packages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Sometimes external packages require additional attributes to be used
+effectively. This information can be defined on a per-package basis
+and stored in the ``extra_attributes`` section of the external package
+configuration. In addition to per-package information, this section
+can be used to define environment modifications to be performed
+whenever the package is used. For example, if an external package is
+built without ``rpath`` support, it may require ``LD_LIBRARY_PATH``
+settings to find its dependencies. This could be configured as
+follows:
+
+.. code-block:: yaml
+
+   packages:
+     mpich:
+       externals:
+       - spec: mpich@3.3 %clang@12.0.0 +hwloc
+         prefix: /path/to/mpich
+         extra_attributes:
+           environment:
+             prepend_path:
+               LD_LIBRARY_PATH: /path/to/hwloc/lib64
+
+See :ref:`configuration_environment_variables` for more information on
+how to configure environment modifications in Spack config files.
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Prevent packages from being built from sources
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -256,6 +285,11 @@ on the command line, because it can specify constraints on packages
 is not possible to specify constraints on dependencies while also keeping
 those dependencies optional.
 
+.. seealso::
+
+   FAQ: :ref:`Why does Spack pick particular versions and variants? <faq-concretizer-precedence>`
+
+
 ^^^^^^^^^^^^^^^^^^^
 Requirements syntax
 ^^^^^^^^^^^^^^^^^^^
@@ -378,7 +412,33 @@ like this:
 
 which means every spec will be required to use ``clang`` as a compiler.
 
-Note that in this case ``all`` represents a *default set of requirements* -
+Requirements on variants for all packages are possible too, but note that they
+are only enforced for those packages that define these variants, otherwise they
+are disregarded. For example:
+
+.. code-block:: yaml
+
+   packages:
+     all:
+       require:
+       - "+shared"
+       - "+cuda"
+
+will just enforce ``+shared`` on ``zlib``, which has a boolean ``shared`` variant but
+no ``cuda`` variant.
+
+Constraints in a single spec literal are always considered as a whole, so in a case like:
+
+.. code-block:: yaml
+
+   packages:
+     all:
+       require: "+shared +cuda"
+
+the default requirement will be enforced only if a package has both a ``cuda`` and
+a ``shared`` variant, and will never be partially enforced.
+
+Finally, ``all`` represents a *default set of requirements* -
 if there are specific package requirements, then the default requirements
 under ``all`` are disregarded. For example, with a configuration like this:
 
@@ -386,12 +446,18 @@ under ``all`` are disregarded. For example, with a configuration like this:
 
    packages:
      all:
-       require: '%clang'
+       require:
+       - 'build_type=Debug'
+       - '%clang'
      cmake:
-       require: '%gcc'
+       require:
+       - 'build_type=Debug'
+       - '%gcc'
 
 Spack requires ``cmake`` to use ``gcc`` and all other nodes (including ``cmake``
-dependencies) to use ``clang``.
+dependencies) to use ``clang``. If enforcing ``build_type=Debug`` is needed also
+on ``cmake``, it must be repeated in the specific ``cmake`` requirements.
+
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Setting requirements on virtual specs
@@ -421,6 +487,56 @@ present. For instance with a configuration like:
 
 you will use ``mvapich2~cuda %gcc`` as an ``mpi`` provider.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Conflicts and strong preferences
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If the semantic of requirements is too strong, you can also express "strong preferences" and "conflicts"
+from configuration files:
+
+.. code-block:: yaml
+
+   packages:
+     all:
+       prefer:
+       - '%clang'
+       conflict:
+       - '+shared'
+
+The ``prefer`` and ``conflict`` sections can be used whenever a ``require`` section is allowed.
+The argument is always a list of constraints, and each constraint can be either a simple string,
+or a more complex object:
+
+.. code-block:: yaml
+
+   packages:
+     all:
+       conflict:
+       - spec: '%clang'
+         when: 'target=x86_64_v3'
+         message: 'reason why clang cannot be used'
+
+The ``spec`` attribute is mandatory, while both ``when`` and ``message`` are optional.
+
+.. note::
+
+   Requirements allow for expressing both "strong preferences" and "conflicts".
+   The syntax for doing so, though, may not be immediately clear. For
+   instance, if we want to prevent any package from using ``%clang``, we can set:
+
+   .. code-block:: yaml
+
+      packages:
+        all:
+          require:
+          - one_of: ['%clang', '@:']
+
+   Since only one of the requirements must hold, and ``@:`` is always true, the rule above is
+   equivalent to a conflict. For "strong preferences" we need to substitute the ``one_of`` policy
+   with ``any_of``.
+
+
+
 .. _package-preferences:
 
 -------------------
@@ -435,6 +551,11 @@ them if it must, due to other constraints, and also prefers reusing
 installed packages over building new ones that are a better match for
 preferences.
 
+.. seealso::
+
+   FAQ: :ref:`Why does Spack pick particular versions and variants? <faq-concretizer-precedence>`
+
+
 Most package preferences (``compilers``, ``target`` and ``providers``)
 can only be set globally under the ``all`` section of ``packages.yaml``:
 

lib/spack/docs/packaging_guide.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -237,7 +237,7 @@ for details):
 .. code-block:: python
    :linenos:
 
-   # Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+   # Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    # Spack Project Developers. See the top-level COPYRIGHT file for details.
    #
    # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -4379,10 +4379,16 @@ implementation was selected for this build:
    elif "mvapich" in spec:
        configure_args.append("--with-mvapich")
 
-It's also a bit more concise than satisfies.  The difference between
-the two functions is that ``satisfies()`` tests whether spec
-constraints overlap at all, while ``in`` tests whether a spec or any
-of its dependencies satisfy the provided spec.
+It's also a bit more concise than satisfies.
+
+.. note::
+
+   The ``satisfies()`` method tests whether this spec has, at least, all the constraints of the argument spec,
+   while ``in`` tests whether a spec or any of its dependencies satisfy the provided spec.
+
+   If the provided spec is anonymous (e.g., ":1.2:", "+shared") or has the
+   same name as the spec being checked, then ``in`` works the same as
+   ``satisfies()``; however, use of ``satisfies()`` is more intuitive.
 
 ^^^^^^^^^^^^^^^^^^^^^^^
 Architecture specifiers
@@ -5284,7 +5290,7 @@ installed example.
            example = which(self.prefix.bin.example)
            example()
 
-Output showing the identification of each test part after runnig the tests
+Output showing the identification of each test part after running the tests
 is illustrated below.
 
 .. code-block:: console
@@ -5781,7 +5787,7 @@ with those implemented in the package itself.
    * - `Cxx
        <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/cxx>`_
      - Compiles and runs several ``hello`` programs
-   * - `Fortan
+   * - `Fortran
        <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/fortran>`_
      - Compiles and runs ``hello`` programs (``F`` and ``f90``)
    * - `Mpi
@@ -6973,3 +6979,18 @@ you probably care most about are:
 You may also care about `license exceptions
 <https://spdx.org/licenses/exceptions-index.html>`_ that use the ``WITH`` operator,
 e.g. ``Apache-2.0 WITH LLVM-exception``.
+
+Many of the licenses that are currently in the spack repositories have been
+automatically determined. While this is great for bulk adding license
+information and is most likely correct, there are sometimes edge cases that
+require manual intervention. To determine which licenses are validated and
+which are not, there is the `checked_by` parameter in the license directive:
+
+.. code-block:: python
+
+   license("<license>", when="<when>", checked_by="<github username>")
+
+When you have validated a github license, either when doing so explicitly or
+as part of packaging a new package, please set the `checked_by` parameter
+to your Github username to signal that the license has been manually
+verified.

lib/spack/docs/pipelines.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/replace_conda_homebrew.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/repositories.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/requirements.txt

@@ -1,13 +1,13 @@
 sphinx==7.2.6
 sphinxcontrib-programoutput==0.17
 sphinx_design==0.5.0
-sphinx-rtd-theme==1.3.0
+sphinx-rtd-theme==2.0.0
 python-levenshtein==0.23.0
-docutils==0.18.1
-pygments==2.16.1
+docutils==0.20.1
+pygments==2.17.2
 urllib3==2.1.0
-pytest==7.4.3
-isort==5.12.0
-black==23.11.0
-flake8==6.1.0
-mypy==1.7.0
+pytest==7.4.4
+isort==5.13.2
+black==23.12.1
+flake8==7.0.0
+mypy==1.8.0

lib/spack/docs/signing.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -142,7 +142,7 @@ Reputational Key
 ----------------
 
 The Reputational Key is the public facing key used to sign complete groups of
-development and release packages. Only one key pair exsits in this class of
+development and release packages. Only one key pair exists in this class of
 keys. In contrast to the Intermediate CI Key the Reputational Key *should* be
 used to verify package integrity. At the end of develop and release pipeline a
 final pipeline job pulls down all signed package metadata built by the pipeline,
@@ -272,7 +272,7 @@ Internal Implementation
 
 The technical implementation of the pipeline signing process includes components
 defined in Amazon Web Services, the Kubernetes cluster, at affilicated
-institutions, and the GitLab/GitLab Runner deployment. We present the techincal
+institutions, and the GitLab/GitLab Runner deployment. We present the technical
 implementation in two interdependent sections. The first addresses how secrets
 are managed through the lifecycle of a develop or release pipeline. The second
 section describes how Gitlab Runner and pipelines are configured and managed to
@@ -295,7 +295,7 @@ infrastructure.
 -----------------------
 
 Multiple intermediate CI signing keys exist, one Intermediate CI Key for jobs
-run in AWS, and one key for each affiliated institution (e.g. Univerity of
+run in AWS, and one key for each affiliated institution (e.g. University of
 Oregon). Here we describe how the Intermediate CI Key is managed in AWS:
 
 The Intermediate CI Key (including the Signing Intermediate CI Private Key is
@@ -305,7 +305,7 @@ contains an ASCII-armored export of just the *public* components of the
 Reputational Key. This secret also contains the *public* components of each of
 the affiliated institutions' Intermediate CI Key. These are potentially needed
 to verify dependent packages which may have been found in the public mirror or
-built by a protected job running on an affiliated institution's infrastrcuture
+built by a protected job running on an affiliated institution's infrastructure
 in an earlier stage of the pipeline.
 
 Procedurally the ``spack-intermediate-ci-signing-key`` secret is used in

lib/spack/docs/spack.yaml

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/env/cc

@@ -1,7 +1,7 @@
 #!/bin/sh -f
 # shellcheck disable=SC2034  # evals in this script fool shellcheck
 #
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/external/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/path.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/string.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/url.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/argparsewriter.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/filesystem.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/lang.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -1047,9 +1047,9 @@ def __bool__(self):
         """Whether any exceptions were handled."""
         return bool(self.exceptions)
 
-    def forward(self, context: str) -> "GroupedExceptionForwarder":
+    def forward(self, context: str, base: type = BaseException) -> "GroupedExceptionForwarder":
         """Return a contextmanager which extracts tracebacks and prefixes a message."""
-        return GroupedExceptionForwarder(context, self)
+        return GroupedExceptionForwarder(context, self, base)
 
     def _receive_forwarded(self, context: str, exc: Exception, tb: List[str]):
         self.exceptions.append((context, exc, tb))
@@ -1072,15 +1072,18 @@ class GroupedExceptionForwarder:
     """A contextmanager to capture exceptions and forward them to a
     GroupedExceptionHandler."""
 
-    def __init__(self, context: str, handler: GroupedExceptionHandler):
+    def __init__(self, context: str, handler: GroupedExceptionHandler, base: type):
         self._context = context
         self._handler = handler
+        self._base = base
 
     def __enter__(self):
         return None
 
     def __exit__(self, exc_type, exc_value, tb):
         if exc_value is not None:
+            if not issubclass(exc_type, self._base):
+                return False
             self._handler._receive_forwarded(self._context, exc_value, traceback.format_tb(tb))
 
         # Suppress any exception from being re-raised:

lib/spack/llnl/util/link_tree.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/lock.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/multiproc.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/symlink.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/tty/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/tty/colify.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/tty/color.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -204,17 +204,23 @@ def color_when(value):
 
 
 class match_to_ansi:
-    def __init__(self, color=True, enclose=False):
+    def __init__(self, color=True, enclose=False, zsh=False):
         self.color = _color_when_value(color)
         self.enclose = enclose
+        self.zsh = zsh
 
     def escape(self, s):
         """Returns a TTY escape sequence for a color"""
         if self.color:
-            if self.enclose:
-                return r"\[\033[%sm\]" % s
+            if self.zsh:
+                result = rf"\e[0;{s}m"
             else:
-                return "\033[%sm" % s
+                result = f"\033[{s}m"
+
+            if self.enclose:
+                result = rf"\[{result}\]"
+
+            return result
         else:
             return ""
 
@@ -261,9 +267,11 @@ def colorize(string, **kwargs):
             codes, for output to non-console devices.
         enclose (bool): If True, enclose ansi color sequences with
             square brackets to prevent misestimation of terminal width.
+        zsh (bool): If True, use zsh ansi codes instead of bash ones (for variables like PS1)
     """
     color = _color_when_value(kwargs.get("color", get_color_when()))
-    string = re.sub(color_re, match_to_ansi(color, kwargs.get("enclose")), string)
+    zsh = kwargs.get("zsh", False)
+    string = re.sub(color_re, match_to_ansi(color, kwargs.get("enclose")), string, zsh)
     string = string.replace("}}", "}")
     return string
 

lib/spack/llnl/util/tty/log.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/tty/pty.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/abi.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/audit.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -286,6 +286,54 @@ def make_error(attribute_name, config_data, summary):
     return errors
 
 
+@config_packages
+def _avoid_mismatched_variants(error_cls):
+    """Warns if variant preferences have mismatched types or names."""
+    errors = []
+    packages_yaml = spack.config.CONFIG.get_config("packages")
+
+    def make_error(config_data, summary):
+        s = io.StringIO()
+        s.write("Occurring in the following file:\n")
+        syaml.dump_config(config_data, stream=s, blame=True)
+        return error_cls(summary=summary, details=[s.getvalue()])
+
+    for pkg_name in packages_yaml:
+        # 'all:' must be more forgiving, since it is setting defaults for everything
+        if pkg_name == "all" or "variants" not in packages_yaml[pkg_name]:
+            continue
+
+        preferences = packages_yaml[pkg_name]["variants"]
+        if not isinstance(preferences, list):
+            preferences = [preferences]
+
+        for variants in preferences:
+            current_spec = spack.spec.Spec(variants)
+            pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
+            for variant in current_spec.variants.values():
+                # Variant does not exist at all
+                if variant.name not in pkg_cls.variants:
+                    summary = (
+                        f"Setting a preference for the '{pkg_name}' package to the "
+                        f"non-existing variant '{variant.name}'"
+                    )
+                    errors.append(make_error(preferences, summary))
+                    continue
+
+                # Variant cannot accept this value
+                s = spack.spec.Spec(pkg_name)
+                try:
+                    s.update_variant_validate(variant.name, variant.value)
+                except Exception:
+                    summary = (
+                        f"Setting the variant '{variant.name}' of the '{pkg_name}' package "
+                        f"to the invalid value '{str(variant)}'"
+                    )
+                    errors.append(make_error(preferences, summary))
+
+    return errors
+
+
 #: Sanity checks on package directives
 package_directives = AuditClass(
     group="packages",
@@ -619,8 +667,8 @@ def _unknown_variants_in_directives(pkgs, error_cls):
         pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
 
         # Check "conflicts" directive
-        for conflict, triggers in pkg_cls.conflicts.items():
-            for trigger, _ in triggers:
+        for trigger, conflicts in pkg_cls.conflicts.items():
+            for conflict, _ in conflicts:
                 vrn = spack.spec.Spec(conflict)
                 try:
                     vrn.constrain(trigger)
@@ -646,25 +694,21 @@ def _unknown_variants_in_directives(pkgs, error_cls):
                 )
 
         # Check "depends_on" directive
-        for _, triggers in pkg_cls.dependencies.items():
-            triggers = list(triggers)
-            for trigger in list(triggers):
-                vrn = spack.spec.Spec(trigger)
-                errors.extend(
-                    _analyze_variants_in_directive(
-                        pkg_cls, vrn, directive="depends_on", error_cls=error_cls
-                    )
+        for trigger in pkg_cls.dependencies:
+            vrn = spack.spec.Spec(trigger)
+            errors.extend(
+                _analyze_variants_in_directive(
+                    pkg_cls, vrn, directive="depends_on", error_cls=error_cls
                 )
+            )
 
-        # Check "patch" directive
-        for _, triggers in pkg_cls.provided.items():
-            triggers = [spack.spec.Spec(x) for x in triggers]
-            for vrn in triggers:
-                errors.extend(
-                    _analyze_variants_in_directive(
-                        pkg_cls, vrn, directive="patch", error_cls=error_cls
-                    )
+        # Check "provides" directive
+        for when_spec in pkg_cls.provided:
+            errors.extend(
+                _analyze_variants_in_directive(
+                    pkg_cls, when_spec, directive="provides", error_cls=error_cls
                 )
+            )
 
         # Check "resource" directive
         for vrn in pkg_cls.resources:
@@ -678,47 +722,82 @@ def _unknown_variants_in_directives(pkgs, error_cls):
 
 
 @package_directives
-def _unknown_variants_in_dependencies(pkgs, error_cls):
-    """Report unknown dependencies and wrong variants for dependencies"""
+def _issues_in_depends_on_directive(pkgs, error_cls):
+    """Reports issues with 'depends_on' directives.
+
+    Issues might be unknown dependencies, unknown variants or variant values, or declaration
+    of nested dependencies.
+    """
     errors = []
     for pkg_name in pkgs:
         pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
         filename = spack.repo.PATH.filename_for_package_name(pkg_name)
-        for dependency_name, dependency_data in pkg_cls.dependencies.items():
-            # No need to analyze virtual packages
-            if spack.repo.PATH.is_virtual(dependency_name):
-                continue
 
-            try:
-                dependency_pkg_cls = spack.repo.PATH.get_pkg_class(dependency_name)
-            except spack.repo.UnknownPackageError:
-                # This dependency is completely missing, so report
-                # and continue the analysis
-                summary = pkg_name + ": unknown package '{0}' in " "'depends_on' directive".format(
-                    dependency_name
-                )
-                details = [" in " + filename]
-                errors.append(error_cls(summary=summary, details=details))
-                continue
+        for when, deps_by_name in pkg_cls.dependencies.items():
+            for dep_name, dep in deps_by_name.items():
+                # Check if there are nested dependencies declared. We don't want directives like:
+                #
+                #     depends_on('foo+bar ^fee+baz')
+                #
+                # but we'd like to have two dependencies listed instead.
+                nested_dependencies = dep.spec.dependencies()
+                if nested_dependencies:
+                    summary = f"{pkg_name}: nested dependency declaration '{dep.spec}'"
+                    ndir = len(nested_dependencies) + 1
+                    details = [
+                        f"split depends_on('{dep.spec}', when='{when}') into {ndir} directives",
+                        f"in {filename}",
+                    ]
+                    errors.append(error_cls(summary=summary, details=details))
 
-            for _, dependency_edge in dependency_data.items():
-                dependency_variants = dependency_edge.spec.variants
+                def check_virtual_with_variants(spec, msg):
+                    if not spec.virtual or not spec.variants:
+                        return
+                    error = error_cls(
+                        f"{pkg_name}: {msg}",
+                        f"remove variants from '{spec}' in depends_on directive in {filename}",
+                    )
+                    errors.append(error)
+
+                check_virtual_with_variants(dep.spec, "virtual dependency cannot have variants")
+                check_virtual_with_variants(dep.spec, "virtual when= spec cannot have variants")
+
+                # No need to analyze virtual packages
+                if spack.repo.PATH.is_virtual(dep_name):
+                    continue
+
+                # check for unknown dependencies
+                try:
+                    dependency_pkg_cls = spack.repo.PATH.get_pkg_class(dep_name)
+                except spack.repo.UnknownPackageError:
+                    # This dependency is completely missing, so report
+                    # and continue the analysis
+                    summary = (
+                        f"{pkg_name}: unknown package '{dep_name}' in " "'depends_on' directive"
+                    )
+                    details = [f" in {filename}"]
+                    errors.append(error_cls(summary=summary, details=details))
+                    continue
+
+                # check variants
+                dependency_variants = dep.spec.variants
                 for name, value in dependency_variants.items():
                     try:
                         v, _ = dependency_pkg_cls.variants[name]
                         v.validate_or_raise(value, pkg_cls=dependency_pkg_cls)
                     except Exception as e:
                         summary = (
-                            pkg_name + ": wrong variant used for a "
-                            "dependency in a 'depends_on' directive"
+                            f"{pkg_name}: wrong variant used for dependency in 'depends_on()'"
                         )
-                        error_msg = str(e).strip()
+
                         if isinstance(e, KeyError):
-                            error_msg = "the variant {0} does not " "exist".format(error_msg)
-                        error_msg += " in package '" + dependency_name + "'"
+                            error_msg = (
+                                f"variant {str(e).strip()} does not exist in package {dep_name}"
+                            )
+                        error_msg += f" in package '{dep_name}'"
 
                         errors.append(
-                            error_cls(summary=summary, details=[error_msg, "in " + filename])
+                            error_cls(summary=summary, details=[error_msg, f"in {filename}"])
                         )
 
     return errors
@@ -785,14 +864,17 @@ def _version_constraints_are_satisfiable_by_some_version_in_repo(pkgs, error_cls
     for pkg_name in pkgs:
         pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
         filename = spack.repo.PATH.filename_for_package_name(pkg_name)
-        dependencies_to_check = []
-        for dependency_name, dependency_data in pkg_cls.dependencies.items():
-            # Skip virtual dependencies for the time being, check on
-            # their versions can be added later
-            if spack.repo.PATH.is_virtual(dependency_name):
-                continue
 
-            dependencies_to_check.extend([edge.spec for edge in dependency_data.values()])
+        dependencies_to_check = []
+
+        for _, deps_by_name in pkg_cls.dependencies.items():
+            for dep_name, dep in deps_by_name.items():
+                # Skip virtual dependencies for the time being, check on
+                # their versions can be added later
+                if spack.repo.PATH.is_virtual(dep_name):
+                    continue
+
+                dependencies_to_check.append(dep.spec)
 
         host_architecture = spack.spec.ArchSpec.default_arch()
         for s in dependencies_to_check:
@@ -864,39 +946,53 @@ def _named_specs_in_when_arguments(pkgs, error_cls):
     for pkg_name in pkgs:
         pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
 
+        def _refers_to_pkg(when):
+            when_spec = spack.spec.Spec(when)
+            return when_spec.name is None or when_spec.name == pkg_name
+
+        def _error_items(when_dict):
+            for when, elts in when_dict.items():
+                if not _refers_to_pkg(when):
+                    yield when, elts, [f"using '{when}', should be '^{when}'"]
+
         def _extracts_errors(triggers, summary):
             _errors = []
             for trigger in list(triggers):
-                when_spec = spack.spec.Spec(trigger)
-                if when_spec.name is not None and when_spec.name != pkg_name:
+                if not _refers_to_pkg(trigger):
                     details = [f"using '{trigger}', should be '^{trigger}'"]
                     _errors.append(error_cls(summary=summary, details=details))
             return _errors
 
-        for dname, triggers in pkg_cls.dependencies.items():
-            summary = f"{pkg_name}: wrong 'when=' condition for the '{dname}' dependency"
-            errors.extend(_extracts_errors(triggers, summary))
+        for when, dnames, details in _error_items(pkg_cls.dependencies):
+            errors.extend(
+                error_cls(f"{pkg_name}: wrong 'when=' condition for '{dname}' dependency", details)
+                for dname in dnames
+            )
 
         for vname, (variant, triggers) in pkg_cls.variants.items():
             summary = f"{pkg_name}: wrong 'when=' condition for the '{vname}' variant"
             errors.extend(_extracts_errors(triggers, summary))
 
-        for provided, triggers in pkg_cls.provided.items():
-            summary = f"{pkg_name}: wrong 'when=' condition for the '{provided}' virtual"
-            errors.extend(_extracts_errors(triggers, summary))
+        for when, providers, details in _error_items(pkg_cls.provided):
+            errors.extend(
+                error_cls(f"{pkg_name}: wrong 'when=' condition for '{provided}' virtual", details)
+                for provided in providers
+            )
 
-        for _, triggers in pkg_cls.requirements.items():
-            triggers = [when_spec for when_spec, _, _ in triggers]
-            summary = f"{pkg_name}: wrong 'when=' condition in 'requires' directive"
-            errors.extend(_extracts_errors(triggers, summary))
+        for when, requirements, details in _error_items(pkg_cls.requirements):
+            errors.append(
+                error_cls(f"{pkg_name}: wrong 'when=' condition in 'requires' directive", details)
+            )
 
-        triggers = list(pkg_cls.patches)
-        summary = f"{pkg_name}: wrong 'when=' condition in 'patch' directives"
-        errors.extend(_extracts_errors(triggers, summary))
+        for when, _, details in _error_items(pkg_cls.patches):
+            errors.append(
+                error_cls(f"{pkg_name}: wrong 'when=' condition in 'patch' directives", details)
+            )
 
-        triggers = list(pkg_cls.resources)
-        summary = f"{pkg_name}: wrong 'when=' condition in 'resource' directives"
-        errors.extend(_extracts_errors(triggers, summary))
+        for when, _, details in _error_items(pkg_cls.resources):
+            errors.append(
+                error_cls(f"{pkg_name}: wrong 'when=' condition in 'resource' directives", details)
+            )
 
     return llnl.util.lang.dedupe(errors)
 

lib/spack/spack/binary_distribution.py

@@ -1,11 +1,10 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import codecs
 import collections
-import errno
 import hashlib
 import io
 import itertools
@@ -23,9 +22,8 @@
 import urllib.parse
 import urllib.request
 import warnings
-from contextlib import closing, contextmanager
-from gzip import GzipFile
-from typing import Dict, List, NamedTuple, Optional, Set, Tuple
+from contextlib import closing
+from typing import Dict, Iterable, List, NamedTuple, Optional, Set, Tuple
 from urllib.error import HTTPError, URLError
 
 import llnl.util.filesystem as fsys
@@ -50,6 +48,7 @@
 import spack.stage
 import spack.store
 import spack.traverse as traverse
+import spack.util.archive
 import spack.util.crypto
 import spack.util.file_cache as file_cache
 import spack.util.gpg
@@ -68,7 +67,10 @@
 
 BUILD_CACHE_RELATIVE_PATH = "build_cache"
 BUILD_CACHE_KEYS_RELATIVE_PATH = "_pgp"
-CURRENT_BUILD_CACHE_LAYOUT_VERSION = 1
+
+#: The build cache layout version that this version of Spack creates.
+#: Version 2: includes parent directories of the package prefix in the tarball
+CURRENT_BUILD_CACHE_LAYOUT_VERSION = 2
 
 
 class BuildCacheDatabase(spack_db.Database):
@@ -230,7 +232,11 @@ def _associate_built_specs_with_mirror(self, cache_key, mirror_url):
                 )
                 return
 
-            spec_list = db.query_local(installed=False, in_buildcache=True)
+            spec_list = [
+                s
+                for s in db.query_local(installed=any, in_buildcache=any)
+                if s.external or db.query_local_by_spec_hash(s.dag_hash()).in_buildcache
+            ]
 
             for indexed_spec in spec_list:
                 dag_hash = indexed_spec.dag_hash()
@@ -1126,195 +1132,46 @@ def generate_key_index(key_prefix, tmpdir=None):
                 shutil.rmtree(tmpdir)
 
 
-@contextmanager
-def gzip_compressed_tarfile(path):
-    """Create a reproducible, compressed tarfile"""
-    # Create gzip compressed tarball of the install prefix
-    # 1) Use explicit empty filename and mtime 0 for gzip header reproducibility.
-    #    If the filename="" is dropped, Python will use fileobj.name instead.
-    #    This should effectively mimick `gzip --no-name`.
-    # 2) On AMD Ryzen 3700X and an SSD disk, we have the following on compression speed:
-    # compresslevel=6 gzip default: llvm takes 4mins, roughly 2.1GB
-    # compresslevel=9 python default: llvm takes 12mins, roughly 2.1GB
-    # So we follow gzip.
-    with open(path, "wb") as f, ChecksumWriter(f) as inner_checksum, closing(
-        GzipFile(filename="", mode="wb", compresslevel=6, mtime=0, fileobj=inner_checksum)
-    ) as gzip_file, ChecksumWriter(gzip_file) as outer_checksum, tarfile.TarFile(
-        name="", mode="w", fileobj=outer_checksum
-    ) as tar:
-        yield tar, inner_checksum, outer_checksum
-
-
-def _tarinfo_name(absolute_path: str, *, _path=pathlib.PurePath) -> str:
-    """Compute tarfile entry name as the relative path from the (system) root."""
-    return _path(*_path(absolute_path).parts[1:]).as_posix()
-
-
 def tarfile_of_spec_prefix(tar: tarfile.TarFile, prefix: str) -> None:
     """Create a tarfile of an install prefix of a spec. Skips existing buildinfo file.
-    Only adds regular files, symlinks and dirs. Skips devices, fifos. Preserves hardlinks.
-    Normalizes permissions like git. Tar entries are added in depth-first pre-order, with
-    dir entries partitioned by file | dir, and sorted alphabetically, for reproducibility.
-    Partitioning ensures only one dir is in memory at a time, and sorting improves compression.
 
     Args:
         tar: tarfile object to add files to
         prefix: absolute install prefix of spec"""
     if not os.path.isabs(prefix) or not os.path.isdir(prefix):
         raise ValueError(f"prefix '{prefix}' must be an absolute path to a directory")
-    hardlink_to_tarinfo_name: Dict[Tuple[int, int], str] = dict()
     stat_key = lambda stat: (stat.st_dev, stat.st_ino)
 
     try:  # skip buildinfo file if it exists
         files_to_skip = [stat_key(os.lstat(buildinfo_file_name(prefix)))]
+        skip = lambda entry: stat_key(entry.stat(follow_symlinks=False)) in files_to_skip
     except OSError:
-        files_to_skip = []
+        skip = lambda entry: False
 
-    dir_stack = [prefix]
-    while dir_stack:
-        dir = dir_stack.pop()
-
-        # Add the dir before its contents
-        dir_info = tarfile.TarInfo(_tarinfo_name(dir))
-        dir_info.type = tarfile.DIRTYPE
-        dir_info.mode = 0o755
-        tar.addfile(dir_info)
-
-        # Sort by name: reproducible & improves compression
-        with os.scandir(dir) as it:
-            entries = sorted(it, key=lambda entry: entry.name)
-
-        new_dirs = []
-        for entry in entries:
-            if entry.is_dir(follow_symlinks=False):
-                new_dirs.append(entry.path)
-                continue
-
-            file_info = tarfile.TarInfo(_tarinfo_name(entry.path))
-
-            s = entry.stat(follow_symlinks=False)
-
-            # Skip existing binary distribution files.
-            id = stat_key(s)
-            if id in files_to_skip:
-                continue
-
-            # Normalize the mode
-            file_info.mode = 0o644 if s.st_mode & 0o100 == 0 else 0o755
-
-            if entry.is_symlink():
-                file_info.type = tarfile.SYMTYPE
-                file_info.linkname = os.readlink(entry.path)
-                tar.addfile(file_info)
-
-            elif entry.is_file(follow_symlinks=False):
-                # Deduplicate hardlinks
-                if s.st_nlink > 1:
-                    if id in hardlink_to_tarinfo_name:
-                        file_info.type = tarfile.LNKTYPE
-                        file_info.linkname = hardlink_to_tarinfo_name[id]
-                        tar.addfile(file_info)
-                        continue
-                    hardlink_to_tarinfo_name[id] = file_info.name
-
-                # If file not yet seen, copy it.
-                file_info.type = tarfile.REGTYPE
-                file_info.size = s.st_size
-
-                with open(entry.path, "rb") as f:
-                    tar.addfile(file_info, f)
-
-        dir_stack.extend(reversed(new_dirs))  # we pop, so reverse to stay alphabetical
-
-
-class ChecksumWriter(io.BufferedIOBase):
-    """Checksum writer computes a checksum while writing to a file."""
-
-    myfileobj = None
-
-    def __init__(self, fileobj, algorithm=hashlib.sha256):
-        self.fileobj = fileobj
-        self.hasher = algorithm()
-        self.length = 0
-
-    def hexdigest(self):
-        return self.hasher.hexdigest()
-
-    def write(self, data):
-        if isinstance(data, (bytes, bytearray)):
-            length = len(data)
-        else:
-            data = memoryview(data)
-            length = data.nbytes
-
-        if length > 0:
-            self.fileobj.write(data)
-            self.hasher.update(data)
-
-        self.length += length
-
-        return length
-
-    def read(self, size=-1):
-        raise OSError(errno.EBADF, "read() on write-only object")
-
-    def read1(self, size=-1):
-        raise OSError(errno.EBADF, "read1() on write-only object")
-
-    def peek(self, n):
-        raise OSError(errno.EBADF, "peek() on write-only object")
-
-    @property
-    def closed(self):
-        return self.fileobj is None
-
-    def close(self):
-        fileobj = self.fileobj
-        if fileobj is None:
-            return
-        self.fileobj.close()
-        self.fileobj = None
-
-    def flush(self):
-        self.fileobj.flush()
-
-    def fileno(self):
-        return self.fileobj.fileno()
-
-    def rewind(self):
-        raise OSError("Can't rewind while computing checksum")
-
-    def readable(self):
-        return False
-
-    def writable(self):
-        return True
-
-    def seekable(self):
-        return True
-
-    def tell(self):
-        return self.fileobj.tell()
-
-    def seek(self, offset, whence=io.SEEK_SET):
-        # In principle forward seek is possible with b"0" padding,
-        # but this is not implemented.
-        if offset == 0 and whence == io.SEEK_CUR:
-            return
-        raise OSError("Can't seek while computing checksum")
-
-    def readline(self, size=-1):
-        raise OSError(errno.EBADF, "readline() on write-only object")
+    spack.util.archive.reproducible_tarfile_from_prefix(
+        tar,
+        prefix,
+        # Spack <= 0.21 did not include parent directories, leading to issues when tarballs are
+        # used in runtimes like AWS lambda.
+        include_parent_directories=True,
+        skip=skip,
+    )
 
 
 def _do_create_tarball(tarfile_path: str, binaries_dir: str, buildinfo: dict):
-    with gzip_compressed_tarfile(tarfile_path) as (tar, inner_checksum, outer_checksum):
+    with spack.util.archive.gzip_compressed_tarfile(tarfile_path) as (
+        tar,
+        inner_checksum,
+        outer_checksum,
+    ):
         # Tarball the install prefix
         tarfile_of_spec_prefix(tar, binaries_dir)
 
         # Serialize buildinfo for the tarball
         bstring = syaml.dump(buildinfo, default_flow_style=True).encode("utf-8")
-        tarinfo = tarfile.TarInfo(name=_tarinfo_name(buildinfo_file_name(binaries_dir)))
+        tarinfo = tarfile.TarInfo(
+            name=spack.util.archive.default_path_to_name(buildinfo_file_name(binaries_dir))
+        )
         tarinfo.type = tarfile.REGTYPE
         tarinfo.size = len(bstring)
         tarinfo.mode = 0o644
@@ -1601,14 +1458,14 @@ def _get_valid_spec_file(path: str, max_supported_layout: int) -> Tuple[Dict, in
     return spec_dict, layout_version
 
 
-def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
+def download_tarball(spec, unsigned: Optional[bool] = False, mirrors_for_spec=None):
     """
     Download binary tarball for given package into stage area, returning
     path to downloaded tarball if successful, None otherwise.
 
     Args:
         spec (spack.spec.Spec): Concrete spec
-        unsigned (bool): Whether or not to require signed binaries
+        unsigned: if ``True`` or ``False`` override the mirror signature verification defaults
         mirrors_for_spec (list): Optional list of concrete specs and mirrors
             obtained by calling binary_distribution.get_mirrors_for_spec().
             These will be checked in order first before looking in other
@@ -1629,7 +1486,9 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
            "signature_verified": "true-if-binary-pkg-was-already-verified"
        }
     """
-    configured_mirrors = spack.mirror.MirrorCollection(binary=True).values()
+    configured_mirrors: Iterable[spack.mirror.Mirror] = spack.mirror.MirrorCollection(
+        binary=True
+    ).values()
     if not configured_mirrors:
         tty.die("Please add a spack mirror to allow download of pre-compiled packages.")
 
@@ -1647,8 +1506,16 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
     # mirror for the spec twice though.
     try_first = [i["mirror_url"] for i in mirrors_for_spec] if mirrors_for_spec else []
     try_next = [i.fetch_url for i in configured_mirrors if i.fetch_url not in try_first]
+    mirror_urls = try_first + try_next
 
-    mirrors = try_first + try_next
+    # TODO: turn `mirrors_for_spec` into a list of Mirror instances, instead of doing that here.
+    def fetch_url_to_mirror(url):
+        for mirror in configured_mirrors:
+            if mirror.fetch_url == url:
+                return mirror
+        return spack.mirror.Mirror(url)
+
+    mirrors = [fetch_url_to_mirror(url) for url in mirror_urls]
 
     tried_to_verify_sigs = []
 
@@ -1657,14 +1524,17 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
     # we remove support for deprecated spec formats and buildcache layouts.
     for try_signed in (True, False):
         for mirror in mirrors:
+            # Override mirror's default if
+            currently_unsigned = unsigned if unsigned is not None else not mirror.signed
+
             # If it's an OCI index, do things differently, since we cannot compose URLs.
-            parsed = urllib.parse.urlparse(mirror)
+            fetch_url = mirror.fetch_url
 
             # TODO: refactor this to some "nice" place.
-            if parsed.scheme == "oci":
-                ref = spack.oci.image.ImageReference.from_string(mirror[len("oci://") :]).with_tag(
-                    spack.oci.image.default_tag(spec)
-                )
+            if fetch_url.startswith("oci://"):
+                ref = spack.oci.image.ImageReference.from_string(
+                    fetch_url[len("oci://") :]
+                ).with_tag(spack.oci.image.default_tag(spec))
 
                 # Fetch the manifest
                 try:
@@ -1701,7 +1571,7 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
                         except InvalidMetadataFile as e:
                             tty.warn(
                                 f"Ignoring binary package for {spec.name}/{spec.dag_hash()[:7]} "
-                                f"from {mirror} due to invalid metadata file: {e}"
+                                f"from {fetch_url} due to invalid metadata file: {e}"
                             )
                             local_specfile_stage.destroy()
                             continue
@@ -1723,13 +1593,16 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
                     "tarball_stage": tarball_stage,
                     "specfile_stage": local_specfile_stage,
                     "signature_verified": False,
+                    "signature_required": not currently_unsigned,
                 }
 
             else:
                 ext = "json.sig" if try_signed else "json"
-                specfile_path = url_util.join(mirror, BUILD_CACHE_RELATIVE_PATH, specfile_prefix)
+                specfile_path = url_util.join(
+                    fetch_url, BUILD_CACHE_RELATIVE_PATH, specfile_prefix
+                )
                 specfile_url = f"{specfile_path}.{ext}"
-                spackfile_url = url_util.join(mirror, BUILD_CACHE_RELATIVE_PATH, tarball)
+                spackfile_url = url_util.join(fetch_url, BUILD_CACHE_RELATIVE_PATH, tarball)
                 local_specfile_stage = try_fetch(specfile_url)
                 if local_specfile_stage:
                     local_specfile_path = local_specfile_stage.save_filename
@@ -1742,21 +1615,21 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
                     except InvalidMetadataFile as e:
                         tty.warn(
                             f"Ignoring binary package for {spec.name}/{spec.dag_hash()[:7]} "
-                            f"from {mirror} due to invalid metadata file: {e}"
+                            f"from {fetch_url} due to invalid metadata file: {e}"
                         )
                         local_specfile_stage.destroy()
                         continue
 
-                    if try_signed and not unsigned:
+                    if try_signed and not currently_unsigned:
                         # If we found a signed specfile at the root, try to verify
                         # the signature immediately.  We will not download the
                         # tarball if we could not verify the signature.
                         tried_to_verify_sigs.append(specfile_url)
                         signature_verified = try_verify(local_specfile_path)
                         if not signature_verified:
-                            tty.warn("Failed to verify: {0}".format(specfile_url))
+                            tty.warn(f"Failed to verify: {specfile_url}")
 
-                    if unsigned or signature_verified or not try_signed:
+                    if currently_unsigned or signature_verified or not try_signed:
                         # We will download the tarball in one of three cases:
                         #     1. user asked for --no-check-signature
                         #     2. user didn't ask for --no-check-signature, but we
@@ -1779,6 +1652,7 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
                                 "tarball_stage": tarball_stage,
                                 "specfile_stage": local_specfile_stage,
                                 "signature_verified": signature_verified,
+                                "signature_required": not currently_unsigned,
                             }
 
                     local_specfile_stage.destroy()
@@ -1977,7 +1851,7 @@ def is_backup_file(file):
             relocate.relocate_text(text_names, prefix_to_prefix_text)
 
 
-def _extract_inner_tarball(spec, filename, extract_to, unsigned, remote_checksum):
+def _extract_inner_tarball(spec, filename, extract_to, signature_required: bool, remote_checksum):
     stagepath = os.path.dirname(filename)
     spackfile_name = tarball_name(spec, ".spack")
     spackfile_path = os.path.join(stagepath, spackfile_name)
@@ -1997,7 +1871,7 @@ def _extract_inner_tarball(spec, filename, extract_to, unsigned, remote_checksum
     else:
         raise ValueError("Cannot find spec file for {0}.".format(extract_to))
 
-    if not unsigned:
+    if signature_required:
         if os.path.exists("%s.asc" % specfile_path):
             suppress = config.get("config:suppress_gpg_warnings", False)
             try:
@@ -2026,11 +1900,12 @@ def _extract_inner_tarball(spec, filename, extract_to, unsigned, remote_checksum
 
 
 def _tar_strip_component(tar: tarfile.TarFile, prefix: str):
-    """Strip the top-level directory `prefix` from the member names in a tarfile."""
+    """Yield all members of tarfile that start with given prefix, and strip that prefix (including
+    symlinks)"""
     # Including trailing /, otherwise we end up with absolute paths.
     regex = re.compile(re.escape(prefix) + "/*")
 
-    # Remove the top-level directory from the member (link)names.
+    # Only yield members in the package prefix.
     # Note: when a tarfile is created, relative in-prefix symlinks are
     # expanded to matching member names of tarfile entries. So, we have
     # to ensure that those are updated too.
@@ -2038,15 +1913,17 @@ def _tar_strip_component(tar: tarfile.TarFile, prefix: str):
     # them.
     for m in tar.getmembers():
         result = regex.match(m.name)
-        assert result is not None
+        if not result:
+            continue
         m.name = m.name[result.end() :]
         if m.linkname:
             result = regex.match(m.linkname)
             if result:
                 m.linkname = m.linkname[result.end() :]
+        yield m
 
 
-def extract_tarball(spec, download_result, unsigned=False, force=False, timer=timer.NULL_TIMER):
+def extract_tarball(spec, download_result, force=False, timer=timer.NULL_TIMER):
     """
     extract binary tarball for given package into install area
     """
@@ -2072,7 +1949,8 @@ def extract_tarball(spec, download_result, unsigned=False, force=False, timer=ti
     bchecksum = spec_dict["binary_cache_checksum"]
 
     filename = download_result["tarball_stage"].save_filename
-    signature_verified = download_result["signature_verified"]
+    signature_verified: bool = download_result["signature_verified"]
+    signature_required: bool = download_result["signature_required"]
     tmpdir = None
 
     if layout_version == 0:
@@ -2081,12 +1959,14 @@ def extract_tarball(spec, download_result, unsigned=False, force=False, timer=ti
         # and another tarball containing the actual install tree.
         tmpdir = tempfile.mkdtemp()
         try:
-            tarfile_path = _extract_inner_tarball(spec, filename, tmpdir, unsigned, bchecksum)
+            tarfile_path = _extract_inner_tarball(
+                spec, filename, tmpdir, signature_required, bchecksum
+            )
         except Exception as e:
             _delete_staged_downloads(download_result)
             shutil.rmtree(tmpdir)
             raise e
-    elif layout_version == 1:
+    elif 1 <= layout_version <= 2:
         # Newer buildcache layout: the .spack file contains just
         # in the install tree, the signature, if it exists, is
         # wrapped around the spec.json at the root.  If sig verify
@@ -2094,9 +1974,10 @@ def extract_tarball(spec, download_result, unsigned=False, force=False, timer=ti
         # the tarball.
         tarfile_path = filename
 
-        if not unsigned and not signature_verified:
+        if signature_required and not signature_verified:
             raise UnsignedPackageException(
-                "To install unsigned packages, use the --no-check-signature option."
+                "To install unsigned packages, use the --no-check-signature option, "
+                "or configure the mirror with signed: false."
             )
 
         # compute the sha256 checksum of the tarball
@@ -2113,8 +1994,10 @@ def extract_tarball(spec, download_result, unsigned=False, force=False, timer=ti
     try:
         with closing(tarfile.open(tarfile_path, "r")) as tar:
             # Remove install prefix from tarfil to extract directly into spec.prefix
-            _tar_strip_component(tar, prefix=_ensure_common_prefix(tar))
-            tar.extractall(path=spec.prefix)
+            tar.extractall(
+                path=spec.prefix,
+                members=_tar_strip_component(tar, prefix=_ensure_common_prefix(tar)),
+            )
     except Exception:
         shutil.rmtree(spec.prefix, ignore_errors=True)
         _delete_staged_downloads(download_result)
@@ -2149,20 +2032,47 @@ def extract_tarball(spec, download_result, unsigned=False, force=False, timer=ti
 
 
 def _ensure_common_prefix(tar: tarfile.TarFile) -> str:
-    # Get the shortest length directory.
-    common_prefix = min((e.name for e in tar.getmembers() if e.isdir()), key=len, default=None)
+    # Find the lowest `binary_distribution` file (hard-coded forward slash is on purpose).
+    binary_distribution = min(
+        (
+            e.name
+            for e in tar.getmembers()
+            if e.isfile() and e.name.endswith(".spack/binary_distribution")
+        ),
+        key=len,
+        default=None,
+    )
 
-    if common_prefix is None:
-        raise ValueError("Tarball does not contain a common prefix")
+    if binary_distribution is None:
+        raise ValueError("Tarball is not a Spack package, missing binary_distribution file")
 
-    # Validate that each file starts with the prefix
+    pkg_path = pathlib.PurePosixPath(binary_distribution).parent.parent
+
+    # Even the most ancient Spack version has required to list the dir of the package itself, so
+    # guard against broken tarballs where `path.parent.parent` is empty.
+    if pkg_path == pathlib.PurePosixPath():
+        raise ValueError("Invalid tarball, missing package prefix dir")
+
+    pkg_prefix = str(pkg_path)
+
+    # Ensure all tar entries are in the pkg_prefix dir, and if they're not, they should be parent
+    # dirs of it.
+    has_prefix = False
     for member in tar.getmembers():
-        if not member.name.startswith(common_prefix):
-            raise ValueError(
-                f"Tarball contains file {member.name} outside of prefix {common_prefix}"
-            )
+        stripped = member.name.rstrip("/")
+        if not (
+            stripped.startswith(pkg_prefix) or member.isdir() and pkg_prefix.startswith(stripped)
+        ):
+            raise ValueError(f"Tarball contains file {stripped} outside of prefix {pkg_prefix}")
+        if member.isdir() and stripped == pkg_prefix:
+            has_prefix = True
 
-    return common_prefix
+    # This is technically not required, but let's be defensive about the existence of the package
+    # prefix dir.
+    if not has_prefix:
+        raise ValueError(f"Tarball does not contain a common prefix {pkg_prefix}")
+
+    return pkg_prefix
 
 
 def install_root_node(spec, unsigned=False, force=False, sha256=None):
@@ -2209,7 +2119,7 @@ def install_root_node(spec, unsigned=False, force=False, sha256=None):
     # don't print long padded paths while extracting/relocating binaries
     with spack.util.path.filter_padding():
         tty.msg('Installing "{0}" from a buildcache'.format(spec.format()))
-        extract_tarball(spec, download_result, unsigned, force)
+        extract_tarball(spec, download_result, force)
         spack.hooks.post_install(spec, False)
         spack.store.STORE.db.add(spec, spack.store.STORE.layout)
 

lib/spack/spack/bootstrap/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/bootstrap/_common.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -16,6 +16,7 @@
 import llnl.util.filesystem as fs
 from llnl.util import tty
 
+import spack.platforms
 import spack.store
 import spack.util.environment
 import spack.util.executable
@@ -206,16 +207,19 @@ def _root_spec(spec_str: str) -> str:
     """Add a proper compiler and target to a spec used during bootstrapping.
 
     Args:
-        spec_str (str): spec to be bootstrapped. Must be without compiler and target.
+        spec_str: spec to be bootstrapped. Must be without compiler and target.
     """
-    # Add a proper compiler hint to the root spec. We use GCC for
-    # everything but MacOS and Windows.
-    if str(spack.platforms.host()) == "darwin":
+    # Add a compiler requirement to the root spec.
+    platform = str(spack.platforms.host())
+    if platform == "darwin":
         spec_str += " %apple-clang"
-    elif str(spack.platforms.host()) == "windows":
-        spec_str += " %msvc"
-    else:
+    elif platform == "windows":
+        # TODO (johnwparent): Remove version constraint when clingo patch is up
+        spec_str += " %msvc@:19.37"
+    elif platform == "linux":
         spec_str += " %gcc"
+    elif platform == "freebsd":
+        spec_str += " %clang"
 
     target = archspec.cpu.host().family
     spec_str += f" target={target}"

lib/spack/spack/bootstrap/config.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)