use underline instead of bold for the highlight

It can be hard to tell when the concretizer has made decisions that are not the defaults
for packages. This introduces a `spack spec -d` option that will highlight the
nondefault options in red.

You can use this to see:
- When a selected version is not the most preferred;
- When a selected compiler is not the most preferred;
- When a nondefault variant is selected;
- When the chosen target is not the native microarchitecture;
- When a chosen virtual provider is not the most preferred.

With `--reuse`, you'll see clearly which versions and options on reused packages are not
current with the latest `package.py` files. With `--fresh`, you can see which options
are forced by particular package preferences. e.g., `spack spec -d tar` shows `zstd`'s
`+programs` option in red because it's off by default, but `tar` requires it via
`depends_on("zstd+programs")`.

This should be useful for debugging strange/unintuitive concretizations.

- [x] Modify concretizer to return optimization weights
- [x] Add `_weights` metadata to the `Spec` object
- [x] Modify `Spec.format()` to colorize nondefaults when weights are nonzero
- [x] Add tests

.github/workflows/audit.yaml

@@ -22,7 +22,7 @@ jobs:
       matrix:
         operating_system: ["ubuntu-latest", "macos-latest"]
     steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # @v2
     - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
       with:
         python-version: ${{inputs.python_version}}
@@ -43,7 +43,7 @@ jobs:
           . share/spack/setup-env.sh
           $(which spack) audit packages
           $(which spack) audit externals
-    - uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # @v2.1.0
+    - uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab # @v2.1.0
       if: ${{ inputs.with_coverage == 'true' }}
       with:
         flags: unittests,audits

.github/workflows/bootstrap.yml

@@ -24,7 +24,7 @@ jobs:
               make patch unzip which xz python3 python3-devel tree \
               cmake bison bison-devel libstdc++-static
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
         with:
           fetch-depth: 0
       - name: Setup non-root user
@@ -62,7 +62,7 @@ jobs:
               make patch unzip xz-utils python3 python3-dev tree \
               cmake bison
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
         with:
           fetch-depth: 0
       - name: Setup non-root user
@@ -99,7 +99,7 @@ jobs:
               bzip2 curl file g++ gcc gfortran git gnupg2 gzip \
               make patch unzip xz-utils python3 python3-dev tree
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
         with:
           fetch-depth: 0
       - name: Setup non-root user
@@ -133,7 +133,7 @@ jobs:
               make patch unzip which xz python3 python3-devel tree \
               cmake bison
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
         with:
           fetch-depth: 0
       - name: Setup repo
@@ -158,7 +158,7 @@ jobs:
         run: |
           brew install cmake bison@2.7 tree
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
         with:
           python-version: "3.12"
@@ -182,7 +182,7 @@ jobs:
         run: |
           brew install tree
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: Bootstrap clingo
         run: |
           set -ex
@@ -207,7 +207,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
         with:
           fetch-depth: 0
       - name: Setup repo
@@ -250,7 +250,7 @@ jobs:
               bzip2 curl file g++ gcc patchelf gfortran git gzip \
               make patch unzip xz-utils python3 python3-dev tree
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
         with:
           fetch-depth: 0
       - name: Setup non-root user
@@ -287,7 +287,7 @@ jobs:
               make patch unzip xz-utils python3 python3-dev tree \
               gawk
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
         with:
           fetch-depth: 0
       - name: Setup non-root user
@@ -320,7 +320,7 @@ jobs:
           # Remove GnuPG since we want to bootstrap it
           sudo rm -rf /usr/local/bin/gpg
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: Bootstrap GnuPG
         run: |
           source share/spack/setup-env.sh
@@ -338,7 +338,7 @@ jobs:
           # Remove GnuPG since we want to bootstrap it
           sudo rm -rf /usr/local/bin/gpg
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: Bootstrap GnuPG
         run: |
           source share/spack/setup-env.sh

.github/workflows/build-containers.yml

@@ -55,7 +55,7 @@ jobs:
     if: github.repository == 'spack/spack'
     steps:
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # @v2
 
       - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
         id: docker_meta
@@ -96,10 +96,10 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226
+        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c
 
       - name: Log in to GitHub Container Registry
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -107,13 +107,13 @@ jobs:
 
       - name: Log in to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build & Deploy ${{ matrix.dockerfile[0] }}
-        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56
+        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e
         with:
           context: dockerfiles/${{ matrix.dockerfile[0] }}
           platforms: ${{ matrix.dockerfile[1] }}

.github/workflows/ci.yaml

@@ -35,12 +35,12 @@ jobs:
       core: ${{ steps.filter.outputs.core }}
       packages: ${{ steps.filter.outputs.packages }}
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # @v2
         if: ${{ github.event_name == 'push' }}
         with:
           fetch-depth: 0
             # For pull requests it's not necessary to checkout the code
-      - uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
         id: filter
         with:
           # See https://github.com/dorny/paths-filter/issues/56 for the syntax used below

.github/workflows/nightly-win-builds.yml

@@ -14,7 +14,7 @@ jobs:
   build-paraview-deps:
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       with:
         fetch-depth: 0
     - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c

.github/workflows/style/requirements.txt

@@ -1,5 +1,5 @@
-black==24.1.1
-clingo==5.6.2
+black==24.3.0
+clingo==5.7.1
 flake8==7.0.0
 isort==5.13.2
 mypy==1.8.0

.github/workflows/unit_tests.yaml

@@ -51,7 +51,7 @@ jobs:
           on_develop: false
 
     steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # @v2
       with:
         fetch-depth: 0
     - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
@@ -91,14 +91,14 @@ jobs:
           UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }}
       run: |
           share/spack/qa/run-unit-tests
-    - uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044
+    - uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab
       with:
         flags: unittests,linux,${{ matrix.concretizer }}
   # Test shell integration
   shell:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # @v2
       with:
         fetch-depth: 0
     - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
@@ -122,7 +122,7 @@ jobs:
           COVERAGE: true
       run: |
           share/spack/qa/run-shell-tests
-    - uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044
+    - uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab
       with:
         flags: shelltests,linux
 
@@ -137,7 +137,7 @@ jobs:
           dnf install -y \
               bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
               make patch tcl unzip which xz
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # @v2
     - name: Setup repo and non-root user
       run: |
           git --version
@@ -156,7 +156,7 @@ jobs:
   clingo-cffi:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # @v2
       with:
         fetch-depth: 0
     - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
@@ -181,7 +181,7 @@ jobs:
           SPACK_TEST_SOLVER: clingo
       run: |
           share/spack/qa/run-unit-tests
-    - uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # @v2.1.0
+    - uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab # @v2.1.0
       with:
         flags: unittests,linux,clingo
   # Run unit tests on MacOS
@@ -191,7 +191,7 @@ jobs:
       matrix:
         python-version: ["3.11"]
     steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # @v2
       with:
         fetch-depth: 0
     - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
@@ -216,6 +216,6 @@ jobs:
         $(which spack) solve zlib
         common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
         $(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
-    - uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044
+    - uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab
       with:
         flags: unittests,macos

.github/workflows/valid-style.yml

@@ -18,7 +18,7 @@ jobs:
   validate:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
     - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
       with:
         python-version: '3.11'
@@ -35,7 +35,7 @@ jobs:
   style:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       with:
         fetch-depth: 0
     - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
@@ -69,7 +69,7 @@ jobs:
           dnf install -y \
               bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
               make patch tcl unzip which xz
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # @v2
       - name: Setup repo and non-root user
         run: |
           git --version

.github/workflows/windows_python.yml

@@ -15,7 +15,7 @@ jobs:
   unit-tests:
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       with:
         fetch-depth: 0
     - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
@@ -33,13 +33,13 @@ jobs:
         ./share/spack/qa/validate_last_exit.ps1
         coverage combine -a
         coverage xml
-    - uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044
+    - uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab
       with:
         flags: unittests,windows
   unit-tests-cmd:
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       with:
         fetch-depth: 0
     - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
@@ -57,13 +57,13 @@ jobs:
         ./share/spack/qa/validate_last_exit.ps1
         coverage combine -a
         coverage xml
-    - uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044
+    - uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab
       with:
         flags: unittests,windows
   build-abseil:
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       with:
         fetch-depth: 0
     - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c

lib/spack/docs/bootstrapping.rst

@@ -87,7 +87,7 @@ You can check what is installed in the bootstrapping store at any time using:
 
 .. code-block:: console
 
-   % spack find -b
+   % spack -b find
    ==> Showing internal bootstrap store at "/Users/spack/.spack/bootstrap/store"
    ==> 11 installed packages
    -- darwin-catalina-x86_64 / apple-clang@12.0.0 ------------------
@@ -101,7 +101,7 @@ In case it is needed you can remove all the software in the current bootstrappin
    % spack clean -b
    ==> Removing bootstrapped software and configuration in "/Users/spack/.spack/bootstrap"
 
-   % spack find -b
+   % spack -b find
    ==> Showing internal bootstrap store at "/Users/spack/.spack/bootstrap/store"
    ==> 0 installed packages
 
@@ -175,4 +175,4 @@ bootstrapping.
 
 This command needs to be run on a machine with internet access and the resulting folder
 has to be moved over to the air-gapped system. Once the local sources are added using the
-commands suggested at the prompt, they can be used to bootstrap Spack.
+commands suggested at the prompt, they can be used to bootstrap Spack.

lib/spack/docs/build_systems/perlpackage.rst

@@ -173,6 +173,72 @@ arguments to ``Makefile.PL`` or ``Build.PL`` by overriding
        ]
 
 
+^^^^^^^
+Testing
+^^^^^^^
+
+``PerlPackage`` provides a simple stand-alone test of the successfully
+installed package to confirm that installed perl module(s) can be used.
+These tests can be performed any time after the installation using
+``spack -v test run``. (For more information on the command, see 
+:ref:`cmd-spack-test-run`.)
+
+The base class automatically detects perl modules based on the presence
+of ``*.pm`` files under the package's library directory. For example,
+the files under ``perl-bignum``'s perl library are:
+
+.. code-block:: console
+
+   $ find . -name "*.pm"
+   ./bigfloat.pm
+   ./bigrat.pm
+   ./Math/BigFloat/Trace.pm
+   ./Math/BigInt/Trace.pm
+   ./Math/BigRat/Trace.pm
+   ./bigint.pm
+   ./bignum.pm
+
+
+which results in the package having the ``use_modules`` property containing:
+
+.. code-block:: python
+
+   use_modules = [
+       "bigfloat",
+       "bigrat",
+       "Math::BigFloat::Trace",
+       "Math::BigInt::Trace",
+       "Math::BigRat::Trace",
+       "bigint",
+       "bignum",
+   ]
+
+.. note::
+
+   This list can often be used to catch missing dependencies.
+
+If the list is somehow wrong, you can provide the names of the modules
+yourself by overriding ``use_modules`` like so:
+
+ .. code-block:: python
+
+    use_modules = ["bigfloat", "bigrat", "bigint", "bignum"]
+
+If you only want a subset of the automatically detected modules to be
+tested, you could instead define the ``skip_modules`` property on the
+package. So, instead of overriding ``use_modules`` as shown above, you
+could define the following:
+
+ .. code-block:: python
+
+    skip_modules = [
+        "Math::BigFloat::Trace",
+        "Math::BigInt::Trace",
+        "Math::BigRat::Trace",
+    ]
+
+for the same use tests.
+
 ^^^^^^^^^^^^^^^^^^^^^
 Alternatives to Spack
 ^^^^^^^^^^^^^^^^^^^^^

lib/spack/docs/configuration.rst

@@ -73,9 +73,12 @@ are six configuration scopes. From lowest to highest:
    Spack instance per project) or for site-wide settings on a multi-user
    machine (e.g., for a common Spack instance).
 
+#. **plugin**: Read from a Python project's entry points. Settings here affect
+   all instances of Spack running with the same Python installation.  This scope takes higher precedence than site, system, and default scopes.
+
 #. **user**: Stored in the home directory: ``~/.spack/``. These settings
    affect all instances of Spack and take higher precedence than site,
-   system, or defaults scopes.
+   system, plugin, or defaults scopes.
 
 #. **custom**: Stored in a custom directory specified by ``--config-scope``.
    If multiple scopes are listed on the command line, they are ordered
@@ -196,6 +199,45 @@ with MPICH. You can create different configuration scopes for use with
                mpi: [mpich]
 
 
+.. _plugin-scopes:
+
+^^^^^^^^^^^^^
+Plugin scopes
+^^^^^^^^^^^^^
+
+.. note::
+   Python version >= 3.8 is required to enable plugin configuration.
+
+Spack can be made aware of configuration scopes that are installed as part of a python package.  To do so, register a function that returns the scope's path to the ``"spack.config"`` entry point.  Consider the Python package ``my_package`` that includes Spack configurations:
+
+.. code-block:: console
+
+  my-package/
+  ├── src
+  │   ├── my_package
+  │   │   ├── __init__.py
+  │   │   └── spack/
+  │   │   │   └── config.yaml
+  └── pyproject.toml
+
+adding the following to ``my_package``'s ``pyproject.toml`` will make ``my_package``'s ``spack/`` configurations visible to Spack when ``my_package`` is installed:
+
+.. code-block:: toml
+
+   [project.entry_points."spack.config"]
+   my_package = "my_package:get_config_path"
+
+The function ``my_package.get_extension_path`` in ``my_package/__init__.py`` might look like
+
+.. code-block:: python
+
+   import importlib.resources
+
+   def get_config_path():
+       dirname = importlib.resources.files("my_package").joinpath("spack")
+       if dirname.exists():
+           return str(dirname)
+
 .. _platform-scopes:
 
 ------------------------

lib/spack/docs/developer_guide.rst

@@ -357,91 +357,23 @@ If there is a hook that you would like and is missing, you can propose to add a
 ``pre_install(spec)``
 """""""""""""""""""""
 
-A ``pre_install`` hook is run within an install subprocess, directly before
-the install starts. It expects a single argument of a spec, and is run in
-a multiprocessing subprocess. Note that if you see ``pre_install`` functions associated with packages these are not hooks
-as we have defined them here, but rather callback functions associated with
-a package install.
+A ``pre_install`` hook is run within the install subprocess, directly before the install starts.
+It expects a single argument of a spec.
 
 
-""""""""""""""""""""""
-``post_install(spec)``
-""""""""""""""""""""""
+"""""""""""""""""""""""""""""""""""""
+``post_install(spec, explicit=None)``
+"""""""""""""""""""""""""""""""""""""
 
-A ``post_install`` hook is run within an install subprocess, directly after
-the install finishes, but before the build stage is removed. If you
-write one of these hooks, you should expect it to accept a spec as the only
-argument. This is run in a multiprocessing subprocess. This ``post_install`` is
-also seen in packages, but in this context not related to the hooks described
-here.
+A ``post_install`` hook is run within the install subprocess, directly after the install finishes,
+but before the build stage is removed and the spec is registered in the database. It expects two
+arguments: spec and an optional boolean indicating whether this spec is being installed explicitly.
 
+""""""""""""""""""""""""""""""""""""""""""""""""""""
+``pre_uninstall(spec)`` and ``post_uninstall(spec)``
+""""""""""""""""""""""""""""""""""""""""""""""""""""
 
-""""""""""""""""""""""""""
-``on_install_start(spec)``
-""""""""""""""""""""""""""
-
-This hook is run at the beginning of ``lib/spack/spack/installer.py``,
-in the install function of a ``PackageInstaller``,
-and importantly is not part of a build process, but before it. This is when
-we have just newly grabbed the task, and are preparing to install. If you
-write a hook of this type, you should provide the spec to it.
-
-.. code-block:: python
-
-    def on_install_start(spec):
-        """On start of an install, we want to...
-        """
-        print('on_install_start')
-
-
-""""""""""""""""""""""""""""
-``on_install_success(spec)``
-""""""""""""""""""""""""""""
-
-This hook is run on a successful install, and is also run inside the build
-process, akin to ``post_install``. The main difference is that this hook
-is run outside of the context of the stage directory, meaning after the
-build stage has been removed and the user is alerted that the install was
-successful. If you need to write a hook that is run on success of a particular
-phase, you should use ``on_phase_success``.
-
-""""""""""""""""""""""""""""
-``on_install_failure(spec)``
-""""""""""""""""""""""""""""
-
-This hook is run given an install failure that happens outside of the build
-subprocess, but somewhere in ``installer.py`` when something else goes wrong.
-If you need to write a hook that is relevant to a failure within a build
-process, you would want to instead use ``on_phase_failure``.
-
-
-"""""""""""""""""""""""""""
-``on_install_cancel(spec)``
-"""""""""""""""""""""""""""
-
-The same, but triggered if a spec install is cancelled for any reason.
-
-
-"""""""""""""""""""""""""""""""""""""""""""""""
-``on_phase_success(pkg, phase_name, log_file)``
-"""""""""""""""""""""""""""""""""""""""""""""""
-
-This hook is run within the install subprocess, and specifically when a phase
-successfully finishes. Since we are interested in the package, the name of
-the phase, and any output from it, we require:
-
- - **pkg**: the package variable, which also has the attached spec at ``pkg.spec``
- - **phase_name**: the name of the phase that was successful (e.g., configure)
- - **log_file**: the path to the file with output, in case you need to inspect or otherwise interact with it.
-
-"""""""""""""""""""""""""""""""""""""""""""""
-``on_phase_error(pkg, phase_name, log_file)``
-"""""""""""""""""""""""""""""""""""""""""""""
-
-In the case of an error during a phase, we might want to trigger some event
-with a hook, and this is the purpose of this particular hook. Akin to
-``on_phase_success`` we require the same variables - the package that failed,
-the name of the phase, and the log file where we might find errors.
+These hooks are currently used for cleaning up module files after uninstall.
 
 
 ^^^^^^^^^^^^^^^^^^^^^^

lib/spack/docs/environments.rst

@@ -416,6 +416,23 @@ that git clone if ``foo`` is in the environment.
 Further development on ``foo`` can be tested by reinstalling the environment,
 and eventually committed and pushed to the upstream git repo.
 
+If the package being developed supports out-of-source builds then users can use the
+``--build_directory`` flag to control the location and name of the build directory. 
+This is a shortcut to set the ``package_attributes:build_directory`` in the
+``packages`` configuration (see :ref:`assigning-package-attributes`).
+The supplied location will become the build-directory for that package in all future builds.
+
+.. warning::
+   Potential pitfalls of setting the build directory
+    Spack does not check for out-of-source build compatibility with the packages and
+    so the onerous of making sure the package supports out-of-source builds is on
+    the user.
+    For example, most ``autotool`` and ``makefile`` packages do not support out-of-source builds
+    while all ``CMake`` packages do.
+    Understanding these nuances are on the software developers and we strongly encourage
+    developers to only redirect the build directory if they understand their package's
+    build-system.
+
 ^^^^^^^
 Loading
 ^^^^^^^
@@ -935,6 +952,17 @@ function, as shown in the example below:
      ^mpi: "{name}-{version}/{^mpi.name}-{^mpi.version}-{compiler.name}-{compiler.version}"
      all: "{name}-{version}/{compiler.name}-{compiler.version}"
 
+Projections also permit environment and spack configuration variable
+expansions as shown below:
+
+.. code-block:: yaml
+
+   projections:
+     all: "{name}-{version}/{compiler.name}-{compiler.version}/$date/$SYSTEM_ENV_VARIBLE"
+
+where ``$date`` is the spack configuration variable that will expand with the ``YYYY-MM-DD``
+format and ``$SYSTEM_ENV_VARIABLE`` is an environment variable defined in the shell.
+
 The entries in the projections configuration file must all be either
 specs or the keyword ``all``. For each spec, the projection used will
 be the first non-``all`` entry that the spec satisfies, or ``all`` if

lib/spack/docs/extensions.rst

@@ -111,3 +111,39 @@ The corresponding unit tests can be run giving the appropriate options to ``spac
 
    (5 durations < 0.005s hidden.  Use -vv to show these durations.)
    =========================================== 5 passed in 5.06s ============================================
+
+---------------------------------------
+Registering Extensions via Entry Points
+---------------------------------------
+
+.. note::
+   Python version >= 3.8 is required to register extensions via entry points.
+
+Spack can be made aware of extensions that are installed as part of a python package.  To do so, register a function that returns the extension path, or paths, to the ``"spack.extensions"`` entry point.  Consider the Python package ``my_package`` that includes a Spack extension:
+
+.. code-block:: console
+
+  my-package/
+  ├── src
+  │   ├── my_package
+  │   │   └── __init__.py
+  │   └── spack-scripting/  # the spack extensions
+  └── pyproject.toml
+
+adding the following to ``my_package``'s ``pyproject.toml`` will make the ``spack-scripting`` extension visible to Spack when ``my_package`` is installed:
+
+.. code-block:: toml
+
+   [project.entry_points."spack.extenions"]
+   my_package = "my_package:get_extension_path"
+
+The function ``my_package.get_extension_path`` in ``my_package/__init__.py`` might look like
+
+.. code-block:: python
+
+   import importlib.resources
+
+   def get_extension_path():
+       dirname = importlib.resources.files("my_package").joinpath("spack-scripting")
+       if dirname.exists():
+           return str(dirname)

lib/spack/docs/getting_started.rst

@@ -250,9 +250,10 @@ Compiler configuration
 
 Spack has the ability to build packages with multiple compilers and
 compiler versions. Compilers can be made available to Spack by
-specifying them manually in ``compilers.yaml``, or automatically by
-running ``spack compiler find``, but for convenience Spack will
-automatically detect compilers the first time it needs them.
+specifying them manually in ``compilers.yaml`` or ``packages.yaml``,
+or automatically by running ``spack compiler find``, but for
+convenience Spack will automatically detect compilers the first time
+it needs them.
 
 .. _cmd-spack-compilers:
 
@@ -457,6 +458,48 @@ specification. The operations available to modify the environment are ``set``, `
          prepend_path: # Similar for append|remove_path
            LD_LIBRARY_PATH: /ld/paths/added/by/setvars/sh
 
+.. note::
+
+   Spack is in the process of moving compilers from a separate
+   attribute to be handled like all other packages. As part of this
+   process, the ``compilers.yaml`` section will eventually be replaced
+   by configuration in the ``packages.yaml`` section. This new
+   configuration is now available, although it is not yet the default
+   behavior.
+
+Compilers can also be configured as external packages in the
+``packages.yaml`` config file. Any external package for a compiler
+(e.g. ``gcc`` or ``llvm``) will be treated as a configured compiler
+assuming the paths to the compiler executables are determinable from
+the prefix.
+
+If the paths to the compiler executable are not determinable from the
+prefix, you can add them to the ``extra_attributes`` field. Similarly,
+all other fields from the compilers config can be added to the
+``extra_attributes`` field for an external representing a compiler.
+
+.. code-block:: yaml
+
+   packages:
+     gcc:
+       external:
+       - spec: gcc@12.2.0 arch=linux-rhel8-skylake
+         prefix: /usr
+         extra_attributes:
+           environment:
+             set:
+               GCC_ROOT: /usr
+       external:
+       - spec: llvm+clang@15.0.0 arch=linux-rhel8-skylake
+         prefix: /usr
+         extra_attributes:
+           paths:
+             cc: /usr/bin/clang-with-suffix
+             cxx: /usr/bin/clang++-with-extra-info
+             fc: /usr/bin/gfortran
+             f77: /usr/bin/gfortran
+           extra_rpaths:
+           - /usr/lib/llvm/
 
 ^^^^^^^^^^^^^^^^^^^^^^^
 Build Your Own Compiler

lib/spack/docs/module_file_support.rst

@@ -10,7 +10,7 @@ Modules (modules.yaml)
 ======================
 
 The use of module systems to manage user environment in a controlled way
-is a common practice at HPC centers that is often embraced also by
+is a common practice at HPC centers that is sometimes embraced also by
 individual programmers on their development machines. To support this
 common practice Spack integrates with `Environment Modules
 <http://modules.sourceforge.net/>`_ and `Lmod
@@ -21,14 +21,38 @@ Modules are one of several ways you can use Spack packages. For other
 options that may fit your use case better, you should also look at
 :ref:`spack load <spack-load>` and :ref:`environments <environments>`.
 
-----------------------------
-Using module files via Spack
-----------------------------
+-----------
+Quick start
+-----------
 
-If you have installed a supported module system you should be able to
-run ``module avail`` to see what module
-files have been installed. Here is sample output of those programs,
-showing lots of installed packages:
+In the current version of Spack, module files are not generated by default. To get started, you
+can generate module files for all currently installed packages by running either
+
+.. code-block:: console
+
+   $ spack module tcl refresh
+
+or
+
+.. code-block:: console
+
+   $ spack module lmod refresh
+
+Spack can also generate module files for all future installations automatically through the
+following configuration:
+
+.. code-block:: console
+
+   $ spack config add modules:default:enable:[tcl]
+
+or
+
+.. code-block:: console
+
+   $ spack config add modules:default:enable:[lmod]
+
+Assuming you have a module system installed, you should now be able to use the ``module`` command
+to interact with them:
 
 .. code-block:: console
 
@@ -65,33 +89,17 @@ scheme used at your site.
 Module file customization
 -------------------------
 
-Module files are generated by post-install hooks after the successful
-installation of a package.
-
-.. note::
-
-   Spack only generates modulefiles when a package is installed. If
-   you attempt to install a package and it is already installed, Spack
-   will not regenerate modulefiles for the package. This may lead to
-   inconsistent modulefiles if the Spack module configuration has
-   changed since the package was installed, either by editing a file
-   or changing scopes or environments.
-
-   Later in this section there is a subsection on :ref:`regenerating
-   modules <cmd-spack-module-refresh>` that will allow you to bring
-   your modules to a consistent state.
-
 The table below summarizes the essential information associated with
 the different file formats that can be generated by Spack:
 
 
-  +-----------------------------+--------------------+-------------------------------+----------------------------------------------+----------------------+
-  |                             | **Hook name**      |  **Default root directory**   | **Default template file**                    | **Compatible tools** |
-  +=============================+====================+===============================+==============================================+======================+
-  |  **Tcl - Non-Hierarchical** | ``tcl``            |  share/spack/modules          | share/spack/templates/modules/modulefile.tcl | Env. Modules/Lmod    |
-  +-----------------------------+--------------------+-------------------------------+----------------------------------------------+----------------------+
-  |  **Lua - Hierarchical**     | ``lmod``           |  share/spack/lmod             | share/spack/templates/modules/modulefile.lua | Lmod                 |
-  +-----------------------------+--------------------+-------------------------------+----------------------------------------------+----------------------+
+  +-----------+--------------+------------------------------+----------------------------------------------+----------------------+
+  |           | Hierarchical |  **Default root directory**  | **Default template file**                    | **Compatible tools** |
+  +===========+==============+==============================+==============================================+======================+
+  |  ``tcl``  | No           | share/spack/modules          | share/spack/templates/modules/modulefile.tcl | Env. Modules/Lmod    |
+  +-----------+--------------+------------------------------+----------------------------------------------+----------------------+
+  |  ``lmod`` | Yes          | share/spack/lmod             | share/spack/templates/modules/modulefile.lua | Lmod                 |
+  +-----------+--------------+------------------------------+----------------------------------------------+----------------------+
 
 
 Spack ships with sensible defaults for the generation of module files, but
@@ -102,7 +110,7 @@ In general you can override or extend the default behavior by:
  2. writing specific rules in the ``modules.yaml`` configuration file
  3. writing your own templates to override or extend the defaults
 
-The former method let you express changes in the run-time environment
+The former method lets you express changes in the run-time environment
 that are needed to use the installed software properly, e.g. injecting variables
 from language interpreters into their extensions. The latter two instead permit to
 fine tune the filesystem layout, content and creation of module files to meet
@@ -110,79 +118,62 @@ site specific conventions.
 
 .. _overide-api-calls-in-package-py:
 
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Override API calls in ``package.py``
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Setting environment variables dynamically in ``package.py``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-There are two methods that you can override in any ``package.py`` to affect the
-content of the module files generated by Spack. The first one:
+There are two methods that you can implement in any ``package.py`` to dynamically affect the
+content of the module files generated by Spack. The most important one is
+``setup_run_environment``, which can be used to set environment variables in the module file that
+depend on the spec:
 
 .. code-block:: python
 
    def setup_run_environment(self, env):
-       pass
+       if self.spec.satisfies("+foo"):
+           env.set("FOO", "bar")
 
-can alter the content of the module file associated with the same package where it is overridden.
-The second method:
+The second, less commonly used, is ``setup_dependent_run_environment(self, env, dependent_spec)``,
+which allows a dependency to set variables in the module file of its dependents. This is typically
+used in packages like ``python``, ``r``, or ``perl`` to prepend the dependent's prefix to the
+search path of the interpreter (``PYTHONPATH``, ``R_LIBS``, ``PERL5LIB`` resp.), so it can locate
+the packages at runtime.
+
+For example, a simplified version of the ``python`` package could look like this:
 
 .. code-block:: python
 
    def setup_dependent_run_environment(self, env, dependent_spec):
-       pass
+       if dependent_spec.package.extends(self.spec):
+           env.prepend_path("PYTHONPATH", dependent_spec.prefix.lib.python)
 
-can instead inject run-time environment modifications in the module files of packages
-that depend on it. In both cases you need to fill ``env`` with the desired
-list of environment modifications.
-
-.. admonition:: The ``r`` package and callback APIs
-
-  An example in which it is crucial to override both methods
-  is given by the ``r`` package. This package installs libraries and headers
-  in non-standard locations and it is possible to prepend the appropriate directory
-  to the corresponding environment variables:
-
-  ================== =================================
-   LD_LIBRARY_PATH    ``self.prefix/rlib/R/lib``
-   PKG_CONFIG_PATH    ``self.prefix/rlib/pkgconfig``
-  ================== =================================
-
-  with the following snippet:
-
-  .. literalinclude:: _spack_root/var/spack/repos/builtin/packages/r/package.py
-     :pyobject: R.setup_run_environment
-
-  The ``r`` package also knows which environment variable should be modified
-  to make language extensions provided by other packages available, and modifies
-  it appropriately in the override of the second method:
-
-  .. literalinclude:: _spack_root/var/spack/repos/builtin/packages/r/package.py
-     :pyobject: R.setup_dependent_run_environment
+and would make any package that ``extends("python")`` have its library directory added to the
+``PYTHONPATH`` environment variable in the module file. It's much more convenient to set this
+variable here, than to repeat it in every Python extension's ``setup_run_environment`` method.
 
 .. _modules-yaml:
 
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-Write a configuration file
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The ``modules.yaml`` config file and module sets
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The configuration files that control module generation behavior
-are named ``modules.yaml``. The default configuration:
+The configuration files that control module generation behavior are named ``modules.yaml``. The
+default configuration looks like this:
 
 .. literalinclude:: _spack_root/etc/spack/defaults/modules.yaml
    :language: yaml
 
-activates the hooks to generate ``tcl`` module files and inspects
-the installation folder of each package for the presence of a set of subdirectories
-(``bin``, ``man``, ``share/man``, etc.). If any is found its full path is prepended
-to the environment variables listed below the folder name.
+You can define one or more **module sets**, each of which can be configured separately with regard
+to install location, naming scheme, inclusion and exclusion, autoloading, et cetera.
 
-Spack modules can be configured for multiple module sets. The default
-module set is named ``default``. All Spack commands which operate on
-modules default to apply the ``default`` module set, but can be
-applied to any module set in the configuration.
+The default module set is aptly named ``default``. All
+:ref:`Spack commands that operate on modules <maintaining-module-files>` apply to the ``default``
+module set, unless another module set is specified explicitly (with the ``--name`` flag).
 
-"""""""""""""""""""""""""
+
+^^^^^^^^^^^^^^^^^^^^^^^^^
 Changing the modules root
-"""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^
 
 As shown in the table above, the default module root for ``lmod`` is
 ``$spack/share/spack/lmod`` and the default root for ``tcl`` is
@@ -224,25 +215,32 @@ location could be confusing to users of your modules. In the next
 section, we will discuss enabling and disabling module types (module
 file generators) for each module set.
 
-""""""""""""""""""""
-Activate other hooks
-""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Automatically generating module files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Any other module file generator shipped with Spack can be activated adding it to the
-list under the ``enable`` key in the module file. Currently the only generator that
-is not active by default is ``lmod``, which produces hierarchical lua module files.
-
-Each module system can then be configured separately. In fact, you should list configuration
-options that affect a particular type of module files under a top level key corresponding
-to the generator being customized:
+Spack can be configured to automatically generate module files as part of package installation.
+This is done by adding the desired module systems to the ``enable`` list.
 
 .. code-block:: yaml
 
    modules:
      default:
        enable:
-         - tcl
-         - lmod
+       - tcl
+       - lmod
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Configuring ``tcl`` and ``lmod`` modules
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can configure the behavior of either module system separately, under a key corresponding to
+the generator being customized:
+
+.. code-block:: yaml
+
+   modules:
+     default:
        tcl:
          # contains environment modules specific customizations
        lmod:
@@ -253,16 +251,82 @@ either change the layout of the module files on the filesystem, or they will aff
 their content. For the latter point it is possible to use anonymous specs
 to fine tune the set of packages on which the modifications should be applied.
 
+.. _autoloading-dependencies:
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Autoloading and hiding dependencies
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A module file should set the variables that are needed for an application to work. But since an
+application often has many dependencies, where should all the environment variables for those be
+set? In Spack the rule is that each package sets the runtime variables that are needed by the
+package itself, and no more. This way, dependencies can be loaded standalone too, and duplication
+of environment variables is avoided.
+
+That means however that if you want to use an application, you need to load the modules for all its
+dependencies. Of course this is not something you would want users to do manually.
+
+Since Spack knows the dependency graph of every package, it can easily generate module files that
+automatically load the modules for its dependencies recursively. It is enabled by default for both
+Lmod and Environment Modules under the ``autoload: direct`` config option. The former system has
+builtin support through the ``depends_on`` function, the latter simply uses a ``module load``
+statement. Both module systems (at least in newer versions) do reference counting, so that if a
+module is loaded by two different modules, it will only be unloaded after the others are.
+
+The ``autoload`` key accepts the values:
+
+  * ``none``: no autoloading
+  * ``run``: autoload direct *run* type dependencies
+  * ``direct``: autoload direct *link and run* type dependencies
+  * ``all``: autoload all dependencies
+
+In case of ``run`` and ``direct``, a ``module load`` triggers a recursive load.
+
+The ``direct`` option is most correct: there are cases where pure link dependencies need to set
+variables for themselves, or need to have variables of their own dependencies set.
+
+In practice however, ``run`` is often sufficient, and may make ``module load`` snappier.
+
+The ``all`` option is discouraged and seldomly used.
+
+A common complaint about autoloading is the large number of modules that are visible to the user.
+Spack has a solution for this as well: ``hide_implicits: true``. This ensures that only those
+packages you've explicitly installed are exposed by ``module avail``, but still allows for
+autoloading of hidden dependencies. Lmod should support hiding implicits in general, while
+Environment Modules requires version 4.7 or higher.
+
+.. note::
+   If supported by your module system, we highly encourage the following configuration that enables
+   autoloading and hiding of implicits. It ensures all runtime variables are set correctly,
+   including those for dependencies, without overwhelming the user with a large number of available
+   modules. Further, it makes it easier to get readable module names without collisions, see the
+   section below on :ref:`modules-projections`.
+
+    .. code-block:: yaml
+  
+        modules:
+          default:
+            tcl:
+              hide_implicits: true
+              all:
+                autoload: direct # or `run`
+            lmod:
+              hide_implicits: true
+              all:
+                autoload: direct # or `run`
+
 .. _anonymous_specs:
 
-""""""""""""""""""""""""""""
-Selection by anonymous specs
-""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Setting environment variables for selected packages in config
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-In the configuration file you can use *anonymous specs* (i.e. specs
-that **are not required to have a root package** and are thus used just
-to express constraints) to apply certain modifications on a selected set
-of the installed software. For instance, in the snippet below:
+In the configuration file you can filter particular specs, and make further changes to the
+environment variables that go into their module files. This is very powerful when you want to avoid
+:ref:`modifying the package itself <overide-api-calls-in-package-py>`, or when you want to set
+certain variables on multiple selected packages at once.
+
+For instance, in the snippet below:
 
 .. code-block:: yaml
 
@@ -305,12 +369,28 @@ the variable ``FOOBAR`` will be unset.
 .. note::
    Order does matter
      The modifications associated with the ``all`` keyword are always evaluated
-     first, no matter where they appear in the configuration file. All the other
-     spec constraints are instead evaluated top to bottom.
+     first, no matter where they appear in the configuration file. All the other changes to
+     environment variables for matching specs are evaluated from top to bottom.
 
-""""""""""""""""""""""""""""""""""""""""""""
+.. warning::
+
+     As general advice, it's often better to set as few unnecessary variables as possible. For
+     example, the following seemingly innocent and potentially useful configuration
+
+     .. code-block:: yaml
+
+        all:
+          environment:
+            set:
+              "{name}_ROOT": "{prefix}"
+
+     sets ``BINUTILS_ROOT`` to its prefix in modules for ``binutils``, which happens to break
+     the ``gcc`` compiler: it uses this variable as its default search path for certain object
+     files and libraries, and by merely setting it, everything fails to link.
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Exclude or include specific module files
-""""""""""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 You can use anonymous specs also to prevent module files from being written or
 to force them to be written. Consider the case where you want to hide from users
@@ -330,14 +410,19 @@ you will prevent the generation of module files for any package that
 is compiled with ``gcc@4.4.7``, with the only exception of any ``gcc``
 or any ``llvm`` installation.
 
+It is safe to combine ``exclude`` and ``autoload``
+:ref:`mentioned above <autoloading-dependencies>`. When ``exclude`` prevents a module file to be
+generated for a dependency, the ``autoload`` feature will simply not generate a statement to load
+it.
+
 
 .. _modules-projections:
 
-"""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Customize the naming of modules
-"""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The names of environment modules generated by spack are not always easy to
+The names of environment modules generated by Spack are not always easy to
 fully comprehend due to the long hash in the name. There are three module
 configuration options to help with that. The first is a global setting to
 adjust the hash length. It can be set anywhere from 0 to 32 and has a default
@@ -353,6 +438,13 @@ shows how to set hash length in the module file names:
        tcl:
          hash_length: 7
 
+.. tip::
+
+   Using ``hide_implicits: true`` (see :ref:`autoloading-dependencies`) vastly reduces the number
+   modules exposed to the user. The hidden modules always contain the hash in their name, and are
+   not influenced by the ``hash_length`` setting. Hidden implicits thus make it easier to use a
+   short hash length or no hash at all, without risking name conflicts.
+
 To help make module names more readable, and to help alleviate name conflicts
 with a short hash, one can use the ``suffixes`` option in the modules
 configuration file. This option will add strings to modules that match a spec.
@@ -365,12 +457,12 @@ For instance, the following config options,
        tcl:
          all:
            suffixes:
-             ^python@2.7.12: 'python-2.7.12'
+             ^python@3.12: 'python-3.12'
              ^openblas: 'openblas'
 
-will add a ``python-2.7.12`` version string to any packages compiled with
-python matching the spec, ``python@2.7.12``. This is useful to know which
-version of python a set of python extensions is associated with. Likewise, the
+will add a ``python-3.12`` version string to any packages compiled with
+Python matching the spec, ``python@3.12``. This is useful to know which
+version of Python a set of Python extensions is associated with. Likewise, the
 ``openblas`` string is attached to any program that has openblas in the spec,
 most likely via the ``+blas`` variant specification.
 
@@ -468,41 +560,11 @@ that are already in the Lmod hierarchy.
    For hierarchies that are deeper than three layers ``lmod spider`` may have some issues.
    See `this discussion on the Lmod project <https://github.com/TACC/Lmod/issues/114>`_.
 
-""""""""""""""""""""""
-Select default modules
-""""""""""""""""""""""
-
-By default, when multiple modules of the same name share a directory,
-the highest version number will be the default module. This behavior
-of the ``module`` command can be overridden with a symlink named
-``default`` to the desired default module. If you wish to configure
-default modules with Spack, add a ``defaults`` key to your modules
-configuration:
-
-.. code-block:: yaml
-
-  modules:
-    my-module-set:
-      tcl:
-        defaults:
-        - gcc@10.2.1
-        - hdf5@1.2.10+mpi+hl%gcc
-
-These defaults may be arbitrarily specific. For any package that
-satisfies a default, Spack will generate the module file in the
-appropriate path, and will generate a default symlink to the module
-file as well.
-
-.. warning::
-  If Spack is configured to generate multiple default packages in the
-  same directory, the last modulefile to be generated will be the
-  default module.
-
 .. _customize-env-modifications:
 
-"""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Customize environment modifications
-"""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 You can control which prefixes in a Spack package are added to
 environment variables with the ``prefix_inspections`` section; this
@@ -600,9 +662,9 @@ stack to users who are likely to inspect the modules to find full
 paths to software, when it is desirable to present the users with a
 simpler set of paths than those generated by the Spack install tree.
 
-""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Filter out environment modifications
-""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Modifications to certain environment variables in module files are there by
 default, for instance because they are generated by prefix inspections.
@@ -622,49 +684,37 @@ do so by using the ``exclude_env_vars``:
 The configuration above will generate module files that will not contain
 modifications to either ``CPATH`` or ``LIBRARY_PATH``.
 
+^^^^^^^^^^^^^^^^^^^^^^
+Select default modules
+^^^^^^^^^^^^^^^^^^^^^^
 
-.. _autoloading-dependencies:
-
-"""""""""""""""""""""
-Autoload dependencies
-"""""""""""""""""""""
-
-Often it is required for a module to have its (transient) dependencies loaded as well.
-One example where this is useful is when one package needs to use executables provided
-by its dependency; when the dependency is autoloaded, the executable will be in the
-PATH. Similarly for scripting languages such as Python, packages and their dependencies
-have to be loaded together.
-
-Autoloading is enabled by default for Lmod and Environment Modules. The former
-has builtin support for through the ``depends_on`` function. The latter uses
-``module load`` statement to load and track dependencies.
-
-Autoloading can also be enabled conditionally:
+By default, when multiple modules of the same name share a directory,
+the highest version number will be the default module. This behavior
+of the ``module`` command can be overridden with a symlink named
+``default`` to the desired default module. If you wish to configure
+default modules with Spack, add a ``defaults`` key to your modules
+configuration:
 
 .. code-block:: yaml
 
-    modules:
-      default:
-        tcl:
-          all:
-            autoload: none
-          ^python:
-            autoload: direct
+  modules:
+    my-module-set:
+      tcl:
+        defaults:
+        - gcc@10.2.1
+        - hdf5@1.2.10+mpi+hl%gcc
 
-The configuration file above will produce module files that will
-load their direct dependencies if the package installed depends on ``python``.
-The allowed values for the ``autoload`` statement are either ``none``,
-``direct`` or ``all``.
+These defaults may be arbitrarily specific. For any package that
+satisfies a default, Spack will generate the module file in the
+appropriate path, and will generate a default symlink to the module
+file as well.
 
-.. note::
-  Tcl prerequisites
-     In the ``tcl`` section of the configuration file it is possible to use
-     the ``prerequisites`` directive that accepts the same values as
-     ``autoload``. It will produce module files that have a ``prereq``
-     statement, which autoloads dependencies on Environment Modules when its
-     ``auto_handling`` configuration option is enabled. If Environment Modules
-     is installed with Spack, ``auto_handling`` is enabled by default starting
-     version 4.2. Otherwise it is enabled by default since version 5.0.
+.. warning::
+  If Spack is configured to generate multiple default packages in the
+  same directory, the last modulefile to be generated will be the
+  default module.
+
+.. _maintaining-module-files:
 
 ------------------------
 Maintaining Module Files

lib/spack/docs/packages_yaml.rst

@@ -647,6 +647,8 @@ manually placed files within the install prefix are owned by the
 assigned group. If no group is assigned, Spack will allow the OS
 default behavior to go as expected.
 
+.. _assigning-package-attributes:
+
 ----------------------------
 Assigning Package Attributes
 ----------------------------

lib/spack/docs/repositories.rst

@@ -17,7 +17,7 @@ experimental software separately from the built-in repository. Spack
 allows you to configure local repositories using either the
 ``repos.yaml`` or the ``spack repo`` command.
 
-A package repository a directory structured like this::
+A package repository is a directory structured like this::
 
   repo/
       repo.yaml

lib/spack/docs/requirements.txt

@@ -2,12 +2,12 @@ sphinx==7.2.6
 sphinxcontrib-programoutput==0.17
 sphinx_design==0.5.0
 sphinx-rtd-theme==2.0.0
-python-levenshtein==0.24.0
+python-levenshtein==0.25.0
 docutils==0.20.1
 pygments==2.17.2
-urllib3==2.2.0
-pytest==8.0.0
+urllib3==2.2.1
+pytest==8.1.1
 isort==5.13.2
-black==24.1.1
+black==24.3.0
 flake8==7.0.0
-mypy==1.8.0
+mypy==1.9.0

lib/spack/external/__init__.py

@@ -18,7 +18,7 @@
 
 * Homepage: https://pypi.python.org/pypi/archspec
 * Usage: Labeling, comparison and detection of microarchitectures
-* Version: 0.2.2 (commit 1dc58a5776dd77e6fc6e4ba5626af5b1fb24996e)
+* Version: 0.2.3 (commit 7b8fe60b69e2861e7dac104bc1c183decfcd3daf)
 
 astunparse
 ----------------

lib/spack/external/archspec/__init__.py

@@ -1,2 +1,3 @@
 """Init file to avoid namespace packages"""
-__version__ = "0.2.2"
+
+__version__ = "0.2.3"

lib/spack/external/archspec/__main__.py

@@ -3,6 +3,7 @@
 """
 
 import sys
+
 from .cli import main
 
 sys.exit(main())

lib/spack/external/archspec/cli.py

@@ -46,7 +46,11 @@ def _make_parser() -> argparse.ArgumentParser:
 
 def cpu() -> int:
     """Run the `archspec cpu` subcommand."""
-    print(archspec.cpu.host())
+    try:
+        print(archspec.cpu.host())
+    except FileNotFoundError as exc:
+        print(exc)
+        return 1
     return 0
 
 

lib/spack/external/archspec/cpu/__init__.py

@@ -5,10 +5,14 @@
 """The "cpu" package permits to query and compare different
 CPU microarchitectures.
 """
-from .microarchitecture import Microarchitecture, UnsupportedMicroarchitecture
-from .microarchitecture import TARGETS, generic_microarchitecture
-from .microarchitecture import version_components
 from .detect import host
+from .microarchitecture import (
+    TARGETS,
+    Microarchitecture,
+    UnsupportedMicroarchitecture,
+    generic_microarchitecture,
+    version_components,
+)
 
 __all__ = [
     "Microarchitecture",

lib/spack/external/archspec/cpu/detect.py

@@ -4,15 +4,17 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Detection of CPU microarchitectures"""
 import collections
-import functools
 import os
 import platform
 import re
+import struct
 import subprocess
 import warnings
+from typing import Dict, List, Optional, Set, Tuple, Union
 
-from .microarchitecture import generic_microarchitecture, TARGETS
-from .schema import TARGETS_JSON
+from ..vendor.cpuid.cpuid import CPUID
+from .microarchitecture import TARGETS, Microarchitecture, generic_microarchitecture
+from .schema import CPUID_JSON, TARGETS_JSON
 
 #: Mapping from operating systems to chain of commands
 #: to obtain a dictionary of raw info on the current cpu
@@ -22,43 +24,46 @@
 #: functions checking the compatibility of the host with a given target
 COMPATIBILITY_CHECKS = {}
 
+# Constants for commonly used architectures
+X86_64 = "x86_64"
+AARCH64 = "aarch64"
+PPC64LE = "ppc64le"
+PPC64 = "ppc64"
+RISCV64 = "riscv64"
 
-def info_dict(operating_system):
-    """Decorator to mark functions that are meant to return raw info on
-    the current cpu.
+
+def detection(operating_system: str):
+    """Decorator to mark functions that are meant to return partial information on the current cpu.
 
     Args:
-        operating_system (str or tuple): operating system for which the marked
-            function is a viable factory of raw info dictionaries.
+        operating_system: operating system where this function can be used.
     """
 
     def decorator(factory):
         INFO_FACTORY[operating_system].append(factory)
-
-        @functools.wraps(factory)
-        def _impl():
-            info = factory()
-
-            # Check that info contains a few mandatory fields
-            msg = 'field "{0}" is missing from raw info dictionary'
-            assert "vendor_id" in info, msg.format("vendor_id")
-            assert "flags" in info, msg.format("flags")
-            assert "model" in info, msg.format("model")
-            assert "model_name" in info, msg.format("model_name")
-
-            return info
-
-        return _impl
+        return factory
 
     return decorator
 
 
-@info_dict(operating_system="Linux")
-def proc_cpuinfo():
-    """Returns a raw info dictionary by parsing the first entry of
-    ``/proc/cpuinfo``
-    """
-    info = {}
+def partial_uarch(
+    name: str = "", vendor: str = "", features: Optional[Set[str]] = None, generation: int = 0
+) -> Microarchitecture:
+    """Construct a partial microarchitecture, from information gathered during system scan."""
+    return Microarchitecture(
+        name=name,
+        parents=[],
+        vendor=vendor,
+        features=features or set(),
+        compilers={},
+        generation=generation,
+    )
+
+
+@detection(operating_system="Linux")
+def proc_cpuinfo() -> Microarchitecture:
+    """Returns a partial Microarchitecture, obtained from scanning ``/proc/cpuinfo``"""
+    data = {}
     with open("/proc/cpuinfo") as file:  # pylint: disable=unspecified-encoding
         for line in file:
             key, separator, value = line.partition(":")
@@ -70,11 +75,96 @@ def proc_cpuinfo():
             #
             # we are on a blank line separating two cpus. Exit early as
             # we want to read just the first entry in /proc/cpuinfo
-            if separator != ":" and info:
+            if separator != ":" and data:
                 break
 
-            info[key.strip()] = value.strip()
-    return info
+            data[key.strip()] = value.strip()
+
+    architecture = _machine()
+    if architecture == X86_64:
+        return partial_uarch(
+            vendor=data.get("vendor_id", "generic"), features=_feature_set(data, key="flags")
+        )
+
+    if architecture == AARCH64:
+        return partial_uarch(
+            vendor=_canonicalize_aarch64_vendor(data),
+            features=_feature_set(data, key="Features"),
+        )
+
+    if architecture in (PPC64LE, PPC64):
+        generation_match = re.search(r"POWER(\d+)", data.get("cpu", ""))
+        try:
+            generation = int(generation_match.group(1))
+        except AttributeError:
+            # There might be no match under emulated environments. For instance
+            # emulating a ppc64le with QEMU and Docker still reports the host
+            # /proc/cpuinfo and not a Power
+            generation = 0
+        return partial_uarch(generation=generation)
+
+    if architecture == RISCV64:
+        if data.get("uarch") == "sifive,u74-mc":
+            data["uarch"] = "u74mc"
+        return partial_uarch(name=data.get("uarch", RISCV64))
+
+    return generic_microarchitecture(architecture)
+
+
+class CpuidInfoCollector:
+    """Collects the information we need on the host CPU from cpuid"""
+
+    # pylint: disable=too-few-public-methods
+    def __init__(self):
+        self.cpuid = CPUID()
+
+        registers = self.cpuid.registers_for(**CPUID_JSON["vendor"]["input"])
+        self.highest_basic_support = registers.eax
+        self.vendor = struct.pack("III", registers.ebx, registers.edx, registers.ecx).decode(
+            "utf-8"
+        )
+
+        registers = self.cpuid.registers_for(**CPUID_JSON["highest_extension_support"]["input"])
+        self.highest_extension_support = registers.eax
+
+        self.features = self._features()
+
+    def _features(self):
+        result = set()
+
+        def check_features(data):
+            registers = self.cpuid.registers_for(**data["input"])
+            for feature_check in data["bits"]:
+                current = getattr(registers, feature_check["register"])
+                if self._is_bit_set(current, feature_check["bit"]):
+                    result.add(feature_check["name"])
+
+        for call_data in CPUID_JSON["flags"]:
+            if call_data["input"]["eax"] > self.highest_basic_support:
+                continue
+            check_features(call_data)
+
+        for call_data in CPUID_JSON["extension-flags"]:
+            if call_data["input"]["eax"] > self.highest_extension_support:
+                continue
+            check_features(call_data)
+
+        return result
+
+    def _is_bit_set(self, register: int, bit: int) -> bool:
+        mask = 1 << bit
+        return register & mask > 0
+
+
+@detection(operating_system="Windows")
+def cpuid_info():
+    """Returns a partial Microarchitecture, obtained from running the cpuid instruction"""
+    architecture = _machine()
+    if architecture == X86_64:
+        data = CpuidInfoCollector()
+        return partial_uarch(vendor=data.vendor, features=data.features)
+
+    return generic_microarchitecture(architecture)
 
 
 def _check_output(args, env):
@@ -83,14 +173,25 @@ def _check_output(args, env):
     return str(output.decode("utf-8"))
 
 
+WINDOWS_MAPPING = {
+    "AMD64": "x86_64",
+    "ARM64": "aarch64",
+}
+
+
 def _machine():
-    """ "Return the machine architecture we are on"""
+    """Return the machine architecture we are on"""
     operating_system = platform.system()
 
-    # If we are not on Darwin, trust what Python tells us
-    if operating_system != "Darwin":
+    # If we are not on Darwin or Windows, trust what Python tells us
+    if operating_system not in ("Darwin", "Windows"):
         return platform.machine()
 
+    # Normalize windows specific names
+    if operating_system == "Windows":
+        platform_machine = platform.machine()
+        return WINDOWS_MAPPING.get(platform_machine, platform_machine)
+
     # On Darwin it might happen that we are on M1, but using an interpreter
     # built for x86_64. In that case "platform.machine() == 'x86_64'", so we
     # need to fix that.
@@ -103,54 +204,47 @@ def _machine():
     if "Apple" in output:
         # Note that a native Python interpreter on Apple M1 would return
         # "arm64" instead of "aarch64". Here we normalize to the latter.
-        return "aarch64"
+        return AARCH64
 
-    return "x86_64"
+    return X86_64
 
 
-@info_dict(operating_system="Darwin")
-def sysctl_info_dict():
+@detection(operating_system="Darwin")
+def sysctl_info() -> Microarchitecture:
     """Returns a raw info dictionary parsing the output of sysctl."""
     child_environment = _ensure_bin_usrbin_in_path()
 
-    def sysctl(*args):
+    def sysctl(*args: str) -> str:
         return _check_output(["sysctl"] + list(args), env=child_environment).strip()
 
-    if _machine() == "x86_64":
-        flags = (
-            sysctl("-n", "machdep.cpu.features").lower()
-            + " "
-            + sysctl("-n", "machdep.cpu.leaf7_features").lower()
+    if _machine() == X86_64:
+        features = (
+            f'{sysctl("-n", "machdep.cpu.features").lower()} '
+            f'{sysctl("-n", "machdep.cpu.leaf7_features").lower()}'
         )
-        info = {
-            "vendor_id": sysctl("-n", "machdep.cpu.vendor"),
-            "flags": flags,
-            "model": sysctl("-n", "machdep.cpu.model"),
-            "model name": sysctl("-n", "machdep.cpu.brand_string"),
-        }
-    else:
-        model = "unknown"
-        model_str = sysctl("-n", "machdep.cpu.brand_string").lower()
-        if "m2" in model_str:
-            model = "m2"
-        elif "m1" in model_str:
-            model = "m1"
-        elif "apple" in model_str:
-            model = "m1"
+        features = set(features.split())
 
-        info = {
-            "vendor_id": "Apple",
-            "flags": [],
-            "model": model,
-            "CPU implementer": "Apple",
-            "model name": sysctl("-n", "machdep.cpu.brand_string"),
-        }
-    return info
+        # Flags detected on Darwin turned to their linux counterpart
+        for darwin_flag, linux_flag in TARGETS_JSON["conversions"]["darwin_flags"].items():
+            if darwin_flag in features:
+                features.update(linux_flag.split())
+
+        return partial_uarch(vendor=sysctl("-n", "machdep.cpu.vendor"), features=features)
+
+    model = "unknown"
+    model_str = sysctl("-n", "machdep.cpu.brand_string").lower()
+    if "m2" in model_str:
+        model = "m2"
+    elif "m1" in model_str:
+        model = "m1"
+    elif "apple" in model_str:
+        model = "m1"
+
+    return partial_uarch(name=model, vendor="Apple")
 
 
 def _ensure_bin_usrbin_in_path():
-    # Make sure that /sbin and /usr/sbin are in PATH as sysctl is
-    # usually found there
+    # Make sure that /sbin and /usr/sbin are in PATH as sysctl is usually found there
     child_environment = dict(os.environ.items())
     search_paths = child_environment.get("PATH", "").split(os.pathsep)
     for additional_path in ("/sbin", "/usr/sbin"):
@@ -160,22 +254,10 @@ def _ensure_bin_usrbin_in_path():
     return child_environment
 
 
-def adjust_raw_flags(info):
-    """Adjust the flags detected on the system to homogenize
-    slightly different representations.
-    """
-    # Flags detected on Darwin turned to their linux counterpart
-    flags = info.get("flags", [])
-    d2l = TARGETS_JSON["conversions"]["darwin_flags"]
-    for darwin_flag, linux_flag in d2l.items():
-        if darwin_flag in flags:
-            info["flags"] += " " + linux_flag
-
-
-def adjust_raw_vendor(info):
-    """Adjust the vendor field to make it human readable"""
-    if "CPU implementer" not in info:
-        return
+def _canonicalize_aarch64_vendor(data: Dict[str, str]) -> str:
+    """Adjust the vendor field to make it human-readable"""
+    if "CPU implementer" not in data:
+        return "generic"
 
     # Mapping numeric codes to vendor (ARM). This list is a merge from
     # different sources:
@@ -185,43 +267,37 @@ def adjust_raw_vendor(info):
     # https://github.com/gcc-mirror/gcc/blob/master/gcc/config/aarch64/aarch64-cores.def
     # https://patchwork.kernel.org/patch/10524949/
     arm_vendors = TARGETS_JSON["conversions"]["arm_vendors"]
-    arm_code = info["CPU implementer"]
-    if arm_code in arm_vendors:
-        info["CPU implementer"] = arm_vendors[arm_code]
+    arm_code = data["CPU implementer"]
+    return arm_vendors.get(arm_code, arm_code)
 
 
-def raw_info_dictionary():
-    """Returns a dictionary with information on the cpu of the current host.
+def _feature_set(data: Dict[str, str], key: str) -> Set[str]:
+    return set(data.get(key, "").split())
 
-    This function calls all the viable factories one after the other until
-    there's one that is able to produce the requested information.
+
+def detected_info() -> Microarchitecture:
+    """Returns a partial Microarchitecture with information on the CPU of the current host.
+
+    This function calls all the viable factories one after the other until there's one that is
+    able to produce the requested information. Falls-back to a generic microarchitecture, if none
+    of the calls succeed.
     """
     # pylint: disable=broad-except
-    info = {}
     for factory in INFO_FACTORY[platform.system()]:
         try:
-            info = factory()
+            return factory()
         except Exception as exc:
             warnings.warn(str(exc))
 
-        if info:
-            adjust_raw_flags(info)
-            adjust_raw_vendor(info)
-            break
-
-    return info
+    return generic_microarchitecture(_machine())
 
 
-def compatible_microarchitectures(info):
-    """Returns an unordered list of known micro-architectures that are
-    compatible with the info dictionary passed as argument.
-
-    Args:
-        info (dict): dictionary containing information on the host cpu
+def compatible_microarchitectures(info: Microarchitecture) -> List[Microarchitecture]:
+    """Returns an unordered list of known micro-architectures that are compatible with the
+    partial Microarchitecture passed as input.
     """
     architecture_family = _machine()
-    # If a tester is not registered, be conservative and assume no known
-    # target is compatible with the host
+    # If a tester is not registered, assume no known target is compatible with the host
     tester = COMPATIBILITY_CHECKS.get(architecture_family, lambda x, y: False)
     return [x for x in TARGETS.values() if tester(info, x)] or [
         generic_microarchitecture(architecture_family)
@@ -230,8 +306,8 @@ def compatible_microarchitectures(info):
 
 def host():
     """Detects the host micro-architecture and returns it."""
-    # Retrieve a dictionary with raw information on the host's cpu
-    info = raw_info_dictionary()
+    # Retrieve information on the host's cpu
+    info = detected_info()
 
     # Get a list of possible candidates for this micro-architecture
     candidates = compatible_microarchitectures(info)
@@ -258,16 +334,15 @@ def sorting_fn(item):
     return max(candidates, key=sorting_fn)
 
 
-def compatibility_check(architecture_family):
+def compatibility_check(architecture_family: Union[str, Tuple[str, ...]]):
     """Decorator to register a function as a proper compatibility check.
 
-    A compatibility check function takes the raw info dictionary as a first
-    argument and an arbitrary target as the second argument. It returns True
-    if the target is compatible with the info dictionary, False otherwise.
+    A compatibility check function takes a partial Microarchitecture object as a first argument,
+    and an arbitrary target Microarchitecture as the second argument. It returns True if the
+    target is compatible with first argument, False otherwise.
 
     Args:
-        architecture_family (str or tuple): architecture family for which
-            this test can be used, e.g. x86_64 or ppc64le etc.
+        architecture_family: architecture family for which this test can be used
     """
     # Turn the argument into something iterable
     if isinstance(architecture_family, str):
@@ -280,86 +355,57 @@ def decorator(func):
     return decorator
 
 
-@compatibility_check(architecture_family=("ppc64le", "ppc64"))
+@compatibility_check(architecture_family=(PPC64LE, PPC64))
 def compatibility_check_for_power(info, target):
     """Compatibility check for PPC64 and PPC64LE architectures."""
-    basename = platform.machine()
-    generation_match = re.search(r"POWER(\d+)", info.get("cpu", ""))
-    try:
-        generation = int(generation_match.group(1))
-    except AttributeError:
-        # There might be no match under emulated environments. For instance
-        # emulating a ppc64le with QEMU and Docker still reports the host
-        # /proc/cpuinfo and not a Power
-        generation = 0
-
     # We can use a target if it descends from our machine type and our
     # generation (9 for POWER9, etc) is at least its generation.
-    arch_root = TARGETS[basename]
+    arch_root = TARGETS[_machine()]
     return (
         target == arch_root or arch_root in target.ancestors
-    ) and target.generation <= generation
+    ) and target.generation <= info.generation
 
 
-@compatibility_check(architecture_family="x86_64")
+@compatibility_check(architecture_family=X86_64)
 def compatibility_check_for_x86_64(info, target):
     """Compatibility check for x86_64 architectures."""
-    basename = "x86_64"
-    vendor = info.get("vendor_id", "generic")
-    features = set(info.get("flags", "").split())
-
     # We can use a target if it descends from our machine type, is from our
     # vendor, and we have all of its features
-    arch_root = TARGETS[basename]
+    arch_root = TARGETS[X86_64]
     return (
         (target == arch_root or arch_root in target.ancestors)
-        and target.vendor in (vendor, "generic")
-        and target.features.issubset(features)
+        and target.vendor in (info.vendor, "generic")
+        and target.features.issubset(info.features)
     )
 
 
-@compatibility_check(architecture_family="aarch64")
+@compatibility_check(architecture_family=AARCH64)
 def compatibility_check_for_aarch64(info, target):
     """Compatibility check for AARCH64 architectures."""
-    basename = "aarch64"
-    features = set(info.get("Features", "").split())
-    vendor = info.get("CPU implementer", "generic")
-
-    # At the moment it's not clear how to detect compatibility with
+    # At the moment, it's not clear how to detect compatibility with
     # a specific version of the architecture
-    if target.vendor == "generic" and target.name != "aarch64":
+    if target.vendor == "generic" and target.name != AARCH64:
         return False
 
-    arch_root = TARGETS[basename]
+    arch_root = TARGETS[AARCH64]
     arch_root_and_vendor = arch_root == target.family and target.vendor in (
-        vendor,
+        info.vendor,
         "generic",
     )
 
     # On macOS it seems impossible to get all the CPU features
     # with syctl info, but for ARM we can get the exact model
     if platform.system() == "Darwin":
-        model_key = info.get("model", basename)
-        model = TARGETS[model_key]
+        model = TARGETS[info.name]
         return arch_root_and_vendor and (target == model or target in model.ancestors)
 
-    return arch_root_and_vendor and target.features.issubset(features)
+    return arch_root_and_vendor and target.features.issubset(info.features)
 
 
-@compatibility_check(architecture_family="riscv64")
+@compatibility_check(architecture_family=RISCV64)
 def compatibility_check_for_riscv64(info, target):
     """Compatibility check for riscv64 architectures."""
-    basename = "riscv64"
-    uarch = info.get("uarch")
-
-    # sifive unmatched board
-    if uarch == "sifive,u74-mc":
-        uarch = "u74mc"
-    # catch-all for unknown uarchs
-    else:
-        uarch = "riscv64"
-
-    arch_root = TARGETS[basename]
+    arch_root = TARGETS[RISCV64]
     return (target == arch_root or arch_root in target.ancestors) and (
-        target == uarch or target.vendor == "generic"
+        target.name == info.name or target.vendor == "generic"
     )

lib/spack/external/archspec/cpu/microarchitecture.py

@@ -13,6 +13,7 @@
 import archspec
 import archspec.cpu.alias
 import archspec.cpu.schema
+
 from .alias import FEATURE_ALIASES
 from .schema import LazyDictionary
 
@@ -47,7 +48,7 @@ class Microarchitecture:
             which has "broadwell" as a parent, supports running binaries
             optimized for "broadwell".
         vendor (str): vendor of the micro-architecture
-        features (list of str): supported CPU flags. Note that the semantic
+        features (set of str): supported CPU flags. Note that the semantic
             of the flags in this field might vary among architectures, if
             at all present. For instance x86_64 processors will list all
             the flags supported by a given CPU while Arm processors will
@@ -180,24 +181,28 @@ def generic(self):
         generics = [x for x in [self] + self.ancestors if x.vendor == "generic"]
         return max(generics, key=lambda x: len(x.ancestors))
 
-    def to_dict(self, return_list_of_items=False):
-        """Returns a dictionary representation of this object.
+    def to_dict(self):
+        """Returns a dictionary representation of this object."""
+        return {
+            "name": str(self.name),
+            "vendor": str(self.vendor),
+            "features": sorted(str(x) for x in self.features),
+            "generation": self.generation,
+            "parents": [str(x) for x in self.parents],
+            "compilers": self.compilers,
+        }
 
-        Args:
-            return_list_of_items (bool): if True returns an ordered list of
-                items instead of the dictionary
-        """
-        list_of_items = [
-            ("name", str(self.name)),
-            ("vendor", str(self.vendor)),
-            ("features", sorted(str(x) for x in self.features)),
-            ("generation", self.generation),
-            ("parents", [str(x) for x in self.parents]),
-        ]
-        if return_list_of_items:
-            return list_of_items
-
-        return dict(list_of_items)
+    @staticmethod
+    def from_dict(data) -> "Microarchitecture":
+        """Construct a microarchitecture from a dictionary representation."""
+        return Microarchitecture(
+            name=data["name"],
+            parents=[TARGETS[x] for x in data["parents"]],
+            vendor=data["vendor"],
+            features=set(data["features"]),
+            compilers=data.get("compilers", {}),
+            generation=data.get("generation", 0),
+        )
 
     def optimization_flags(self, compiler, version):
         """Returns a string containing the optimization flags that needs
@@ -271,9 +276,7 @@ def tuplify(ver):
                 flags = flags_fmt.format(**compiler_entry)
                 return flags
 
-        msg = (
-            "cannot produce optimized binary for micro-architecture '{0}' with {1}@{2}"
-        )
+        msg = "cannot produce optimized binary for micro-architecture '{0}' with {1}@{2}"
         if compiler_info:
             versions = [x["versions"] for x in compiler_info]
             msg += f' [supported compiler versions are {", ".join(versions)}]'
@@ -289,9 +292,7 @@ def generic_microarchitecture(name):
     Args:
         name (str): name of the micro-architecture
     """
-    return Microarchitecture(
-        name, parents=[], vendor="generic", features=[], compilers={}
-    )
+    return Microarchitecture(name, parents=[], vendor="generic", features=[], compilers={})
 
 
 def version_components(version):
@@ -345,9 +346,7 @@ def fill_target_from_dict(name, data, targets):
         compilers = values.get("compilers", {})
         generation = values.get("generation", 0)
 
-        targets[name] = Microarchitecture(
-            name, parents, vendor, features, compilers, generation
-        )
+        targets[name] = Microarchitecture(name, parents, vendor, features, compilers, generation)
 
     known_targets = {}
     data = archspec.cpu.schema.TARGETS_JSON["microarchitectures"]

lib/spack/external/archspec/cpu/schema.py

@@ -7,7 +7,9 @@
 """
 import collections.abc
 import json
-import os.path
+import os
+import pathlib
+from typing import Tuple
 
 
 class LazyDictionary(collections.abc.MutableMapping):
@@ -46,21 +48,65 @@ def __len__(self):
         return len(self.data)
 
 
-def _load_json_file(json_file):
-    json_dir = os.path.join(os.path.dirname(__file__), "..", "json", "cpu")
-    json_dir = os.path.abspath(json_dir)
+#: Environment variable that might point to a directory with a user defined JSON file
+DIR_FROM_ENVIRONMENT = "ARCHSPEC_CPU_DIR"
 
-    def _factory():
-        filename = os.path.join(json_dir, json_file)
-        with open(filename, "r", encoding="utf-8") as file:
-            return json.load(file)
+#: Environment variable that might point to a directory with extensions to JSON files
+EXTENSION_DIR_FROM_ENVIRONMENT = "ARCHSPEC_EXTENSION_CPU_DIR"
 
-    return _factory
+
+def _json_file(filename: str, allow_custom: bool = False) -> Tuple[pathlib.Path, pathlib.Path]:
+    """Given a filename, returns the absolute path for the main JSON file, and an
+    optional absolute path for an extension JSON file.
+
+    Args:
+        filename: filename for the JSON file
+        allow_custom: if True, allows overriding the location  where the file resides
+    """
+    json_dir = pathlib.Path(__file__).parent / ".." / "json" / "cpu"
+    if allow_custom and DIR_FROM_ENVIRONMENT in os.environ:
+        json_dir = pathlib.Path(os.environ[DIR_FROM_ENVIRONMENT])
+    json_dir = json_dir.absolute()
+    json_file = json_dir / filename
+
+    extension_file = None
+    if allow_custom and EXTENSION_DIR_FROM_ENVIRONMENT in os.environ:
+        extension_dir = pathlib.Path(os.environ[EXTENSION_DIR_FROM_ENVIRONMENT])
+        extension_dir.absolute()
+        extension_file = extension_dir / filename
+
+    return json_file, extension_file
+
+
+def _load(json_file: pathlib.Path, extension_file: pathlib.Path):
+    with open(json_file, "r", encoding="utf-8") as file:
+        data = json.load(file)
+
+    if not extension_file or not extension_file.exists():
+        return data
+
+    with open(extension_file, "r", encoding="utf-8") as file:
+        extension_data = json.load(file)
+
+    top_level_sections = list(data.keys())
+    for key in top_level_sections:
+        if key not in extension_data:
+            continue
+
+        data[key].update(extension_data[key])
+
+    return data
 
 
 #: In memory representation of the data in microarchitectures.json,
 #: loaded on first access
-TARGETS_JSON = LazyDictionary(_load_json_file("microarchitectures.json"))
+TARGETS_JSON = LazyDictionary(_load, *_json_file("microarchitectures.json", allow_custom=True))
 
 #: JSON schema for microarchitectures.json, loaded on first access
-SCHEMA = LazyDictionary(_load_json_file("microarchitectures_schema.json"))
+TARGETS_JSON_SCHEMA = LazyDictionary(_load, *_json_file("microarchitectures_schema.json"))
+
+#: Information on how to call 'cpuid' to get information on the HOST CPU
+CPUID_JSON = LazyDictionary(_load, *_json_file("cpuid.json", allow_custom=True))
+
+#: JSON schema for cpuid.json, loaded on first access
+CPUID_JSON_SCHEMA = LazyDictionary(_load, *_json_file("cpuid_schema.json"))

lib/spack/external/archspec/json/README.md

@@ -9,11 +9,11 @@ language specific APIs.
 
 Currently the repository contains the following JSON files:
 ```console
-.
-├── COPYRIGHT
-└── cpu
-    ├── microarchitectures.json         # Contains information on CPU microarchitectures
-    └── microarchitectures_schema.json  # Schema for the file above
+cpu/
+├── cpuid.json                     # Contains information on CPUID calls to retrieve vendor and features on x86_64
+├── cpuid_schema.json              # Schema for the file above
+├── microarchitectures.json        # Contains information on CPU microarchitectures
+└── microarchitectures_schema.json # Schema for the file above
  ```
 
 

lib/spack/external/archspec/json/cpu/cpuid_schema.json

@@ -0,0 +1,134 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Schema for microarchitecture definitions and feature aliases",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "vendor": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "description": {
+          "type": "string"
+        },
+        "input": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "eax": {
+              "type": "integer"
+            },
+            "ecx": {
+              "type": "integer"
+            }
+          }
+        }
+      }
+    },
+    "highest_extension_support": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "description": {
+          "type": "string"
+        },
+        "input": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "eax": {
+              "type": "integer"
+            },
+            "ecx": {
+              "type": "integer"
+            }
+          }
+        }
+      }
+    },
+    "flags": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "description": {
+            "type": "string"
+          },
+          "input": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "eax": {
+                "type": "integer"
+              },
+              "ecx": {
+                "type": "integer"
+              }
+            }
+          },
+          "bits": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "register": {
+                  "type": "string"
+                },
+                "bit": {
+                  "type": "integer"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "extension-flags": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "description": {
+            "type": "string"
+          },
+          "input": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "eax": {
+                "type": "integer"
+              },
+              "ecx": {
+                "type": "integer"
+              }
+            }
+          },
+          "bits": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "register": {
+                  "type": "string"
+                },
+                "bit": {
+                  "type": "integer"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

lib/spack/external/archspec/vendor/cpuid/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Anders Høst
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

lib/spack/external/archspec/vendor/cpuid/README.md

@@ -0,0 +1,76 @@
+cpuid.py
+========
+
+Now, this is silly!
+
+Pure Python library for accessing information about x86 processors
+by querying the [CPUID](http://en.wikipedia.org/wiki/CPUID)
+instruction. Well, not exactly pure Python...
+
+It works by allocating a small piece of virtual memory, copying
+a raw x86 function to that memory, giving the memory execute
+permissions and then calling the memory as a function. The injected
+function executes the CPUID instruction and copies the result back
+to a ctypes.Structure where is can be read by Python.
+
+It should work fine on both 32 and 64 bit versions of Windows and Linux
+running x86 processors. Apple OS X and other BSD systems should also work,
+not tested though...
+
+
+Why?
+----
+For poops and giggles. Plus, having access to a low-level feature
+without having to compile a C wrapper is pretty neat.
+
+
+Examples
+--------
+Getting info with eax=0:
+
+    import cpuid
+
+    q = cpuid.CPUID()
+    eax, ebx, ecx, edx = q(0)
+
+Running the files:
+
+    $ python example.py 
+    Vendor ID : GenuineIntel
+    CPU name  : Intel(R) Xeon(R) CPU           W3550  @ 3.07GHz
+    
+    Vector instructions supported:
+    SSE       : Yes
+    SSE2      : Yes
+    SSE3      : Yes
+    SSSE3     : Yes
+    SSE4.1    : Yes
+    SSE4.2    : Yes
+    SSE4a     : --
+    AVX       : --
+    AVX2      : --
+    
+    $ python cpuid.py
+    CPUID    A        B        C        D
+    00000000 0000000b 756e6547 6c65746e 49656e69
+    00000001 000106a5 00100800 009ce3bd bfebfbff
+    00000002 55035a01 00f0b2e4 00000000 09ca212c
+    00000003 00000000 00000000 00000000 00000000
+    00000004 00000000 00000000 00000000 00000000
+    00000005 00000040 00000040 00000003 00001120
+    00000006 00000003 00000002 00000001 00000000
+    00000007 00000000 00000000 00000000 00000000
+    00000008 00000000 00000000 00000000 00000000
+    00000009 00000000 00000000 00000000 00000000
+    0000000a 07300403 00000044 00000000 00000603
+    0000000b 00000000 00000000 00000095 00000000
+    80000000 80000008 00000000 00000000 00000000
+    80000001 00000000 00000000 00000001 28100800
+    80000002 65746e49 2952286c 6f655820 2952286e
+    80000003 55504320 20202020 20202020 57202020
+    80000004 30353533 20402020 37302e33 007a4847
+    80000005 00000000 00000000 00000000 00000000
+    80000006 00000000 00000000 01006040 00000000
+    80000007 00000000 00000000 00000000 00000100
+    80000008 00003024 00000000 00000000 00000000
+

lib/spack/external/archspec/vendor/cpuid/cpuid.py

@@ -0,0 +1,172 @@
+# -*- coding: utf-8 -*-
+#
+#     Copyright (c) 2024 Anders Høst
+#
+
+from __future__ import print_function
+
+import platform
+import os
+import ctypes
+from ctypes import c_uint32, c_long, c_ulong, c_size_t, c_void_p, POINTER, CFUNCTYPE
+
+# Posix x86_64:
+# Three first call registers : RDI, RSI, RDX
+# Volatile registers         : RAX, RCX, RDX, RSI, RDI, R8-11
+
+# Windows x86_64:
+# Three first call registers : RCX, RDX, R8
+# Volatile registers         : RAX, RCX, RDX, R8-11
+
+# cdecl 32 bit:
+# Three first call registers : Stack (%esp)
+# Volatile registers         : EAX, ECX, EDX
+
+_POSIX_64_OPC = [
+        0x53,                    # push   %rbx
+        0x89, 0xf0,              # mov    %esi,%eax
+        0x89, 0xd1,              # mov    %edx,%ecx
+        0x0f, 0xa2,              # cpuid
+        0x89, 0x07,              # mov    %eax,(%rdi)
+        0x89, 0x5f, 0x04,        # mov    %ebx,0x4(%rdi)
+        0x89, 0x4f, 0x08,        # mov    %ecx,0x8(%rdi)
+        0x89, 0x57, 0x0c,        # mov    %edx,0xc(%rdi)
+        0x5b,                    # pop    %rbx
+        0xc3                     # retq
+]
+
+_WINDOWS_64_OPC = [
+        0x53,                    # push   %rbx
+        0x89, 0xd0,              # mov    %edx,%eax
+        0x49, 0x89, 0xc9,        # mov    %rcx,%r9
+        0x44, 0x89, 0xc1,        # mov    %r8d,%ecx
+        0x0f, 0xa2,              # cpuid
+        0x41, 0x89, 0x01,        # mov    %eax,(%r9)
+        0x41, 0x89, 0x59, 0x04,  # mov    %ebx,0x4(%r9)
+        0x41, 0x89, 0x49, 0x08,  # mov    %ecx,0x8(%r9)
+        0x41, 0x89, 0x51, 0x0c,  # mov    %edx,0xc(%r9)
+        0x5b,                    # pop    %rbx
+        0xc3                     # retq
+]
+
+_CDECL_32_OPC = [
+        0x53,                    # push   %ebx
+        0x57,                    # push   %edi
+        0x8b, 0x7c, 0x24, 0x0c,  # mov    0xc(%esp),%edi
+        0x8b, 0x44, 0x24, 0x10,  # mov    0x10(%esp),%eax
+        0x8b, 0x4c, 0x24, 0x14,  # mov    0x14(%esp),%ecx
+        0x0f, 0xa2,              # cpuid
+        0x89, 0x07,              # mov    %eax,(%edi)
+        0x89, 0x5f, 0x04,        # mov    %ebx,0x4(%edi)
+        0x89, 0x4f, 0x08,        # mov    %ecx,0x8(%edi)
+        0x89, 0x57, 0x0c,        # mov    %edx,0xc(%edi)
+        0x5f,                    # pop    %edi
+        0x5b,                    # pop    %ebx
+        0xc3                     # ret
+]
+
+is_windows = os.name == "nt"
+is_64bit = ctypes.sizeof(ctypes.c_voidp) == 8
+
+
+class CPUID_struct(ctypes.Structure):
+    _register_names = ("eax", "ebx", "ecx", "edx")
+    _fields_ = [(r, c_uint32) for r in _register_names]
+
+    def __getitem__(self, item):
+        if item not in self._register_names:
+            raise KeyError(item)
+        return getattr(self, item)
+
+    def __repr__(self):
+        return "eax=0x{:x}, ebx=0x{:x}, ecx=0x{:x}, edx=0x{:x}".format(self.eax, self.ebx, self.ecx, self.edx)
+
+
+class CPUID(object):
+    def __init__(self):
+        if platform.machine() not in ("AMD64", "x86_64", "x86", "i686"):
+            raise SystemError("Only available for x86")
+
+        if is_windows:
+            if is_64bit:
+                # VirtualAlloc seems to fail under some weird
+                # circumstances when ctypes.windll.kernel32 is
+                # used under 64 bit Python. CDLL fixes this.
+                self.win = ctypes.CDLL("kernel32.dll")
+                opc = _WINDOWS_64_OPC
+            else:
+                # Here ctypes.windll.kernel32 is needed to get the
+                # right DLL. Otherwise it will fail when running
+                # 32 bit Python on 64 bit Windows.
+                self.win = ctypes.windll.kernel32
+                opc = _CDECL_32_OPC
+        else:
+            opc = _POSIX_64_OPC if is_64bit else _CDECL_32_OPC
+
+        size = len(opc)
+        code = (ctypes.c_ubyte * size)(*opc)
+
+        if is_windows:
+            self.win.VirtualAlloc.restype = c_void_p
+            self.win.VirtualAlloc.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_ulong, ctypes.c_ulong]
+            self.addr = self.win.VirtualAlloc(None, size, 0x1000, 0x40)
+            if not self.addr:
+                raise MemoryError("Could not allocate RWX memory")
+            ctypes.memmove(self.addr, code, size)
+        else:
+            from mmap import (
+                mmap,
+                MAP_PRIVATE,
+                MAP_ANONYMOUS,
+                PROT_WRITE,
+                PROT_READ,
+                PROT_EXEC,
+            )
+            self.mm = mmap(
+                -1,
+                size,
+                flags=MAP_PRIVATE | MAP_ANONYMOUS,
+                prot=PROT_WRITE | PROT_READ | PROT_EXEC,
+            )
+            self.mm.write(code)
+            self.addr = ctypes.addressof(ctypes.c_int.from_buffer(self.mm))
+
+        func_type = CFUNCTYPE(None, POINTER(CPUID_struct), c_uint32, c_uint32)
+        self.func_ptr = func_type(self.addr)
+
+    def __call__(self, eax, ecx=0):
+        struct = self.registers_for(eax=eax, ecx=ecx)
+        return struct.eax, struct.ebx, struct.ecx, struct.edx
+
+    def registers_for(self, eax, ecx=0):
+        """Calls cpuid with eax and ecx set as the input arguments, and returns a structure
+        containing eax, ebx, ecx, and edx.
+        """
+        struct = CPUID_struct()
+        self.func_ptr(struct, eax, ecx)
+        return struct
+
+    def __del__(self):
+        if is_windows:
+            self.win.VirtualFree.restype = c_long
+            self.win.VirtualFree.argtypes = [c_void_p, c_size_t, c_ulong]
+            self.win.VirtualFree(self.addr, 0, 0x8000)
+        else:
+            self.mm.close()
+
+
+
+if __name__ == "__main__":
+    def valid_inputs():
+        cpuid = CPUID()
+        for eax in (0x0, 0x80000000):
+            highest, _, _, _ = cpuid(eax)
+            while eax <= highest:
+                regs = cpuid(eax)
+                yield (eax, regs)
+                eax += 1
+
+
+    print(" ".join(x.ljust(8) for x in ("CPUID", "A", "B", "C", "D")).strip())
+    for eax, regs in valid_inputs():
+        print("%08x" % eax, " ".join("%08x" % reg for reg in regs))

lib/spack/external/archspec/vendor/cpuid/example.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+#
+#     Copyright (c) 2024 Anders Høst
+#
+
+from __future__ import print_function
+
+import struct
+import cpuid
+
+
+def cpu_vendor(cpu):
+    _, b, c, d = cpu(0)
+    return struct.pack("III", b, d, c).decode("utf-8")
+
+
+def cpu_name(cpu):
+    name = "".join((struct.pack("IIII", *cpu(0x80000000 + i)).decode("utf-8")
+                    for i in range(2, 5)))
+
+    return name.split('\x00', 1)[0]
+
+
+def is_set(cpu, leaf, subleaf, reg_idx, bit):
+    """
+    @param {leaf} %eax
+    @param {sublead} %ecx, 0 in most cases
+    @param {reg_idx} idx of [%eax, %ebx, %ecx, %edx], 0-based
+    @param {bit} bit of reg selected by {reg_idx}, 0-based
+    """
+
+    regs = cpu(leaf, subleaf)
+
+    if (1 << bit) & regs[reg_idx]:
+        return "Yes"
+    else:
+        return "--"
+
+
+if __name__ == "__main__":
+    cpu = cpuid.CPUID()
+
+    print("Vendor ID : %s" % cpu_vendor(cpu))
+    print("CPU name  : %s" % cpu_name(cpu))
+    print()
+    print("Vector instructions supported:")
+    print("SSE       : %s" % is_set(cpu, 1, 0, 3, 25))
+    print("SSE2      : %s" % is_set(cpu, 1, 0, 3, 26))
+    print("SSE3      : %s" % is_set(cpu, 1, 0, 2, 0))
+    print("SSSE3     : %s" % is_set(cpu, 1, 0, 2, 9))
+    print("SSE4.1    : %s" % is_set(cpu, 1, 0, 2, 19))
+    print("SSE4.2    : %s" % is_set(cpu, 1, 0, 2, 20))
+    print("SSE4a     : %s" % is_set(cpu, 0x80000001, 0, 2, 6))
+    print("AVX       : %s" % is_set(cpu, 1, 0, 2, 28))
+    print("AVX2      : %s" % is_set(cpu, 7, 0, 1, 5))
+    print("BMI1      : %s" % is_set(cpu, 7, 0, 1, 3))
+    print("BMI2      : %s" % is_set(cpu, 7, 0, 1, 8))
+    # Intel RDT CMT/MBM
+    print("L3 Monitoring : %s" % is_set(cpu, 0xf, 0, 3, 1))
+    print("L3 Occupancy  : %s" % is_set(cpu, 0xf, 1, 3, 0))
+    print("L3 Total BW   : %s" % is_set(cpu, 0xf, 1, 3, 1))
+    print("L3 Local BW   : %s" % is_set(cpu, 0xf, 1, 3, 2))

lib/spack/llnl/path.py

@@ -42,11 +42,6 @@ def convert_to_posix_path(path: str) -> str:
     return format_os_path(path, mode=Path.unix)
 
 
-def convert_to_windows_path(path: str) -> str:
-    """Converts the input path to Windows style."""
-    return format_os_path(path, mode=Path.windows)
-
-
 def convert_to_platform_path(path: str) -> str:
     """Converts the input path to the current platform's native style."""
     return format_os_path(path, mode=Path.platform_path)

lib/spack/llnl/util/filesystem.py

@@ -171,7 +171,7 @@ def polite_path(components: Iterable[str]):
 @memoized
 def _polite_antipattern():
     # A regex of all the characters we don't want in a filename
-    return re.compile(r"[^A-Za-z0-9_.-]")
+    return re.compile(r"[^A-Za-z0-9_+.-]")
 
 
 def polite_filename(filename: str) -> str:
@@ -237,16 +237,6 @@ def _get_mime_type():
         return file_command("-b", "-h", "--mime-type")
 
 
-@memoized
-def _get_mime_type_compressed():
-    """Same as _get_mime_type but attempts to check for
-    compression first
-    """
-    mime_uncompressed = _get_mime_type()
-    mime_uncompressed.add_default_arg("-Z")
-    return mime_uncompressed
-
-
 def mime_type(filename):
     """Returns the mime type and subtype of a file.
 
@@ -262,21 +252,6 @@ def mime_type(filename):
     return type, subtype
 
 
-def compressed_mime_type(filename):
-    """Same as mime_type but checks for type that has been compressed
-
-    Args:
-        filename (str): file to be analyzed
-
-    Returns:
-        Tuple containing the MIME type and subtype
-    """
-    output = _get_mime_type_compressed()(filename, output=str, error=str).strip()
-    tty.debug("==> " + output)
-    type, _, subtype = output.partition("/")
-    return type, subtype
-
-
 #: This generates the library filenames that may appear on any OS.
 library_extensions = ["a", "la", "so", "tbd", "dylib"]
 
@@ -308,13 +283,6 @@ def paths_containing_libs(paths, library_names):
     return rpaths_to_include
 
 
-@system_path_filter
-def same_path(path1, path2):
-    norm1 = os.path.abspath(path1).rstrip(os.path.sep)
-    norm2 = os.path.abspath(path2).rstrip(os.path.sep)
-    return norm1 == norm2
-
-
 def filter_file(
     regex: str,
     repl: Union[str, Callable[[Match], str]],
@@ -909,39 +877,34 @@ def is_exe(path):
     return os.path.isfile(path) and os.access(path, os.X_OK)
 
 
-@system_path_filter
-def get_filetype(path_name):
-    """
-    Return the output of file path_name as a string to identify file type.
-    """
-    file = Executable("file")
-    file.add_default_env("LC_ALL", "C")
-    output = file("-b", "-h", "%s" % path_name, output=str, error=str)
-    return output.strip()
+def has_shebang(path):
+    """Returns whether a path has a shebang line. Returns False if the file cannot be opened."""
+    try:
+        with open(path, "rb") as f:
+            return f.read(2) == b"#!"
+    except OSError:
+        return False
 
 
 @system_path_filter
 def is_nonsymlink_exe_with_shebang(path):
-    """
-    Returns whether the path is an executable script with a shebang.
-    Return False when the path is a *symlink* to an executable script.
-    """
+    """Returns whether the path is an executable regular file with a shebang. Returns False too
+    when the path is a symlink to a script, and also when the file cannot be opened."""
     try:
         st = os.lstat(path)
-        # Should not be a symlink
-        if stat.S_ISLNK(st.st_mode):
-            return False
-
-        # Should be executable
-        if not st.st_mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH):
-            return False
-
-        # Should start with a shebang
-        with open(path, "rb") as f:
-            return f.read(2) == b"#!"
-    except (IOError, OSError):
+    except OSError:
         return False
 
+    # Should not be a symlink
+    if stat.S_ISLNK(st.st_mode):
+        return False
+
+    # Should be executable
+    if not st.st_mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH):
+        return False
+
+    return has_shebang(path)
+
 
 @system_path_filter(arg_slice=slice(1))
 def chgrp_if_not_world_writable(path, group):
@@ -1163,20 +1126,6 @@ def write_tmp_and_move(filename):
     shutil.move(tmp, filename)
 
 
-@contextmanager
-@system_path_filter
-def open_if_filename(str_or_file, mode="r"):
-    """Takes either a path or a file object, and opens it if it is a path.
-
-    If it's a file object, just yields the file object.
-    """
-    if isinstance(str_or_file, str):
-        with open(str_or_file, mode) as f:
-            yield f
-    else:
-        yield str_or_file
-
-
 @system_path_filter
 def touch(path):
     """Creates an empty file at the specified path."""
@@ -1234,6 +1183,47 @@ def get_single_file(directory):
     return fnames[0]
 
 
+@system_path_filter
+def windows_sfn(path: os.PathLike):
+    """Returns 8.3 Filename (SFN) representation of
+    path
+
+    8.3 Filenames (SFN or short filename) is a file
+    naming convention used prior to Win95 that Windows
+    still (and will continue to) support. This convention
+    caps filenames at 8 characters, and most importantly
+    does not allow for spaces in addition to other specifications.
+    The scheme is generally the same as a normal Windows
+    file scheme, but all spaces are removed and the filename
+    is capped at 6 characters. The remaining characters are
+    replaced with ~N where N is the number file in a directory
+    that a given file represents i.e. Program Files and Program Files (x86)
+    would be PROGRA~1 and PROGRA~2 respectively.
+    Further, all file/directory names are all caps (although modern Windows
+    is case insensitive in practice).
+    Conversion is accomplished by fileapi.h GetShortPathNameW
+
+    Returns paths in 8.3 Filename form
+
+    Note: this method is a no-op on Linux
+
+    Args:
+        path: Path to be transformed into SFN (8.3 filename) format
+    """
+    # This should not be run-able on linux/macos
+    if sys.platform != "win32":
+        return path
+    path = str(path)
+    import ctypes
+
+    k32 = ctypes.WinDLL("kernel32", use_last_error=True)
+    # stub Windows types TCHAR[LENGTH]
+    TCHAR_arr = ctypes.c_wchar * len(path)
+    ret_str = TCHAR_arr()
+    k32.GetShortPathNameW(path, ret_str, len(path))
+    return ret_str.value
+
+
 @contextmanager
 def temp_cwd():
     tmp_dir = tempfile.mkdtemp()
@@ -1248,19 +1238,6 @@ def temp_cwd():
         shutil.rmtree(tmp_dir, **kwargs)
 
 
-@contextmanager
-@system_path_filter
-def temp_rename(orig_path, temp_path):
-    same_path = os.path.realpath(orig_path) == os.path.realpath(temp_path)
-    if not same_path:
-        shutil.move(orig_path, temp_path)
-    try:
-        yield
-    finally:
-        if not same_path:
-            shutil.move(temp_path, orig_path)
-
-
 @system_path_filter
 def can_access(file_name):
     """True if we have read/write access to the file."""

lib/spack/llnl/util/lang.py

@@ -98,36 +98,6 @@ def caller_locals():
         del stack
 
 
-def get_calling_module_name():
-    """Make sure that the caller is a class definition, and return the
-    enclosing module's name.
-    """
-    # Passing zero here skips line context for speed.
-    stack = inspect.stack(0)
-    try:
-        # Make sure locals contain __module__
-        caller_locals = stack[2][0].f_locals
-    finally:
-        del stack
-
-    if "__module__" not in caller_locals:
-        raise RuntimeError(
-            "Must invoke get_calling_module_name() " "from inside a class definition!"
-        )
-
-    module_name = caller_locals["__module__"]
-    base_name = module_name.split(".")[-1]
-    return base_name
-
-
-def attr_required(obj, attr_name):
-    """Ensure that a class has a required attribute."""
-    if not hasattr(obj, attr_name):
-        raise RequiredAttributeError(
-            "No required attribute '%s' in class '%s'" % (attr_name, obj.__class__.__name__)
-        )
-
-
 def attr_setdefault(obj, name, value):
     """Like dict.setdefault, but for objects."""
     if not hasattr(obj, name):
@@ -513,42 +483,6 @@ def copy(self):
         return clone
 
 
-def in_function(function_name):
-    """True if the caller was called from some function with
-    the supplied Name, False otherwise."""
-    stack = inspect.stack()
-    try:
-        for elt in stack[2:]:
-            if elt[3] == function_name:
-                return True
-        return False
-    finally:
-        del stack
-
-
-def check_kwargs(kwargs, fun):
-    """Helper for making functions with kwargs.  Checks whether the kwargs
-    are empty after all of them have been popped off.  If they're
-    not, raises an error describing which kwargs are invalid.
-
-    Example::
-
-       def foo(self, **kwargs):
-           x = kwargs.pop('x', None)
-           y = kwargs.pop('y', None)
-           z = kwargs.pop('z', None)
-           check_kwargs(kwargs, self.foo)
-
-       # This raises a TypeError:
-       foo(w='bad kwarg')
-    """
-    if kwargs:
-        raise TypeError(
-            "'%s' is an invalid keyword argument for function %s()."
-            % (next(iter(kwargs)), fun.__name__)
-        )
-
-
 def match_predicate(*args):
     """Utility function for making string matching predicates.
 
@@ -764,11 +698,6 @@ def pretty_seconds(seconds):
     return pretty_seconds_formatter(seconds)(seconds)
 
 
-class RequiredAttributeError(ValueError):
-    def __init__(self, message):
-        super().__init__(message)
-
-
 class ObjectWrapper:
     """Base class that wraps an object. Derived classes can add new behavior
     while staying undercover.
@@ -843,6 +772,30 @@ def __repr__(self):
         return repr(self.instance)
 
 
+def get_entry_points(*, group: str):
+    """Wrapper for ``importlib.metadata.entry_points``
+
+    Args:
+        group: entry points to select
+
+    Returns:
+        EntryPoints for ``group`` or empty list if unsupported
+    """
+
+    try:
+        import importlib.metadata  # type: ignore  # novermin
+    except ImportError:
+        return []
+
+    try:
+        return importlib.metadata.entry_points(group=group)
+    except TypeError:
+        # Prior to Python 3.10, entry_points accepted no parameters and always
+        # returned a dictionary of entry points, keyed by group.  See
+        # https://docs.python.org/3/library/importlib.metadata.html#entry-points
+        return importlib.metadata.entry_points().get(group, [])
+
+
 def load_module_from_file(module_name, module_path):
     """Loads a python module from the path of the corresponding file.
 
@@ -911,25 +864,6 @@ def uniq(sequence):
     return uniq_list
 
 
-def star(func):
-    """Unpacks arguments for use with Multiprocessing mapping functions"""
-
-    def _wrapper(args):
-        return func(*args)
-
-    return _wrapper
-
-
-class Devnull:
-    """Null stream with less overhead than ``os.devnull``.
-
-    See https://stackoverflow.com/a/2929954.
-    """
-
-    def write(self, *_):
-        pass
-
-
 def elide_list(line_list, max_num=10):
     """Takes a long list and limits it to a smaller number of elements,
     replacing intervening elements with '...'.  For example::

lib/spack/llnl/util/lock.py

@@ -815,10 +815,6 @@ def __init__(self, path):
         super().__init__(msg)
 
 
-class LockLimitError(LockError):
-    """Raised when exceed maximum attempts to acquire a lock."""
-
-
 class LockTimeoutError(LockError):
     """Raised when an attempt to acquire a lock times out."""
 

lib/spack/llnl/util/symlink.py

@@ -189,6 +189,7 @@ def _windows_can_symlink() -> bool:
     import llnl.util.filesystem as fs
 
     fs.touchp(fpath)
+    fs.mkdirp(dpath)
 
     try:
         os.symlink(dpath, dlink)

lib/spack/llnl/util/tty/__init__.py

@@ -44,10 +44,6 @@ def is_debug(level=1):
     return _debug >= level
 
 
-def is_stacktrace():
-    return _stacktrace
-
-
 def set_debug(level=0):
     global _debug
     assert level >= 0, "Debug level must be a positive value"
@@ -252,37 +248,6 @@ def die(message, *args, **kwargs) -> NoReturn:
     sys.exit(1)
 
 
-def get_number(prompt, **kwargs):
-    default = kwargs.get("default", None)
-    abort = kwargs.get("abort", None)
-
-    if default is not None and abort is not None:
-        prompt += " (default is %s, %s to abort) " % (default, abort)
-    elif default is not None:
-        prompt += " (default is %s) " % default
-    elif abort is not None:
-        prompt += " (%s to abort) " % abort
-
-    number = None
-    while number is None:
-        msg(prompt, newline=False)
-        ans = input()
-        if ans == str(abort):
-            return None
-
-        if ans:
-            try:
-                number = int(ans)
-                if number < 1:
-                    msg("Please enter a valid number.")
-                    number = None
-            except ValueError:
-                msg("Please enter a valid number.")
-        elif default is not None:
-            number = default
-    return number
-
-
 def get_yes_or_no(prompt, **kwargs):
     default_value = kwargs.get("default", None)
 

lib/spack/spack/audit.py

@@ -796,13 +796,33 @@ def check_virtual_with_variants(spec, msg):
                 except spack.repo.UnknownPackageError:
                     # This dependency is completely missing, so report
                     # and continue the analysis
-                    summary = (
-                        f"{pkg_name}: unknown package '{dep_name}' in " "'depends_on' directive"
-                    )
+                    summary = f"{pkg_name}: unknown package '{dep_name}' in 'depends_on' directive"
                     details = [f" in {filename}"]
                     errors.append(error_cls(summary=summary, details=details))
                     continue
 
+                # Check for self-referential specs similar to:
+                #
+                # depends_on("foo@X.Y", when="^foo+bar")
+                #
+                # That would allow clingo to choose whether to have foo@X.Y+bar in the graph.
+                problematic_edges = [
+                    x for x in when.edges_to_dependencies(dep_name) if not x.virtuals
+                ]
+                if problematic_edges and not dep.patches:
+                    summary = (
+                        f"{pkg_name}: dependency on '{dep.spec}' when '{when}' is self-referential"
+                    )
+                    details = [
+                        (
+                            f" please specify better using '^[virtuals=...] {dep_name}', or "
+                            f"substitute with an equivalent condition on '{pkg_name}'"
+                        ),
+                        f" in {filename}",
+                    ]
+                    errors.append(error_cls(summary=summary, details=details))
+                    continue
+
                 # check variants
                 dependency_variants = dep.spec.variants
                 for name, value in dependency_variants.items():

lib/spack/spack/binary_distribution.py

@@ -1541,7 +1541,7 @@ def fetch_url_to_mirror(url):
                     response = spack.oci.opener.urlopen(
                         urllib.request.Request(
                             url=ref.manifest_url(),
-                            headers={"Accept": "application/vnd.oci.image.manifest.v1+json"},
+                            headers={"Accept": ", ".join(spack.oci.oci.manifest_content_type)},
                         )
                     )
                 except Exception:

lib/spack/spack/bootstrap/_common.py

@@ -213,9 +213,6 @@ def _root_spec(spec_str: str) -> str:
     platform = str(spack.platforms.host())
     if platform == "darwin":
         spec_str += " %apple-clang"
-    elif platform == "windows":
-        # TODO (johnwparent): Remove version constraint when clingo patch is up
-        spec_str += " %msvc@:19.37"
     elif platform == "linux":
         spec_str += " %gcc"
     elif platform == "freebsd":

lib/spack/spack/bootstrap/config.py

@@ -147,7 +147,7 @@ def _add_compilers_if_missing() -> None:
             mixed_toolchain=sys.platform == "darwin"
         )
         if new_compilers:
-            spack.compilers.add_compilers_to_config(new_compilers, init_config=False)
+            spack.compilers.add_compilers_to_config(new_compilers)
 
 
 @contextlib.contextmanager

lib/spack/spack/build_systems/autotools.py

@@ -541,7 +541,7 @@ def autoreconf(self, pkg, spec, prefix):
         if os.path.exists(self.configure_abs_path):
             return
 
-        # Else try to regenerate it, which reuquires a few build dependencies
+        # Else try to regenerate it, which requires a few build dependencies
         ensure_build_dependencies_or_raise(
             spec=spec,
             dependencies=["autoconf", "automake", "libtool"],

lib/spack/spack/build_systems/cmake.py

@@ -58,6 +58,20 @@ def _maybe_set_python_hints(pkg: spack.package_base.PackageBase, args: List[str]
     )
 
 
+def _supports_compilation_databases(pkg: spack.package_base.PackageBase) -> bool:
+    """Check if this package (and CMake) can support compilation databases."""
+
+    # CMAKE_EXPORT_COMPILE_COMMANDS only exists for CMake >= 3.5
+    if not pkg.spec.satisfies("^cmake@3.5:"):
+        return False
+
+    # CMAKE_EXPORT_COMPILE_COMMANDS is only implemented for Makefile and Ninja generators
+    if not (pkg.spec.satisfies("generator=make") or pkg.spec.satisfies("generator=ninja")):
+        return False
+
+    return True
+
+
 def _conditional_cmake_defaults(pkg: spack.package_base.PackageBase, args: List[str]) -> None:
     """Set a few default defines for CMake, depending on its version."""
     cmakes = pkg.spec.dependencies("cmake", dt.BUILD)
@@ -95,6 +109,10 @@ def _conditional_cmake_defaults(pkg: spack.package_base.PackageBase, args: List[
         args.append(CMakeBuilder.define("CMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY", False))
         args.append(CMakeBuilder.define("CMAKE_FIND_PACKAGE_NO_SYSTEM_PACKAGE_REGISTRY", False))
 
+    # Export a compilation database if supported.
+    if _supports_compilation_databases(pkg):
+        args.append(CMakeBuilder.define("CMAKE_EXPORT_COMPILE_COMMANDS", True))
+
 
 def generator(*names: str, default: Optional[str] = None):
     """The build system generator to use.
@@ -284,7 +302,10 @@ class CMakeBuilder(BaseBuilder):
     @property
     def archive_files(self):
         """Files to archive for packages based on CMake"""
-        return [os.path.join(self.build_directory, "CMakeCache.txt")]
+        files = [os.path.join(self.build_directory, "CMakeCache.txt")]
+        if _supports_compilation_databases(self):
+            files.append(os.path.join(self.build_directory, "compile_commands.json"))
+        return files
 
     @property
     def root_cmakelists_dir(self):

lib/spack/spack/build_systems/msbuild.py

@@ -69,7 +69,7 @@ class MSBuildBuilder(BaseBuilder):
     @property
     def build_directory(self):
         """Return the directory containing the MSBuild solution or vcxproj."""
-        return self.pkg.stage.source_path
+        return fs.windows_sfn(self.pkg.stage.source_path)
 
     @property
     def toolchain_version(self):

lib/spack/spack/build_systems/nmake.py

@@ -77,7 +77,11 @@ def ignore_quotes(self):
     @property
     def build_directory(self):
         """Return the directory containing the makefile."""
-        return self.pkg.stage.source_path if not self.makefile_root else self.makefile_root
+        return (
+            fs.windows_sfn(self.pkg.stage.source_path)
+            if not self.makefile_root
+            else fs.windows_sfn(self.makefile_root)
+        )
 
     @property
     def std_nmake_args(self):

lib/spack/spack/build_systems/oneapi.py

@@ -9,10 +9,13 @@
 import shutil
 from os.path import basename, isdir
 
-from llnl.util.filesystem import HeaderList, find_libraries, join_path, mkdirp
+from llnl.util import tty
+from llnl.util.filesystem import HeaderList, LibraryList, find_libraries, join_path, mkdirp
 from llnl.util.link_tree import LinkTree
 
+from spack.build_environment import dso_suffix
 from spack.directives import conflicts, variant
+from spack.package_base import InstallError
 from spack.util.environment import EnvironmentModifications
 from spack.util.executable import Executable
 
@@ -179,16 +182,72 @@ class IntelOneApiLibraryPackage(IntelOneApiPackage):
 
     """
 
+    def openmp_libs(self):
+        """Supply LibraryList for linking OpenMP"""
+
+        # NB: Hunting down explicit library files may be the Spack way of
+        # doing things, but it is better to add the compiler defined option
+        # e.g. -fopenmp
+
+        # If other packages use openmp, then all the packages need to
+        # support the same ABI. Spack usually uses the same compiler
+        # for all the packages, but you can force it if necessary:
+        #
+        # e.g. spack install blaspp%oneapi@2024 ^intel-oneapi-mkl%oneapi@2024
+        #
+        if self.spec.satisfies("%intel") or self.spec.satisfies("%oneapi"):
+            libname = "libiomp5"
+        elif self.spec.satisfies("%gcc"):
+            libname = "libgomp"
+        elif self.spec.satisfies("%clang"):
+            libname = "libomp"
+        else:
+            raise InstallError(
+                "OneAPI package with OpenMP threading requires one of %clang, %gcc, %oneapi, "
+                "or %intel"
+            )
+
+        # query the compiler for the library path
+        with self.compiler.compiler_environment():
+            omp_lib_path = Executable(self.compiler.cc)(
+                "--print-file-name", f"{libname}.{dso_suffix}", output=str
+            ).strip()
+
+        # Newer versions of clang do not give the full path to libomp. If that's
+        # the case, look in a path relative to the compiler where libomp is
+        # typically found. If it's not found there, error out.
+        if not os.path.exists(omp_lib_path) and self.spec.satisfies("%clang"):
+            compiler_root = os.path.dirname(os.path.dirname(os.path.realpath(self.compiler.cc)))
+            omp_lib_path_compiler = os.path.join(compiler_root, "lib", f"{libname}.{dso_suffix}")
+            if os.path.exists(omp_lib_path_compiler):
+                omp_lib_path = omp_lib_path_compiler
+
+        # if the compiler cannot find the file, it returns the input path
+        if not os.path.exists(omp_lib_path):
+            raise InstallError(f"OneAPI package cannot locate OpenMP library: {omp_lib_path}")
+
+        omp_libs = LibraryList(omp_lib_path)
+        tty.info(f"OneAPI package requires OpenMP library: {omp_libs}")
+        return omp_libs
+
+    # find_headers uses heuristics to determine the include directory
+    # that does not work for oneapi packages. Use explicit directories
+    # instead.
     def header_directories(self, dirs):
         h = HeaderList([])
         h.directories = dirs
+        # trilinos passes the directories to cmake, and cmake requires
+        # that the directory exists
+        for dir in dirs:
+            if not isdir(dir):
+                raise RuntimeError(f"{dir} does not exist")
         return h
 
     @property
     def headers(self):
-        return self.header_directories(
-            [self.component_prefix.include, self.component_prefix.include.join(self.component_dir)]
-        )
+        # This should match the directories added to CPATH by
+        # env/vars.sh for the component
+        return self.header_directories([self.component_prefix.include])
 
     @property
     def libs(self):

lib/spack/spack/build_systems/perl.py

@@ -4,12 +4,15 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import inspect
 import os
+from typing import Iterable
 
-from llnl.util.filesystem import filter_file
+from llnl.util.filesystem import filter_file, find
+from llnl.util.lang import memoized
 
 import spack.builder
 import spack.package_base
 from spack.directives import build_system, extends
+from spack.install_test import SkipTest, test_part
 from spack.util.executable import Executable
 
 from ._checks import BaseBuilder, execute_build_time_tests
@@ -28,6 +31,58 @@ class PerlPackage(spack.package_base.PackageBase):
 
     extends("perl", when="build_system=perl")
 
+    @property
+    @memoized
+    def _platform_dir(self):
+        """Name of platform-specific module subdirectory."""
+        perl = self.spec["perl"].command
+        options = "-E", "use Config; say $Config{archname}"
+        out = perl(*options, output=str.split, error=str.split)
+        return out.strip()
+
+    @property
+    def use_modules(self) -> Iterable[str]:
+        """Names of the package's perl modules."""
+        module_files = find(self.prefix.lib, ["*.pm"], recursive=True)
+
+        # Drop the platform directory, if present
+        if self._platform_dir:
+            platform_dir = self._platform_dir + os.sep
+            module_files = [m.replace(platform_dir, "") for m in module_files]
+
+        # Drop the extension and library path
+        prefix = self.prefix.lib + os.sep
+        modules = [os.path.splitext(m)[0].replace(prefix, "") for m in module_files]
+
+        # Drop the perl subdirectory as well
+        return ["::".join(m.split(os.sep)[1:]) for m in modules]
+
+    @property
+    def skip_modules(self) -> Iterable[str]:
+        """Names of modules that should be skipped when running tests.
+
+        These are a subset of use_modules.
+
+        Returns:
+            List of strings of module names.
+        """
+        return []
+
+    def test_use(self):
+        """Test 'use module'"""
+        if not self.use_modules:
+            raise SkipTest("Test requires use_modules package property.")
+
+        perl = self.spec["perl"].command
+        for module in self.use_modules:
+            if module in self.skip_modules:
+                continue
+
+            with test_part(self, f"test_use-{module}", purpose=f"checking use of {module}"):
+                options = ["-we", f'use strict; use {module}; print("OK\n")']
+                out = perl(*options, output=str.split, error=str.split)
+                assert "OK" in out
+
 
 @spack.builder.builder("perl")
 class PerlBuilder(BaseBuilder):
@@ -52,7 +107,7 @@ class PerlBuilder(BaseBuilder):
     phases = ("configure", "build", "install")
 
     #: Names associated with package methods in the old build-system format
-    legacy_methods = ("configure_args", "check")
+    legacy_methods = ("configure_args", "check", "test_use")
 
     #: Names associated with package attributes in the old build-system format
     legacy_attributes = ()

lib/spack/spack/build_systems/python.py

@@ -2,11 +2,15 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import functools
 import inspect
+import operator
 import os
 import re
 import shutil
-from typing import Iterable, List, Mapping, Optional
+import stat
+from typing import Dict, Iterable, List, Mapping, Optional, Tuple
 
 import archspec
 
@@ -23,7 +27,7 @@
 import spack.package_base
 import spack.spec
 import spack.store
-from spack.directives import build_system, depends_on, extends, maintainers
+from spack.directives import build_system, depends_on, extends
 from spack.error import NoHeadersError, NoLibrariesError
 from spack.install_test import test_part
 from spack.spec import Spec
@@ -52,8 +56,6 @@ def _flatten_dict(dictionary: Mapping[str, object]) -> Iterable[str]:
 
 
 class PythonExtension(spack.package_base.PackageBase):
-    maintainers("adamjstewart")
-
     @property
     def import_modules(self) -> Iterable[str]:
         """Names of modules that the Python package provides.
@@ -136,31 +138,52 @@ def view_file_conflicts(self, view, merge_map):
         return conflicts
 
     def add_files_to_view(self, view, merge_map, skip_if_exists=True):
-        if not self.extendee_spec:
+        # Patch up shebangs to the python linked in the view only if python is built by Spack.
+        if not self.extendee_spec or self.extendee_spec.external:
             return super().add_files_to_view(view, merge_map, skip_if_exists)
 
+        # We only patch shebangs in the bin directory.
+        copied_files: Dict[Tuple[int, int], str] = {}  # File identifier -> source
+        delayed_links: List[Tuple[str, str]] = []  # List of symlinks from merge map
+
         bin_dir = self.spec.prefix.bin
         python_prefix = self.extendee_spec.prefix
-        python_is_external = self.extendee_spec.external
-        global_view = fs.same_path(python_prefix, view.get_projection_for_spec(self.spec))
         for src, dst in merge_map.items():
-            if os.path.exists(dst):
+            if skip_if_exists and os.path.lexists(dst):
                 continue
-            elif global_view or not fs.path_contains_subdirectory(src, bin_dir):
+
+            if not fs.path_contains_subdirectory(src, bin_dir):
                 view.link(src, dst)
-            elif not os.path.islink(src):
+                continue
+
+            s = os.lstat(src)
+
+            # Symlink is delayed because we may need to re-target if its target is copied in view
+            if stat.S_ISLNK(s.st_mode):
+                delayed_links.append((src, dst))
+                continue
+
+            # If it's executable and has a shebang, copy and patch it.
+            if (s.st_mode & 0b111) and fs.has_shebang(src):
+                copied_files[(s.st_dev, s.st_ino)] = dst
                 shutil.copy2(src, dst)
-                is_script = fs.is_nonsymlink_exe_with_shebang(src)
-                if is_script and not python_is_external:
-                    fs.filter_file(
-                        python_prefix,
-                        os.path.abspath(view.get_projection_for_spec(self.spec)),
-                        dst,
-                    )
+                fs.filter_file(
+                    python_prefix, os.path.abspath(view.get_projection_for_spec(self.spec)), dst
+                )
             else:
-                orig_link_target = os.path.realpath(src)
-                new_link_target = os.path.abspath(merge_map[orig_link_target])
-                view.link(new_link_target, dst)
+                view.link(src, dst)
+
+        # Finally re-target the symlinks that point to copied files.
+        for src, dst in delayed_links:
+            try:
+                s = os.stat(src)
+                target = copied_files[(s.st_dev, s.st_ino)]
+            except (OSError, KeyError):
+                target = None
+            if target:
+                os.symlink(os.path.relpath(target, os.path.dirname(dst)), dst)
+            else:
+                view.link(src, dst, spec=self.spec)
 
     def remove_files_from_view(self, view, merge_map):
         ignore_namespace = False
@@ -346,16 +369,19 @@ def headers(self) -> HeaderList:
         # Remove py- prefix in package name
         name = self.spec.name[3:]
 
-        # Headers may be in either location
+        # Headers should only be in include or platlib, but no harm in checking purelib too
         include = self.prefix.join(self.spec["python"].package.include).join(name)
         platlib = self.prefix.join(self.spec["python"].package.platlib).join(name)
-        headers = fs.find_all_headers(include) + fs.find_all_headers(platlib)
+        purelib = self.prefix.join(self.spec["python"].package.purelib).join(name)
+
+        headers_list = map(fs.find_all_headers, [include, platlib, purelib])
+        headers = functools.reduce(operator.add, headers_list)
 
         if headers:
             return headers
 
-        msg = "Unable to locate {} headers in {} or {}"
-        raise NoHeadersError(msg.format(self.spec.name, include, platlib))
+        msg = "Unable to locate {} headers in {}, {}, or {}"
+        raise NoHeadersError(msg.format(self.spec.name, include, platlib, purelib))
 
     @property
     def libs(self) -> LibraryList:
@@ -364,15 +390,19 @@ def libs(self) -> LibraryList:
         # Remove py- prefix in package name
         name = self.spec.name[3:]
 
-        root = self.prefix.join(self.spec["python"].package.platlib).join(name)
+        # Libraries should only be in platlib, but no harm in checking purelib too
+        platlib = self.prefix.join(self.spec["python"].package.platlib).join(name)
+        purelib = self.prefix.join(self.spec["python"].package.purelib).join(name)
 
-        libs = fs.find_all_libraries(root, recursive=True)
+        find_all_libraries = functools.partial(fs.find_all_libraries, recursive=True)
+        libs_list = map(find_all_libraries, [platlib, purelib])
+        libs = functools.reduce(operator.add, libs_list)
 
         if libs:
             return libs
 
-        msg = "Unable to recursively locate {} libraries in {}"
-        raise NoLibrariesError(msg.format(self.spec.name, root))
+        msg = "Unable to recursively locate {} libraries in {} or {}"
+        raise NoLibrariesError(msg.format(self.spec.name, platlib, purelib))
 
 
 @spack.builder.builder("python_pip")

lib/spack/spack/build_systems/rocm.py

@@ -162,23 +162,9 @@ def hip_flags(amdgpu_target):
 
     # Add compiler minimum versions based on the first release where the
     # processor is included in llvm/lib/Support/TargetParser.cpp
-    depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx900:xnack-")
-    depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx906:xnack-")
-    depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx908:xnack-")
-    depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx90c")
-    depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx90a")
-    depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx90a:xnack-")
-    depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx90a:xnack+")
     depends_on("llvm-amdgpu@5.2.0:", when="amdgpu_target=gfx940")
     depends_on("llvm-amdgpu@5.7.0:", when="amdgpu_target=gfx941")
     depends_on("llvm-amdgpu@5.7.0:", when="amdgpu_target=gfx942")
-    depends_on("llvm-amdgpu@4.5.0:", when="amdgpu_target=gfx1013")
-    depends_on("llvm-amdgpu@3.8.0:", when="amdgpu_target=gfx1030")
-    depends_on("llvm-amdgpu@3.9.0:", when="amdgpu_target=gfx1031")
-    depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx1032")
-    depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx1033")
-    depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx1034")
-    depends_on("llvm-amdgpu@4.5.0:", when="amdgpu_target=gfx1035")
     depends_on("llvm-amdgpu@5.2.0:", when="amdgpu_target=gfx1036")
     depends_on("llvm-amdgpu@5.3.0:", when="amdgpu_target=gfx1100")
     depends_on("llvm-amdgpu@5.3.0:", when="amdgpu_target=gfx1101")

lib/spack/spack/builder.py

@@ -9,6 +9,8 @@
 import inspect
 from typing import List, Optional, Tuple
 
+from llnl.util import lang
+
 import spack.build_environment
 
 #: Builder classes, as registered by the "builder" decorator
@@ -231,24 +233,27 @@ def __new__(mcs, name, bases, attr_dict):
         for temporary_stage in (_RUN_BEFORE, _RUN_AFTER):
             staged_callbacks = temporary_stage.callbacks
 
-            # We don't have callbacks in this class, move on
-            if not staged_callbacks:
+            # Here we have an adapter from an old-style package. This means there is no
+            # hierarchy of builders, and every callback that had to be combined between
+            # *Package and *Builder has been combined already by _PackageAdapterMeta
+            if name == "Adapter":
                 continue
 
-            # If we are here we have callbacks. To get a complete list, get first what
-            # was attached to parent classes, then prepend what we have registered here.
+            # If we are here we have callbacks. To get a complete list, we accumulate all the
+            # callbacks from base classes, we deduplicate them, then prepend what we have
+            # registered here.
             #
             # The order should be:
             # 1. Callbacks are registered in order within the same class
             # 2. Callbacks defined in derived classes precede those defined in base
             #    classes
+            callbacks_from_base = []
             for base in bases:
-                callbacks_from_base = getattr(base, temporary_stage.attribute_name, None)
-                if callbacks_from_base:
-                    break
-            else:
-                callbacks_from_base = []
-
+                current_callbacks = getattr(base, temporary_stage.attribute_name, None)
+                if not current_callbacks:
+                    continue
+                callbacks_from_base.extend(current_callbacks)
+            callbacks_from_base = list(lang.dedupe(callbacks_from_base))
             # Set the callbacks in this class and flush the temporary stage
             attr_dict[temporary_stage.attribute_name] = staged_callbacks[:] + callbacks_from_base
             del temporary_stage.callbacks[:]

lib/spack/spack/cmd/buildcache.py

@@ -594,6 +594,15 @@ def _put_manifest(
     base_manifest, base_config = base_images[architecture]
     env = _retrieve_env_dict_from_config(base_config)
 
+    # If the base image uses `vnd.docker.distribution.manifest.v2+json`, then we use that too.
+    # This is because Singularity / Apptainer is very strict about not mixing them.
+    base_manifest_mediaType = base_manifest.get(
+        "mediaType", "application/vnd.oci.image.manifest.v1+json"
+    )
+    use_docker_format = (
+        base_manifest_mediaType == "application/vnd.docker.distribution.manifest.v2+json"
+    )
+
     spack.user_environment.environment_modifications_for_specs(*specs).apply_modifications(env)
 
     # Create an oci.image.config file
@@ -625,8 +634,8 @@ def _put_manifest(
     # Upload the config file
     upload_blob_with_retry(image_ref, file=config_file, digest=config_file_checksum)
 
-    oci_manifest = {
-        "mediaType": "application/vnd.oci.image.manifest.v1+json",
+    manifest = {
+        "mediaType": base_manifest_mediaType,
         "schemaVersion": 2,
         "config": {
             "mediaType": base_manifest["config"]["mediaType"],
@@ -637,7 +646,11 @@ def _put_manifest(
             *(layer for layer in base_manifest["layers"]),
             *(
                 {
-                    "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+                    "mediaType": (
+                        "application/vnd.docker.image.rootfs.diff.tar.gzip"
+                        if use_docker_format
+                        else "application/vnd.oci.image.layer.v1.tar+gzip"
+                    ),
                     "digest": str(checksums[s.dag_hash()].compressed_digest),
                     "size": checksums[s.dag_hash()].size,
                 }
@@ -646,11 +659,11 @@ def _put_manifest(
         ],
     }
 
-    if annotations:
-        oci_manifest["annotations"] = annotations
+    if not use_docker_format and annotations:
+        manifest["annotations"] = annotations
 
     # Finally upload the manifest
-    upload_manifest_with_retry(image_ref, oci_manifest=oci_manifest)
+    upload_manifest_with_retry(image_ref, manifest=manifest)
 
     # delete the config file
     os.unlink(config_file)

lib/spack/spack/cmd/common/arguments.py

@@ -404,6 +404,17 @@ def no_install_status():
     )
 
 
+@arg
+def debug_nondefaults():
+    return Args(
+        "-d",
+        "--debug-nondefaults",
+        action="store_true",
+        default=False,
+        help="show non-default decisions in red",
+    )
+
+
 @arg
 def no_checksum():
     return Args(
@@ -570,6 +581,14 @@ def add_concretizer_args(subparser):
         default=None,
         help="reuse installed dependencies only",
     )
+    subgroup.add_argument(
+        "--deprecated",
+        action=ConfigSetAction,
+        dest="config:deprecated",
+        const=True,
+        default=None,
+        help="allow concretizer to select deprecated versions",
+    )
 
 
 def add_connection_args(subparser, add_help):

lib/spack/spack/cmd/compiler.py

@@ -89,7 +89,7 @@ def compiler_find(args):
         paths, scope=None, mixed_toolchain=args.mixed_toolchain
     )
     if new_compilers:
-        spack.compilers.add_compilers_to_config(new_compilers, scope=args.scope, init_config=False)
+        spack.compilers.add_compilers_to_config(new_compilers, scope=args.scope)
         n = len(new_compilers)
         s = "s" if n > 1 else ""
 

lib/spack/spack/cmd/dev_build.py

@@ -19,7 +19,7 @@
 
 
 def setup_parser(subparser):
-    arguments.add_common_arguments(subparser, ["jobs"])
+    arguments.add_common_arguments(subparser, ["jobs", "no_checksum", "spec"])
     subparser.add_argument(
         "-d",
         "--source-path",
@@ -34,7 +34,6 @@ def setup_parser(subparser):
         dest="ignore_deps",
         help="do not try to install dependencies of requested packages",
     )
-    arguments.add_common_arguments(subparser, ["no_checksum", "deprecated"])
     subparser.add_argument(
         "--keep-prefix",
         action="store_true",
@@ -63,7 +62,6 @@ def setup_parser(subparser):
         choices=["root", "all"],
         help="run tests on only root packages or all packages",
     )
-    arguments.add_common_arguments(subparser, ["spec"])
 
     stop_group = subparser.add_mutually_exclusive_group()
     stop_group.add_argument(
@@ -125,9 +123,6 @@ def dev_build(self, args):
     if args.no_checksum:
         spack.config.set("config:checksum", False, scope="command_line")
 
-    if args.deprecated:
-        spack.config.set("config:deprecated", True, scope="command_line")
-
     tests = False
     if args.test == "all":
         tests = True

lib/spack/spack/cmd/develop.py

@@ -8,6 +8,7 @@
 import llnl.util.tty as tty
 
 import spack.cmd
+import spack.config
 import spack.spec
 import spack.util.path
 import spack.version
@@ -21,6 +22,7 @@
 
 def setup_parser(subparser):
     subparser.add_argument("-p", "--path", help="source location of package")
+    subparser.add_argument("-b", "--build-directory", help="build directory for the package")
 
     clone_group = subparser.add_mutually_exclusive_group()
     clone_group.add_argument(
@@ -151,4 +153,11 @@ def develop(parser, args):
     env = spack.cmd.require_active_env(cmd_name="develop")
     tty.debug("Updating develop config for {0} transactionally".format(env.name))
     with env.write_transaction():
+        if args.build_directory is not None:
+            spack.config.add(
+                "packages:{}:package_attributes:build_directory:{}".format(
+                    spec.name, args.build_directory
+                ),
+                env.scope_name,
+            )
         _update_config(spec, path)

lib/spack/spack/cmd/env.py

@@ -270,7 +270,8 @@ def create_temp_env_directory():
 def _tty_info(msg):
     """tty.info like function that prints the equivalent printf statement for eval."""
     decorated = f'{colorize("@*b{==>}")} {msg}\n'
-    print(f"printf {shlex.quote(decorated)};")
+    executor = "echo" if sys.platform == "win32" else "printf"
+    print(f"{executor} {shlex.quote(decorated)};")
 
 
 def env_activate(args):

lib/spack/spack/cmd/external.py

@@ -18,6 +18,7 @@
 import spack.cray_manifest as cray_manifest
 import spack.detection
 import spack.error
+import spack.repo
 import spack.util.environment
 from spack.cmd.common import arguments
 
@@ -152,9 +153,9 @@ def external_find(args):
 def packages_to_search_for(
     *, names: Optional[List[str]], tags: List[str], exclude: Optional[List[str]]
 ):
-    result = []
-    for current_tag in tags:
-        result.extend(spack.repo.PATH.packages_with_tags(current_tag, full=True))
+    result = list(
+        {pkg for tag in tags for pkg in spack.repo.PATH.packages_with_tags(tag, full=True)}
+    )
 
     if names:
         # Match both fully qualified and unqualified

lib/spack/spack/cmd/fetch.py

@@ -18,7 +18,7 @@
 
 
 def setup_parser(subparser):
-    arguments.add_common_arguments(subparser, ["no_checksum", "deprecated"])
+    arguments.add_common_arguments(subparser, ["no_checksum", "specs"])
     subparser.add_argument(
         "-m",
         "--missing",
@@ -28,7 +28,7 @@ def setup_parser(subparser):
     subparser.add_argument(
         "-D", "--dependencies", action="store_true", help="also fetch all dependencies"
     )
-    arguments.add_common_arguments(subparser, ["specs"])
+    arguments.add_concretizer_args(subparser)
     subparser.epilog = (
         "With an active environment, the specs "
         "parameter can be omitted. In this case all (uninstalled"
@@ -40,9 +40,6 @@ def fetch(parser, args):
     if args.no_checksum:
         spack.config.set("config:checksum", False, scope="command_line")
 
-    if args.deprecated:
-        spack.config.set("config:deprecated", True, scope="command_line")
-
     if args.specs:
         specs = spack.cmd.parse_specs(args.specs, concretize=True)
     else:

lib/spack/spack/cmd/help.py

@@ -30,6 +30,7 @@
       @c{@min:max}                      version range (inclusive)
       @c{@min:}                         version <min> or higher
       @c{@:max}                         up to version <max> (inclusive)
+      @c{@=version}                     exact version
 
     compilers:
       @g{%compiler}                     build with <compiler>

lib/spack/spack/cmd/info.py

@@ -475,10 +475,7 @@ def print_virtuals(pkg, args):
     color.cprint(section_title("Virtual Packages: "))
     if pkg.provided:
         for when, specs in reversed(sorted(pkg.provided.items())):
-            line = "    %s provides %s" % (
-                when.colorized(),
-                ", ".join(s.colorized() for s in specs),
-            )
+            line = "    %s provides %s" % (when.cformat(), ", ".join(s.cformat() for s in specs))
             print(line)
 
     else:

lib/spack/spack/cmd/install.py

@@ -176,7 +176,7 @@ def setup_parser(subparser):
         dest="install_source",
         help="install source files in prefix",
     )
-    arguments.add_common_arguments(subparser, ["no_checksum", "deprecated"])
+    arguments.add_common_arguments(subparser, ["no_checksum"])
     subparser.add_argument(
         "-v",
         "--verbose",
@@ -326,9 +326,6 @@ def install(parser, args):
     if args.no_checksum:
         spack.config.set("config:checksum", False, scope="command_line")
 
-    if args.deprecated:
-        spack.config.set("config:deprecated", True, scope="command_line")
-
     if args.log_file and not args.log_format:
         msg = "the '--log-format' must be specified when using '--log-file'"
         tty.die(msg)

lib/spack/spack/cmd/location.py

@@ -53,6 +53,7 @@ def setup_parser(subparser):
         "-S", "--stages", action="store_true", help="top level stage directory"
     )
     directories.add_argument(
+        "-c",
         "--source-dir",
         action="store_true",
         help="source directory for a spec (requires it to be staged first)",

lib/spack/spack/cmd/mirror.py

@@ -28,7 +28,7 @@
 
 
 def setup_parser(subparser):
-    arguments.add_common_arguments(subparser, ["no_checksum", "deprecated"])
+    arguments.add_common_arguments(subparser, ["no_checksum"])
 
     sp = subparser.add_subparsers(metavar="SUBCOMMAND", dest="mirror_command")
 
@@ -72,6 +72,7 @@ def setup_parser(subparser):
         " retrieve all versions of each package",
     )
     arguments.add_common_arguments(create_parser, ["specs"])
+    arguments.add_concretizer_args(create_parser)
 
     # Destroy
     destroy_parser = sp.add_parser("destroy", help=mirror_destroy.__doc__)
@@ -549,7 +550,4 @@ def mirror(parser, args):
     if args.no_checksum:
         spack.config.set("config:checksum", False, scope="command_line")
 
-    if args.deprecated:
-        spack.config.set("config:deprecated", True, scope="command_line")
-
     action[args.mirror_command](args)

lib/spack/spack/cmd/patch.py

@@ -19,7 +19,7 @@
 
 
 def setup_parser(subparser):
-    arguments.add_common_arguments(subparser, ["no_checksum", "deprecated", "specs"])
+    arguments.add_common_arguments(subparser, ["no_checksum", "specs"])
     arguments.add_concretizer_args(subparser)
 
 
@@ -33,9 +33,6 @@ def patch(parser, args):
     if args.no_checksum:
         spack.config.set("config:checksum", False, scope="command_line")
 
-    if args.deprecated:
-        spack.config.set("config:deprecated", True, scope="command_line")
-
     specs = spack.cmd.parse_specs(args.specs, concretize=False)
     for spec in specs:
         _patch(spack.cmd.matching_spec_from_env(spec).package)

lib/spack/spack/cmd/solve.py

@@ -45,7 +45,9 @@ def setup_parser(subparser):
     arguments.add_common_arguments(subparser, ["long", "very_long", "namespaces"])
 
     install_status_group = subparser.add_mutually_exclusive_group()
-    arguments.add_common_arguments(install_status_group, ["install_status", "no_install_status"])
+    arguments.add_common_arguments(
+        install_status_group, ["install_status", "no_install_status", "debug_nondefaults"]
+    )
 
     subparser.add_argument(
         "-y",
@@ -127,10 +129,7 @@ def _process_result(result, show, required_format, kwargs):
         print()
 
     if result.unsolved_specs and "solutions" in show:
-        tty.msg("Unsolved specs")
-        for spec in result.unsolved_specs:
-            print(spec)
-        print()
+        tty.msg(asp.Result.format_unsolved(result.unsolved_specs))
 
 
 def solve(parser, args):
@@ -148,6 +147,7 @@ def solve(parser, args):
         "show_types": args.types,
         "status_fn": install_status_fn if args.install_status else None,
         "hashes": args.long or args.very_long,
+        "nondefaults": args.debug_nondefaults,
     }
 
     # process output options

lib/spack/spack/cmd/spec.py

@@ -32,7 +32,9 @@ def setup_parser(subparser):
     arguments.add_common_arguments(subparser, ["long", "very_long", "namespaces"])
 
     install_status_group = subparser.add_mutually_exclusive_group()
-    arguments.add_common_arguments(install_status_group, ["install_status", "no_install_status"])
+    arguments.add_common_arguments(
+        install_status_group, ["install_status", "no_install_status", "debug_nondefaults"]
+    )
 
     format_group = subparser.add_mutually_exclusive_group()
     format_group.add_argument(
@@ -84,6 +86,8 @@ def spec(parser, args):
     tree_kwargs = {
         "cover": args.cover,
         "format": fmt,
+        "hashes": args.long or args.very_long,
+        "nondefaults": args.debug_nondefaults,
         "hashlen": None if args.very_long else 7,
         "show_types": args.types,
         "status_fn": install_status_fn if args.install_status else None,
@@ -125,12 +129,14 @@ def spec(parser, args):
             # repeated output. This happens because parse_specs outputs concrete
             # specs for `/hash` inputs.
             if not input.concrete:
-                tree_kwargs["hashes"] = False  # Always False for input spec
+                # NOTE: can use overrides | tree_kwargs in python 3.9+
+                overrides = {"hashes": False}
+                overriden_kwargs = {**overrides, **tree_kwargs}
+
                 print("Input spec")
                 print("--------------------------------")
-                print(input.tree(**tree_kwargs))
+                print(input.tree(**overriden_kwargs))
                 print("Concretized")
                 print("--------------------------------")
 
-            tree_kwargs["hashes"] = args.long or args.very_long
             print(output.tree(**tree_kwargs))

lib/spack/spack/cmd/stage.py

@@ -22,7 +22,7 @@
 
 
 def setup_parser(subparser):
-    arguments.add_common_arguments(subparser, ["no_checksum", "deprecated", "specs"])
+    arguments.add_common_arguments(subparser, ["no_checksum", "specs"])
     subparser.add_argument(
         "-p", "--path", dest="path", help="path to stage package, does not add to spack tree"
     )
@@ -33,9 +33,6 @@ def stage(parser, args):
     if args.no_checksum:
         spack.config.set("config:checksum", False, scope="command_line")
 
-    if args.deprecated:
-        spack.config.set("config:deprecated", True, scope="command_line")
-
     if not args.specs:
         env = ev.active_environment()
         if not env:

lib/spack/spack/cmd/test.py

@@ -228,7 +228,7 @@ def create_reporter(args, specs_to_test, test_suite):
 
 def test_list(args):
     """list installed packages with available tests"""
-    tagged = set(spack.repo.PATH.packages_with_tags(*args.tag)) if args.tag else set()
+    tagged = spack.repo.PATH.packages_with_tags(*args.tag) if args.tag else set()
 
     def has_test_and_tags(pkg_class):
         tests = spack.install_test.test_functions(pkg_class)

lib/spack/spack/compiler.py

@@ -334,6 +334,40 @@ def __init__(
         # used for version checks for API, e.g. C++11 flag
         self._real_version = None
 
+    def __eq__(self, other):
+        return (
+            self.cc == other.cc
+            and self.cxx == other.cxx
+            and self.fc == other.fc
+            and self.f77 == other.f77
+            and self.spec == other.spec
+            and self.operating_system == other.operating_system
+            and self.target == other.target
+            and self.flags == other.flags
+            and self.modules == other.modules
+            and self.environment == other.environment
+            and self.extra_rpaths == other.extra_rpaths
+            and self.enable_implicit_rpaths == other.enable_implicit_rpaths
+        )
+
+    def __hash__(self):
+        return hash(
+            (
+                self.cc,
+                self.cxx,
+                self.fc,
+                self.f77,
+                self.spec,
+                self.operating_system,
+                self.target,
+                str(self.flags),
+                str(self.modules),
+                str(self.environment),
+                str(self.extra_rpaths),
+                self.enable_implicit_rpaths,
+            )
+        )
+
     def verify_executables(self):
         """Raise an error if any of the compiler executables is not valid.
 
@@ -389,8 +423,7 @@ def implicit_rpaths(self):
 
         # Put CXX first since it has the most linking issues
         # And because it has flags that affect linking
-        exe_paths = [x for x in [self.cxx, self.cc, self.fc, self.f77] if x]
-        link_dirs = self._get_compiler_link_paths(exe_paths)
+        link_dirs = self._get_compiler_link_paths()
 
         all_required_libs = list(self.required_libs) + Compiler._all_compiler_rpath_libraries
         return list(paths_containing_libs(link_dirs, all_required_libs))
@@ -403,43 +436,33 @@ def required_libs(self):
         # By default every compiler returns the empty list
         return []
 
-    def _get_compiler_link_paths(self, paths):
-        first_compiler = next((c for c in paths if c), None)
-        if not first_compiler:
-            return []
-        if not self.verbose_flag:
-            # In this case there is no mechanism to learn what link directories
-            # are used by the compiler
+    def _get_compiler_link_paths(self):
+        cc = self.cc if self.cc else self.cxx
+        if not cc or not self.verbose_flag:
+            # Cannot determine implicit link paths without a compiler / verbose flag
             return []
 
         # What flag types apply to first_compiler, in what order
-        flags = ["cppflags", "ldflags"]
-        if first_compiler == self.cc:
-            flags = ["cflags"] + flags
-        elif first_compiler == self.cxx:
-            flags = ["cxxflags"] + flags
+        if cc == self.cc:
+            flags = ["cflags", "cppflags", "ldflags"]
         else:
-            flags.append("fflags")
+            flags = ["cxxflags", "cppflags", "ldflags"]
 
         try:
             tmpdir = tempfile.mkdtemp(prefix="spack-implicit-link-info")
             fout = os.path.join(tmpdir, "output")
             fin = os.path.join(tmpdir, "main.c")
 
-            with open(fin, "w+") as csource:
+            with open(fin, "w") as csource:
                 csource.write(
-                    "int main(int argc, char* argv[]) { " "(void)argc; (void)argv; return 0; }\n"
+                    "int main(int argc, char* argv[]) { (void)argc; (void)argv; return 0; }\n"
                 )
-            compiler_exe = spack.util.executable.Executable(first_compiler)
+            cc_exe = spack.util.executable.Executable(cc)
             for flag_type in flags:
-                for flag in self.flags.get(flag_type, []):
-                    compiler_exe.add_default_arg(flag)
+                cc_exe.add_default_arg(*self.flags.get(flag_type, []))
 
-            output = ""
             with self.compiler_environment():
-                output = str(
-                    compiler_exe(self.verbose_flag, fin, "-o", fout, output=str, error=str)
-                )  # str for py2
+                output = cc_exe(self.verbose_flag, fin, "-o", fout, output=str, error=str)
             return _parse_non_system_link_dirs(output)
         except spack.util.executable.ProcessError as pe:
             tty.debug("ProcessError: Command exited with non-zero status: " + pe.long_message)

lib/spack/spack/compilers/__init__.py

@@ -109,7 +109,7 @@ def _to_dict(compiler):
     return {"compiler": d}
 
 
-def get_compiler_config(scope=None, init_config=True):
+def get_compiler_config(scope=None, init_config=False):
     """Return the compiler configuration for the specified architecture."""
 
     config = spack.config.get("compilers", scope=scope) or []
@@ -118,6 +118,8 @@ def get_compiler_config(scope=None, init_config=True):
 
     merged_config = spack.config.get("compilers")
     if merged_config:
+        # Config is empty for this scope
+        # Do not init config because there is a non-empty scope
         return config
 
     _init_compiler_config(scope=scope)
@@ -125,6 +127,95 @@ def get_compiler_config(scope=None, init_config=True):
     return config
 
 
+def get_compiler_config_from_packages(scope=None):
+    """Return the compiler configuration from packages.yaml"""
+    config = spack.config.get("packages", scope=scope)
+    if not config:
+        return []
+
+    packages = []
+    compiler_package_names = supported_compilers() + list(package_name_to_compiler_name.keys())
+    for name, entry in config.items():
+        if name not in compiler_package_names:
+            continue
+        externals_config = entry.get("externals", None)
+        if not externals_config:
+            continue
+        packages.extend(_compiler_config_from_package_config(externals_config))
+
+    return packages
+
+
+def _compiler_config_from_package_config(config):
+    compilers = []
+    for entry in config:
+        compiler = _compiler_config_from_external(entry)
+        if compiler:
+            compilers.append(compiler)
+
+    return compilers
+
+
+def _compiler_config_from_external(config):
+    spec = spack.spec.parse_with_version_concrete(config["spec"])
+    # use str(spec.versions) to allow `@x.y.z` instead of `@=x.y.z`
+    compiler_spec = spack.spec.CompilerSpec(
+        package_name_to_compiler_name.get(spec.name, spec.name), spec.version
+    )
+
+    extra_attributes = config.get("extra_attributes", {})
+    prefix = config.get("prefix", None)
+
+    compiler_class = class_for_compiler_name(compiler_spec.name)
+    paths = extra_attributes.get("paths", {})
+    compiler_langs = ["cc", "cxx", "fc", "f77"]
+    for lang in compiler_langs:
+        if paths.setdefault(lang, None):
+            continue
+
+        if not prefix:
+            continue
+
+        # Check for files that satisfy the naming scheme for this compiler
+        bindir = os.path.join(prefix, "bin")
+        for f, regex in itertools.product(os.listdir(bindir), compiler_class.search_regexps(lang)):
+            if regex.match(f):
+                paths[lang] = os.path.join(bindir, f)
+
+    if all(v is None for v in paths.values()):
+        return None
+
+    if not spec.architecture:
+        host_platform = spack.platforms.host()
+        operating_system = host_platform.operating_system("default_os")
+        target = host_platform.target("default_target").microarchitecture
+    else:
+        target = spec.target
+        if not target:
+            host_platform = spack.platforms.host()
+            target = host_platform.target("default_target").microarchitecture
+
+        operating_system = spec.os
+        if not operating_system:
+            host_platform = spack.platforms.host()
+            operating_system = host_platform.operating_system("default_os")
+
+    compiler_entry = {
+        "compiler": {
+            "spec": str(compiler_spec),
+            "paths": paths,
+            "flags": extra_attributes.get("flags", {}),
+            "operating_system": str(operating_system),
+            "target": str(target.family),
+            "modules": config.get("modules", []),
+            "environment": extra_attributes.get("environment", {}),
+            "extra_rpaths": extra_attributes.get("extra_rpaths", []),
+            "implicit_rpaths": extra_attributes.get("implicit_rpaths", None),
+        }
+    }
+    return compiler_entry
+
+
 def _init_compiler_config(*, scope):
     """Compiler search used when Spack has no compilers."""
     compilers = find_compilers()
@@ -142,17 +233,20 @@ def compiler_config_files():
         compiler_config = config.get("compilers", scope=name)
         if compiler_config:
             config_files.append(config.get_config_filename(name, "compilers"))
+        compiler_config_from_packages = get_compiler_config_from_packages(scope=name)
+        if compiler_config_from_packages:
+            config_files.append(config.get_config_filename(name, "packages"))
     return config_files
 
 
-def add_compilers_to_config(compilers, scope=None, init_config=True):
+def add_compilers_to_config(compilers, scope=None):
     """Add compilers to the config for the specified architecture.
 
     Arguments:
         compilers: a list of Compiler objects.
         scope: configuration scope to modify.
     """
-    compiler_config = get_compiler_config(scope, init_config)
+    compiler_config = get_compiler_config(scope, init_config=False)
     for compiler in compilers:
         if not compiler.cc:
             tty.debug(f"{compiler.spec} does not have a C compiler")
@@ -184,6 +278,9 @@ def remove_compiler_from_config(compiler_spec, scope=None):
     for current_scope in candidate_scopes:
         removal_happened |= _remove_compiler_from_scope(compiler_spec, scope=current_scope)
 
+    msg = "`spack compiler remove` will not remove compilers defined in packages.yaml"
+    msg += "\nTo remove these compilers, either edit the config or use `spack external remove`"
+    tty.debug(msg)
     return removal_happened
 
 
@@ -198,7 +295,7 @@ def _remove_compiler_from_scope(compiler_spec, scope):
          True if one or more compiler entries were actually removed, False otherwise
     """
     assert scope is not None, "a specific scope is needed when calling this function"
-    compiler_config = get_compiler_config(scope)
+    compiler_config = get_compiler_config(scope, init_config=False)
     filtered_compiler_config = [
         compiler_entry
         for compiler_entry in compiler_config
@@ -221,7 +318,14 @@ def all_compilers_config(scope=None, init_config=True):
     """Return a set of specs for all the compiler versions currently
     available to build with.  These are instances of CompilerSpec.
     """
-    return get_compiler_config(scope, init_config)
+    from_packages_yaml = get_compiler_config_from_packages(scope)
+    if from_packages_yaml:
+        init_config = False
+    from_compilers_yaml = get_compiler_config(scope, init_config)
+
+    result = from_compilers_yaml + from_packages_yaml
+    key = lambda c: _compiler_from_config_entry(c["compiler"])
+    return list(llnl.util.lang.dedupe(result, key=key))
 
 
 def all_compiler_specs(scope=None, init_config=True):
@@ -388,7 +492,7 @@ def find_specs_by_arch(compiler_spec, arch_spec, scope=None, init_config=True):
 
 
 def all_compilers(scope=None, init_config=True):
-    config = get_compiler_config(scope, init_config=init_config)
+    config = all_compilers_config(scope, init_config=init_config)
     compilers = list()
     for items in config:
         items = items["compiler"]
@@ -403,10 +507,7 @@ def compilers_for_spec(
     """This gets all compilers that satisfy the supplied CompilerSpec.
     Returns an empty list if none are found.
     """
-    if use_cache:
-        config = all_compilers_config(scope, init_config)
-    else:
-        config = get_compiler_config(scope, init_config)
+    config = all_compilers_config(scope, init_config)
 
     matches = set(find(compiler_spec, scope, init_config))
     compilers = []
@@ -514,9 +615,10 @@ def get_compilers(config, cspec=None, arch_spec=None):
     for items in config:
         items = items["compiler"]
 
-        # NOTE: in principle this should be equality not satisfies, but config can still
-        # be written in old format gcc@10.1.0 instead of gcc@=10.1.0.
-        if cspec and not cspec.satisfies(items["spec"]):
+        # We might use equality here.
+        if cspec and not spack.spec.parse_with_version_concrete(
+            items["spec"], compiler=True
+        ).satisfies(cspec):
             continue
 
         # If an arch spec is given, confirm that this compiler
@@ -582,9 +684,7 @@ def get_compiler_duplicates(compiler_spec, arch_spec):
 
     scope_to_compilers = {}
     for scope in config.scopes:
-        compilers = compilers_for_spec(
-            compiler_spec, arch_spec=arch_spec, scope=scope, use_cache=False
-        )
+        compilers = compilers_for_spec(compiler_spec, arch_spec=arch_spec, scope=scope)
         if compilers:
             scope_to_compilers[scope] = compilers
 

lib/spack/spack/compilers/msvc.py

@@ -10,6 +10,8 @@
 import tempfile
 from typing import Dict, List, Set
 
+import archspec.cpu
+
 import spack.compiler
 import spack.operating_systems.windows_os
 import spack.platforms
@@ -186,6 +188,9 @@ def __init__(self, *args, **kwargs):
         # get current platform architecture and format for vcvars argument
         arch = spack.platforms.real_host().default.lower()
         arch = arch.replace("-", "_")
+        if str(archspec.cpu.host().family) == "x86_64":
+            arch = "amd64"
+
         self.vcvars_call = VCVarsInvocation(vcvars_script_path, arch, self.msvc_version)
         env_cmds.append(self.vcvars_call)
         # Below is a check for a valid fortran path
@@ -318,7 +323,7 @@ def fc_version(cls, fc):
         fc_path[fc_ver] = fc
         if os.getenv("ONEAPI_ROOT"):
             try:
-                sps = spack.operating_systems.windows_os.WindowsOs.compiler_search_paths
+                sps = spack.operating_systems.windows_os.WindowsOs().compiler_search_paths
             except AttributeError:
                 raise SpackError("Windows compiler search paths not established")
             clp = spack.util.executable.which_string("cl", path=sps)

lib/spack/spack/config.py

@@ -764,6 +764,31 @@ def _add_platform_scope(
     cfg.push_scope(scope_type(plat_name, plat_path))
 
 
+def config_paths_from_entry_points() -> List[Tuple[str, str]]:
+    """Load configuration paths from entry points
+
+    A python package can register entry point metadata so that Spack can find
+    its configuration by adding the following to the project's pyproject.toml:
+
+    .. code-block:: toml
+
+       [project.entry-points."spack.config"]
+       baz = "baz:get_spack_config_path"
+
+    The function ``get_spack_config_path`` returns the path to the package's
+    spack configuration scope
+
+    """
+    config_paths: List[Tuple[str, str]] = []
+    for entry_point in lang.get_entry_points(group="spack.config"):
+        hook = entry_point.load()
+        if callable(hook):
+            config_path = hook()
+            if config_path and os.path.exists(config_path):
+                config_paths.append(("plugin-%s" % entry_point.name, str(config_path)))
+    return config_paths
+
+
 def _add_command_line_scopes(
     cfg: Union[Configuration, lang.Singleton], command_line_scopes: List[str]
 ) -> None:
@@ -816,6 +841,9 @@ def create() -> Configuration:
     # No site-level configs should be checked into spack by default.
     configuration_paths.append(("site", os.path.join(spack.paths.etc_path)))
 
+    # Python package's can register configuration scopes via entry_points
+    configuration_paths.extend(config_paths_from_entry_points())
+
     # User configuration can override both spack defaults and site config
     # This is disabled if user asks for no local configuration.
     if not disable_local_config:

lib/spack/spack/container/images.json

@@ -19,9 +19,6 @@
       },
       "os_package_manager": "dnf",
       "build": "spack/fedora38",
-      "build_tags": {
-        "develop": "latest"
-      },
       "final": {
         "image": "docker.io/fedora:38"
       }
@@ -33,9 +30,6 @@
       },
       "os_package_manager": "dnf",
       "build": "spack/fedora37",
-      "build_tags": {
-        "develop": "latest"
-      },
       "final": {
         "image": "docker.io/fedora:37"
       }
@@ -47,9 +41,6 @@
       },
       "os_package_manager": "dnf_epel",
       "build": "spack/rockylinux9",
-      "build_tags": {
-        "develop": "latest"
-      },
       "final": {
         "image": "docker.io/rockylinux:9"
       }
@@ -61,9 +52,6 @@
       },
       "os_package_manager": "dnf_epel",
       "build": "spack/rockylinux8",
-      "build_tags": {
-        "develop": "latest"
-      },
       "final": {
         "image": "docker.io/rockylinux:8"
       }
@@ -75,9 +63,6 @@
       },
       "os_package_manager": "dnf_epel",
       "build": "spack/almalinux9",
-      "build_tags": {
-        "develop": "latest"
-      },
       "final": {
         "image": "quay.io/almalinuxorg/almalinux:9"
       }
@@ -89,9 +74,6 @@
       },
       "os_package_manager": "dnf_epel",
       "build": "spack/almalinux8",
-      "build_tags": {
-        "develop": "latest"
-      },
       "final": {
         "image": "quay.io/almalinuxorg/almalinux:8"
       }
@@ -105,9 +87,6 @@
       "build": "spack/centos-stream",
       "final": {
         "image": "quay.io/centos/centos:stream"
-      },
-      "build_tags": {
-        "develop": "latest"
       }
     },
     "centos:7": {
@@ -115,10 +94,7 @@
         "template": "container/centos_7.dockerfile"
       },
       "os_package_manager": "yum",
-      "build": "spack/centos7",
-      "build_tags": {
-        "develop": "latest"
-      }
+      "build": "spack/centos7"
     },
     "opensuse/leap:15": {
       "bootstrap": {
@@ -126,9 +102,6 @@
       },
       "os_package_manager": "zypper",
       "build": "spack/leap15",
-      "build_tags": {
-        "develop": "latest"
-      },
       "final": {
         "image": "opensuse/leap:latest"
       }
@@ -148,19 +121,13 @@
         "template": "container/ubuntu_2204.dockerfile"
       },
       "os_package_manager": "apt",
-      "build": "spack/ubuntu-jammy",
-      "build_tags": {
-        "develop": "latest"
-      }
+      "build": "spack/ubuntu-jammy"
     },
     "ubuntu:20.04": {
       "bootstrap": {
         "template": "container/ubuntu_2004.dockerfile"
       },
       "build": "spack/ubuntu-focal",
-      "build_tags": {
-        "develop": "latest"
-      },
       "os_package_manager": "apt"
     },
     "ubuntu:18.04": {
@@ -168,10 +135,7 @@
         "template": "container/ubuntu_1804.dockerfile"
       },
       "os_package_manager": "apt",
-      "build": "spack/ubuntu-bionic",
-      "build_tags": {
-        "develop": "latest"
-      }
+      "build": "spack/ubuntu-bionic"
     }
   },
   "os_package_managers": {

lib/spack/spack/container/images.py

@@ -50,10 +50,7 @@ def build_info(image, spack_version):
     if not build_image:
         return None, None
 
-    # Translate version from git to docker if necessary
-    build_tag = image_data["build_tags"].get(spack_version, spack_version)
-
-    return build_image, build_tag
+    return build_image, spack_version
 
 
 def os_package_manager_for(image):

lib/spack/spack/cray_manifest.py

@@ -227,7 +227,7 @@ def read(path, apply_updates):
     if apply_updates and compilers:
         for compiler in compilers:
             try:
-                spack.compilers.add_compilers_to_config([compiler], init_config=False)
+                spack.compilers.add_compilers_to_config([compiler])
             except Exception:
                 warnings.warn(
                     f"Could not add compiler {str(compiler.spec)}: "

lib/spack/spack/database.py

@@ -1687,7 +1687,11 @@ def root(key, record):
         with self.read_transaction():
             roots = [rec.spec for key, rec in self._data.items() if root(key, rec)]
             needed = set(id(spec) for spec in tr.traverse_nodes(roots, deptype=deptype))
-            return [rec.spec for rec in self._data.values() if id(rec.spec) not in needed]
+            return [
+                rec.spec
+                for rec in self._data.values()
+                if id(rec.spec) not in needed and rec.installed
+            ]
 
     def update_explicit(self, spec, explicit):
         """

lib/spack/spack/directives.py

@@ -34,7 +34,7 @@ class OpenMpi(Package):
 import functools
 import os.path
 import re
-from typing import Any, Callable, List, Optional, Set, Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, List, Optional, Set, Tuple, Union
 
 import llnl.util.lang
 import llnl.util.tty.color
@@ -57,6 +57,9 @@ class OpenMpi(Package):
     VersionLookupError,
 )
 
+if TYPE_CHECKING:
+    import spack.package_base
+
 __all__ = [
     "DirectiveError",
     "DirectiveMeta",
@@ -349,6 +352,7 @@ def remove_directives(arg):
         return _decorator
 
 
+SubmoduleCallback = Callable[["spack.package_base.PackageBase"], Union[str, List[str], bool]]
 directive = DirectiveMeta.directive
 
 
@@ -380,7 +384,7 @@ def version(
     tag: Optional[str] = None,
     branch: Optional[str] = None,
     get_full_repo: Optional[bool] = None,
-    submodules: Optional[bool] = None,
+    submodules: Union[SubmoduleCallback, Optional[bool]] = None,
     submodules_delete: Optional[bool] = None,
     # other version control
     svn: Optional[str] = None,
@@ -656,6 +660,7 @@ def patch(
     level: int = 1,
     when: WhenType = None,
     working_dir: str = ".",
+    reverse: bool = False,
     sha256: Optional[str] = None,
     archive_sha256: Optional[str] = None,
 ) -> Patcher:
@@ -669,10 +674,10 @@ def patch(
         level: patch level (as in the patch shell command)
         when: optional anonymous spec that specifies when to apply the patch
         working_dir: dir to change to before applying
+        reverse: reverse the patch
         sha256: sha256 sum of the patch, used to verify the patch (only required for URL patches)
         archive_sha256: sha256 sum of the *archive*, if the patch is compressed (only required for
             compressed URL patches)
-
     """
 
     def _execute_patch(pkg_or_dep: Union["spack.package_base.PackageBase", Dependency]):
@@ -699,18 +704,22 @@ def _execute_patch(pkg_or_dep: Union["spack.package_base.PackageBase", Dependenc
 
         patch: spack.patch.Patch
         if "://" in url_or_filename:
+            if sha256 is None:
+                raise ValueError("patch() with a url requires a sha256")
+
             patch = spack.patch.UrlPatch(
                 pkg,
                 url_or_filename,
                 level,
-                working_dir,
+                working_dir=working_dir,
+                reverse=reverse,
                 ordering_key=ordering_key,
                 sha256=sha256,
                 archive_sha256=archive_sha256,
             )
         else:
             patch = spack.patch.FilePatch(
-                pkg, url_or_filename, level, working_dir, ordering_key=ordering_key
+                pkg, url_or_filename, level, working_dir, reverse, ordering_key=ordering_key
             )
 
         cur_patches.append(patch)

lib/spack/spack/environment/environment.py

@@ -626,14 +626,13 @@ def view(self, new: Optional[str] = None) -> SimpleFilesystemView:
             new: If a string, create a FilesystemView rooted at that path. Default None. This
                 should only be used to regenerate the view, and cannot be used to access specs.
         """
-        root = new if new else self._current_root
-        if not root:
+        path = new if new else self._current_root
+        if not path:
             # This can only be hit if we write a future bug
             raise SpackEnvironmentViewError(
-                "Attempting to get nonexistent view from environment. "
-                f"View root is at {self.root}"
+                f"Attempting to get nonexistent view from environment. View root is at {self.root}"
             )
-        return self._view(root)
+        return self._view(path)
 
     def _view(self, root: str) -> SimpleFilesystemView:
         """Returns a view object for a given root dir."""
@@ -678,7 +677,9 @@ def specs_for_view(self, concrete_roots: List[Spec]) -> List[Spec]:
 
         # Filter selected, installed specs
         with spack.store.STORE.db.read_transaction():
-            return [s for s in specs if s in self and s.installed]
+            result = [s for s in specs if s in self and s.installed]
+
+        return self._exclude_duplicate_runtimes(result)
 
     def regenerate(self, concrete_roots: List[Spec]) -> None:
         specs = self.specs_for_view(concrete_roots)
@@ -765,6 +766,16 @@ def regenerate(self, concrete_roots: List[Spec]) -> None:
                 msg += str(e)
                 tty.warn(msg)
 
+    def _exclude_duplicate_runtimes(self, nodes):
+        all_runtimes = spack.repo.PATH.packages_with_tags("runtime")
+        runtimes_by_name = {}
+        for s in nodes:
+            if s.name not in all_runtimes:
+                continue
+            current_runtime = runtimes_by_name.get(s.name, s)
+            runtimes_by_name[s.name] = max(current_runtime, s, key=lambda x: x.version)
+        return [x for x in nodes if x.name not in all_runtimes or runtimes_by_name[x.name] == x]
+
 
 def _create_environment(path):
     return Environment(path)
@@ -1416,7 +1427,7 @@ def _concretize_separately(self, tests=False):
 
         # Ensure we have compilers in compilers.yaml to avoid that
         # processes try to write the config file in parallel
-        _ = spack.compilers.get_compiler_config()
+        _ = spack.compilers.get_compiler_config(init_config=True)
 
         # Early return if there is nothing to do
         if len(args) == 0:
@@ -1485,44 +1496,6 @@ def _concretize_separately(self, tests=False):
         ]
         return results
 
-    def concretize_and_add(self, user_spec, concrete_spec=None, tests=False):
-        """Concretize and add a single spec to the environment.
-
-        Concretize the provided ``user_spec`` and add it along with the
-        concretized result to the environment. If the given ``user_spec`` was
-        already present in the environment, this does not add a duplicate.
-        The concretized spec will be added unless the ``user_spec`` was
-        already present and an associated concrete spec was already present.
-
-        Args:
-            concrete_spec: if provided, then it is assumed that it is the
-                result of concretizing the provided ``user_spec``
-        """
-        if self.unify is True:
-            msg = (
-                "cannot install a single spec in an environment that is "
-                "configured to be concretized together. Run instead:\n\n"
-                "    $ spack add <spec>\n"
-                "    $ spack install\n"
-            )
-            raise SpackEnvironmentError(msg)
-
-        spec = Spec(user_spec)
-
-        if self.add(spec):
-            concrete = concrete_spec or spec.concretized(tests=tests)
-            self._add_concrete_spec(spec, concrete)
-        else:
-            # spec might be in the user_specs, but not installed.
-            # TODO: Redo name-based comparison for old style envs
-            spec = next(s for s in self.user_specs if s.satisfies(user_spec))
-            concrete = self.specs_by_hash.get(spec.dag_hash())
-            if not concrete:
-                concrete = spec.concretized(tests=tests)
-                self._add_concrete_spec(spec, concrete)
-
-        return concrete
-
     @property
     def default_view(self):
         if not self.has_view(default_view_name):
@@ -2090,7 +2063,6 @@ def write(self, regenerate: bool = True) -> None:
 
         if regenerate:
             self.regenerate_views()
-            spack.hooks.post_env_write(self)
 
         self.new_specs.clear()
 

lib/spack/spack/extensions.py

@@ -12,6 +12,7 @@
 import re
 import sys
 import types
+from pathlib import Path
 from typing import List
 
 import llnl.util.lang
@@ -132,10 +133,38 @@ def load_extension(name: str) -> str:
 def get_extension_paths():
     """Return the list of canonicalized extension paths from config:extensions."""
     extension_paths = spack.config.get("config:extensions") or []
+    extension_paths.extend(extension_paths_from_entry_points())
     paths = [spack.util.path.canonicalize_path(p) for p in extension_paths]
     return paths
 
 
+def extension_paths_from_entry_points() -> List[str]:
+    """Load extensions from a Python package's entry points.
+
+    A python package can register entry point metadata so that Spack can find
+    its extensions by adding the following to the project's pyproject.toml:
+
+    .. code-block:: toml
+
+       [project.entry-points."spack.extensions"]
+       baz = "baz:get_spack_extensions"
+
+    The function ``get_spack_extensions`` returns paths to the package's
+    spack extensions
+
+    """
+    extension_paths: List[str] = []
+    for entry_point in llnl.util.lang.get_entry_points(group="spack.extensions"):
+        hook = entry_point.load()
+        if callable(hook):
+            paths = hook() or []
+            if isinstance(paths, (Path, str)):
+                extension_paths.append(str(paths))
+            else:
+                extension_paths.extend(paths)
+    return extension_paths
+
+
 def get_command_paths():
     """Return the list of paths where to search for command files."""
     command_paths = []

lib/spack/spack/fetch_strategy.py

@@ -929,9 +929,12 @@ def clone(self, dest=None, commit=None, branch=None, tag=None, bare=False):
         git_commands = []
         submodules = self.submodules
         if callable(submodules):
-            submodules = list(submodules(self.package))
-            git_commands.append(["submodule", "init", "--"] + submodules)
-            git_commands.append(["submodule", "update", "--recursive"])
+            submodules = submodules(self.package)
+            if submodules:
+                if isinstance(submodules, str):
+                    submodules = [submodules]
+                git_commands.append(["submodule", "init", "--"] + submodules)
+                git_commands.append(["submodule", "update", "--recursive"])
         elif submodules:
             git_commands.append(["submodule", "update", "--init", "--recursive"])
 

lib/spack/spack/hooks/__init__.py

@@ -15,13 +15,6 @@
     * post_install(spec, explicit)
     * pre_uninstall(spec)
     * post_uninstall(spec)
-    * on_install_start(spec)
-    * on_install_success(spec)
-    * on_install_failure(spec)
-    * on_phase_success(pkg, phase_name, log_file)
-    * on_phase_error(pkg, phase_name, log_file)
-    * on_phase_error(pkg, phase_name, log_file)
-    * post_env_write(env)
 
 This can be used to implement support for things like module
 systems (e.g. modules, lmod, etc.) or to add other custom
@@ -78,17 +71,5 @@ def __call__(self, *args, **kwargs):
 pre_install = _HookRunner("pre_install")
 post_install = _HookRunner("post_install")
 
-# These hooks are run within an install subprocess
 pre_uninstall = _HookRunner("pre_uninstall")
 post_uninstall = _HookRunner("post_uninstall")
-on_phase_success = _HookRunner("on_phase_success")
-on_phase_error = _HookRunner("on_phase_error")
-
-# These are hooks in installer.py, before starting install subprocess
-on_install_start = _HookRunner("on_install_start")
-on_install_success = _HookRunner("on_install_success")
-on_install_failure = _HookRunner("on_install_failure")
-on_install_cancel = _HookRunner("on_install_cancel")
-
-# Environment hooks
-post_env_write = _HookRunner("post_env_write")

lib/spack/spack/installer.py

@@ -1705,7 +1705,6 @@ def _install_task(self, task: BuildTask, install_status: InstallStatus) -> None:
         except spack.build_environment.StopPhase as e:
             # A StopPhase exception means that do_install was asked to
             # stop early from clients, and is not an error at this point
-            spack.hooks.on_install_failure(task.request.pkg.spec)
             pid = f"{self.pid}: " if tty.show_pid() else ""
             tty.debug(f"{pid}{str(e)}")
             tty.debug(f"Package stage directory: {pkg.stage.source_path}")
@@ -2011,7 +2010,6 @@ def install(self) -> None:
             if task is None:
                 continue
 
-            spack.hooks.on_install_start(task.request.pkg.spec)
             install_args = task.request.install_args
             keep_prefix = install_args.get("keep_prefix")
 
@@ -2037,9 +2035,6 @@ def install(self) -> None:
                     tty.warn(f"{pkg_id} does NOT actually have any uninstalled deps left")
                 dep_str = "dependencies" if task.priority > 1 else "dependency"
 
-                # Hook to indicate task failure, but without an exception
-                spack.hooks.on_install_failure(task.request.pkg.spec)
-
                 raise InstallError(
                     f"Cannot proceed with {pkg_id}: {task.priority} uninstalled "
                     f"{dep_str}: {','.join(task.uninstalled_deps)}",
@@ -2062,11 +2057,6 @@ def install(self) -> None:
                 tty.warn(f"{pkg_id} failed to install")
                 self._update_failed(task)
 
-                # Mark that the package failed
-                # TODO: this should also be for the task.pkg, but we don't
-                # model transitive yet.
-                spack.hooks.on_install_failure(task.request.pkg.spec)
-
                 if self.fail_fast:
                     raise InstallError(fail_fast_err, pkg=pkg)
 
@@ -2169,7 +2159,6 @@ def install(self) -> None:
                 tty.error(
                     f"Failed to install {pkg.name} due to " f"{exc.__class__.__name__}: {str(exc)}"
                 )
-                spack.hooks.on_install_cancel(task.request.pkg.spec)
                 raise
 
             except binary_distribution.NoChecksumException as exc:
@@ -2188,7 +2177,6 @@ def install(self) -> None:
 
             except (Exception, SystemExit) as exc:
                 self._update_failed(task, True, exc)
-                spack.hooks.on_install_failure(task.request.pkg.spec)
 
                 # Best effort installs suppress the exception and mark the
                 # package as a failure.
@@ -2372,9 +2360,6 @@ def run(self) -> bool:
         _print_timer(pre=self.pre, pkg_id=self.pkg_id, timer=self.timer)
         _print_installed_pkg(self.pkg.prefix)
 
-        # Send final status that install is successful
-        spack.hooks.on_install_success(self.pkg.spec)
-
         # preserve verbosity across runs
         return self.echo
 
@@ -2453,15 +2438,10 @@ def _real_install(self) -> None:
                         # Catch any errors to report to logging
                         self.timer.start(phase_fn.name)
                         phase_fn.execute()
-                        spack.hooks.on_phase_success(pkg, phase_fn.name, log_file)
                         self.timer.stop(phase_fn.name)
 
                 except BaseException:
                     combine_phase_logs(pkg.phase_log_files, pkg.log_path)
-                    spack.hooks.on_phase_error(pkg, phase_fn.name, log_file)
-
-                    # phase error indicates install error
-                    spack.hooks.on_install_failure(pkg.spec)
                     raise
 
                 # We assume loggers share echo True/False

lib/spack/spack/main.py

@@ -950,14 +950,10 @@ def _main(argv=None):
         parser.print_help()
         return 1
 
-    # -h, -H, and -V are special as they do not require a command, but
-    # all the other options do nothing without a command.
+    # version is special as it does not require a command or loading and additional infrastructure
     if args.version:
         print(get_version())
         return 0
-    elif args.help:
-        sys.stdout.write(parser.format_help(level=args.help))
-        return 0
 
     # ------------------------------------------------------------------------
     # This part of the `main()` sets up Spack's configuration.
@@ -996,6 +992,12 @@ def _main(argv=None):
         print_setup_info(*args.print_shell_vars.split(","))
         return 0
 
+    # -h and -H are special as they do not require a command, but
+    # all the other options do nothing without a command.
+    if args.help:
+        sys.stdout.write(parser.format_help(level=args.help))
+        return 0
+
     # At this point we've considered all the options to spack itself, so we
     # need a command or we're done.
     if not args.command:
@@ -1038,9 +1040,9 @@ def finish_parse_and_run(parser, cmd_name, main_args, env_format_error):
     set_working_dir()
 
     # now we can actually execute the command.
-    if args.spack_profile or args.sorted_profile:
+    if main_args.spack_profile or main_args.sorted_profile:
         _profile_wrapper(command, parser, args, unknown)
-    elif args.pdb:
+    elif main_args.pdb:
         import pdb
 
         pdb.runctx("_invoke_command(command, parser, args, unknown)", globals(), locals())

lib/spack/spack/modules/common.py

@@ -121,43 +121,26 @@ def update_dictionary_extending_lists(target, update):
             target[key] = update[key]
 
 
-def dependencies(spec, request="all"):
-    """Returns the list of dependent specs for a given spec, according to the
-    request passed as parameter.
+def dependencies(spec: spack.spec.Spec, request: str = "all") -> List[spack.spec.Spec]:
+    """Returns the list of dependent specs for a given spec.
 
     Args:
         spec: spec to be analyzed
-        request: either 'none', 'direct' or 'all'
+        request: one of "none", "run", "direct", "all"
 
     Returns:
-        list of dependencies
-
-        The return list will be empty if request is 'none', will contain
-        the direct dependencies if request is 'direct', or the entire DAG
-        if request is 'all'.
+        list of requested dependencies
     """
-    if request not in ("none", "direct", "all"):
-        message = "Wrong value for argument 'request' : "
-        message += "should be one of ('none', 'direct', 'all')"
-        raise tty.error(message + " [current value is '%s']" % request)
-
     if request == "none":
         return []
+    elif request == "run":
+        return spec.dependencies(deptype=dt.RUN)
+    elif request == "direct":
+        return spec.dependencies(deptype=dt.RUN | dt.LINK)
+    elif request == "all":
+        return list(spec.traverse(order="topo", deptype=dt.LINK | dt.RUN, root=False))
 
-    if request == "direct":
-        return spec.dependencies(deptype=("link", "run"))
-
-    # FIXME : during module file creation nodes seem to be visited multiple
-    # FIXME : times even if cover='nodes' is given. This work around permits
-    # FIXME : to get a unique list of spec anyhow. Do we miss a merge
-    # FIXME : step among nodes that refer to the same package?
-    seen = set()
-    seen_add = seen.add
-    deps = sorted(
-        spec.traverse(order="post", cover="nodes", deptype=("link", "run"), root=False),
-        reverse=True,
-    )
-    return [d for d in deps if not (d in seen or seen_add(d))]
+    raise ValueError(f'request "{request}" is not one of "none", "direct", "run", "all"')
 
 
 def merge_config_rules(configuration, spec):

lib/spack/spack/oci/oci.py

@@ -161,7 +161,7 @@ def upload_blob(
 
 def upload_manifest(
     ref: ImageReference,
-    oci_manifest: dict,
+    manifest: dict,
     tag: bool = True,
     _urlopen: spack.oci.opener.MaybeOpen = None,
 ):
@@ -169,7 +169,7 @@ def upload_manifest(
 
     Args:
         ref: The image reference.
-        oci_manifest: The OCI manifest or index.
+        manifest: The manifest or index.
         tag: When true, use the tag, otherwise use the digest,
             this is relevant for multi-arch images, where the
             tag is an index, referencing the manifests by digest.
@@ -179,7 +179,7 @@ def upload_manifest(
     """
     _urlopen = _urlopen or spack.oci.opener.urlopen
 
-    data = json.dumps(oci_manifest, separators=(",", ":")).encode()
+    data = json.dumps(manifest, separators=(",", ":")).encode()
     digest = Digest.from_sha256(hashlib.sha256(data).hexdigest())
     size = len(data)
 
@@ -190,7 +190,7 @@ def upload_manifest(
         url=ref.manifest_url(),
         method="PUT",
         data=data,
-        headers={"Content-Type": oci_manifest["mediaType"]},
+        headers={"Content-Type": manifest["mediaType"]},
     )
 
     response = _urlopen(request)

lib/spack/spack/operating_systems/windows_os.py

@@ -9,6 +9,8 @@
 import platform
 import subprocess
 
+from llnl.util import tty
+
 from spack.error import SpackError
 from spack.util import windows_registry as winreg
 from spack.version import Version
@@ -83,11 +85,50 @@ def compiler_search_paths(self):
                     os.path.join(str(os.getenv("ONEAPI_ROOT")), "compiler", "*", "windows", "bin")
                 )
             )
+
         # Second strategy: Find MSVC via the registry
-        msft = winreg.WindowsRegistryView(
-            "SOFTWARE\\WOW6432Node\\Microsoft", winreg.HKEY.HKEY_LOCAL_MACHINE
-        )
-        vs_entries = msft.find_subkeys(r"VisualStudio_.*")
+        def try_query_registry(retry=False):
+            winreg_report_error = lambda e: tty.debug(
+                'Windows registry query on "SOFTWARE\\WOW6432Node\\Microsoft"'
+                f"under HKEY_LOCAL_MACHINE: {str(e)}"
+            )
+            try:
+                # Registry interactions are subject to race conditions, etc and can generally
+                # be flakey, do this in a catch block to prevent reg issues from interfering
+                # with compiler detection
+                msft = winreg.WindowsRegistryView(
+                    "SOFTWARE\\WOW6432Node\\Microsoft", winreg.HKEY.HKEY_LOCAL_MACHINE
+                )
+                return msft.find_subkeys(r"VisualStudio_.*", recursive=False)
+            except OSError as e:
+                # OSErrors propagated into caller by Spack's registry module are expected
+                # and indicate a known issue with the registry query
+                # i.e. user does not have permissions or the key/value
+                # doesn't exist
+                winreg_report_error(e)
+                return []
+            except winreg.InvalidRegistryOperation as e:
+                # Other errors raised by the Spack's reg module indicate
+                # an unexpected error type, and are handled specifically
+                # as the underlying cause is difficult/impossible to determine
+                # without manually exploring the registry
+                # These errors can also be spurious (race conditions)
+                # and may resolve on re-execution of the query
+                # or are permanent (specific types of permission issues)
+                # but the registry raises the same exception for all types of
+                # atypical errors
+                if retry:
+                    winreg_report_error(e)
+                return []
+
+        vs_entries = try_query_registry()
+        if not vs_entries:
+            # Occasional spurious race conditions can arise when reading the MS reg
+            # typically these race conditions resolve immediately and we can safely
+            # retry the reg query without waiting
+            # Note: Winreg does not support locking
+            vs_entries = try_query_registry(retry=True)
+
         vs_paths = []
 
         def clean_vs_path(path):
@@ -99,11 +140,8 @@ def clean_vs_path(path):
                 val = entry.get_subkey("Capabilities").get_value("ApplicationDescription").value
                 vs_paths.append(clean_vs_path(val))
             except FileNotFoundError as e:
-                if hasattr(e, "winerror"):
-                    if e.winerror == 2:
-                        pass
-                    else:
-                        raise
+                if hasattr(e, "winerror") and e.winerror == 2:
+                    pass
                 else:
                     raise
 

lib/spack/spack/package_base.py

@@ -64,7 +64,7 @@
     install_test_root,
 )
 from spack.installer import InstallError, PackageInstaller
-from spack.stage import DIYStage, ResourceStage, Stage, StageComposite, compute_stage_name
+from spack.stage import DevelopStage, ResourceStage, Stage, StageComposite, compute_stage_name
 from spack.util.executable import ProcessError, which
 from spack.util.package_hash import package_hash
 from spack.version import GitVersion, StandardVersion
@@ -566,6 +566,7 @@ class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta):
     provided: Dict["spack.spec.Spec", Set["spack.spec.Spec"]]
     provided_together: Dict["spack.spec.Spec", List[Set[str]]]
     patches: Dict["spack.spec.Spec", List["spack.patch.Patch"]]
+    variants: Dict[str, Tuple["spack.variant.Variant", "spack.spec.Spec"]]
 
     #: By default, packages are not virtual
     #: Virtual packages override this attribute
@@ -1074,7 +1075,12 @@ def _make_stage(self):
         # If it's a dev package (not transitively), use a DIY stage object
         dev_path_var = self.spec.variants.get("dev_path", None)
         if dev_path_var:
-            return DIYStage(dev_path_var.value)
+            dev_path = dev_path_var.value
+            link_format = spack.config.get("config:develop_stage_link")
+            if not link_format:
+                link_format = "build-{arch}-{hash:7}"
+            stage_link = self.spec.format_path(link_format)
+            return DevelopStage(compute_stage_name(self.spec), dev_path, stage_link)
 
         # To fetch the current version
         source_stage = self._make_root_stage(self.fetcher)
@@ -1406,7 +1412,7 @@ def do_fetch(self, mirror_only=False):
             return
 
         checksum = spack.config.get("config:checksum")
-        fetch = self.stage.managed_by_spack
+        fetch = self.stage.needs_fetching
         if (
             checksum
             and fetch
@@ -1479,9 +1485,6 @@ def do_stage(self, mirror_only=False):
         if self.has_code:
             self.do_fetch(mirror_only)
             self.stage.expand_archive()
-
-            if not os.listdir(self.stage.path):
-                raise spack.error.FetchError("Archive was empty for %s" % self.name)
         else:
             # Support for post-install hooks requires a stage.source_path
             fsys.mkdirp(self.stage.source_path)
@@ -1515,7 +1518,7 @@ def do_patch(self):
         # If we encounter an archive that failed to patch, restage it
         # so that we can apply all the patches again.
         if os.path.isfile(bad_file):
-            if self.stage.managed_by_spack:
+            if self.stage.requires_patch_success:
                 tty.debug("Patching failed last time. Restaging.")
                 self.stage.restage()
             else:
@@ -1536,6 +1539,8 @@ def do_patch(self):
             tty.msg("No patches needed for {0}".format(self.name))
             return
 
+        errors = []
+
         # Apply all the patches for specs that match this one
         patched = False
         for patch in patches:
@@ -1545,12 +1550,16 @@ def do_patch(self):
                 tty.msg("Applied patch {0}".format(patch.path_or_url))
                 patched = True
             except spack.error.SpackError as e:
-                tty.debug(e)
-
                 # Touch bad file if anything goes wrong.
-                tty.msg("Patch %s failed." % patch.path_or_url)
                 fsys.touch(bad_file)
-                raise
+                error_msg = f"Patch {patch.path_or_url} failed."
+                if self.stage.requires_patch_success:
+                    tty.msg(error_msg)
+                    raise
+                else:
+                    tty.debug(error_msg)
+                    tty.debug(e)
+                    errors.append(e)
 
         if has_patch_fun:
             try:
@@ -1568,24 +1577,29 @@ def do_patch(self):
                     # printed a message for each patch.
                     tty.msg("No patches needed for {0}".format(self.name))
             except spack.error.SpackError as e:
-                tty.debug(e)
-
                 # Touch bad file if anything goes wrong.
-                tty.msg("patch() function failed for {0}".format(self.name))
                 fsys.touch(bad_file)
-                raise
+                error_msg = f"patch() function failed for {self.name}"
+                if self.stage.requires_patch_success:
+                    tty.msg(error_msg)
+                    raise
+                else:
+                    tty.debug(error_msg)
+                    tty.debug(e)
+                    errors.append(e)
 
-        # Get rid of any old failed file -- patches have either succeeded
-        # or are not needed.  This is mostly defensive -- it's needed
-        # if the restage() method doesn't clean *everything* (e.g., for a repo)
-        if os.path.isfile(bad_file):
-            os.remove(bad_file)
+        if not errors:
+            # Get rid of any old failed file -- patches have either succeeded
+            # or are not needed.  This is mostly defensive -- it's needed
+            # if we didn't restage
+            if os.path.isfile(bad_file):
+                os.remove(bad_file)
 
-        # touch good or no patches file so that we skip next time.
-        if patched:
-            fsys.touch(good_file)
-        else:
-            fsys.touch(no_patches_file)
+            # touch good or no patches file so that we skip next time.
+            if patched:
+                fsys.touch(good_file)
+            else:
+                fsys.touch(no_patches_file)
 
     @classmethod
     def all_patches(cls):

lib/spack/spack/patch.py

@@ -9,9 +9,9 @@
 import os.path
 import pathlib
 import sys
+from typing import Any, Dict, Optional, Tuple, Type
 
 import llnl.util.filesystem
-import llnl.util.lang
 from llnl.url import allowed_archive
 
 import spack
@@ -25,15 +25,21 @@
 from spack.util.executable import which, which_string
 
 
-def apply_patch(stage, patch_path, level=1, working_dir="."):
+def apply_patch(
+    stage: "spack.stage.Stage",
+    patch_path: str,
+    level: int = 1,
+    working_dir: str = ".",
+    reverse: bool = False,
+) -> None:
     """Apply the patch at patch_path to code in the stage.
 
     Args:
-        stage (spack.stage.Stage): stage with code that will be patched
-        patch_path (str): filesystem location for the patch to apply
-        level (int or None): patch level (default 1)
-        working_dir (str): relative path *within* the stage to change to
-            (default '.')
+        stage: stage with code that will be patched
+        patch_path: filesystem location for the patch to apply
+        level: patch level
+        working_dir: relative path *within* the stage to change to
+        reverse: reverse the patch
     """
     git_utils_path = os.environ.get("PATH", "")
     if sys.platform == "win32":
@@ -44,6 +50,10 @@ def apply_patch(stage, patch_path, level=1, working_dir="."):
             git_root = git_root / "usr" / "bin"
             git_utils_path = os.pathsep.join([str(git_root), git_utils_path])
 
+    args = ["-s", "-p", str(level), "-i", patch_path, "-d", working_dir]
+    if reverse:
+        args.append("-R")
+
     # TODO: Decouple Spack's patch support on Windows from Git
     # for Windows, and instead have Spack directly fetch, install, and
     # utilize that patch.
@@ -52,22 +62,36 @@ def apply_patch(stage, patch_path, level=1, working_dir="."):
     # flag is passed.
     patch = which("patch", required=True, path=git_utils_path)
     with llnl.util.filesystem.working_dir(stage.source_path):
-        patch("-s", "-p", str(level), "-i", patch_path, "-d", working_dir)
+        patch(*args)
 
 
 class Patch:
     """Base class for patches.
 
-    Arguments:
-        pkg (str): the package that owns the patch
-
     The owning package is not necessarily the package to apply the patch
     to -- in the case where a dependent package patches its dependency,
     it is the dependent's fullname.
-
     """
 
-    def __init__(self, pkg, path_or_url, level, working_dir):
+    sha256: str
+
+    def __init__(
+        self,
+        pkg: "spack.package_base.PackageBase",
+        path_or_url: str,
+        level: int,
+        working_dir: str,
+        reverse: bool = False,
+    ) -> None:
+        """Initialize a new Patch instance.
+
+        Args:
+            pkg: the package that owns the patch
+            path_or_url: the relative path or URL to a patch file
+            level: patch level
+            working_dir: relative path *within* the stage to change to
+            reverse: reverse the patch
+        """
         # validate level (must be an integer >= 0)
         if not isinstance(level, int) or not level >= 0:
             raise ValueError("Patch level needs to be a non-negative integer.")
@@ -75,59 +99,88 @@ def __init__(self, pkg, path_or_url, level, working_dir):
         # Attributes shared by all patch subclasses
         self.owner = pkg.fullname
         self.path_or_url = path_or_url  # needed for debug output
-        self.path = None  # must be set before apply()
+        self.path: Optional[str] = None  # must be set before apply()
         self.level = level
         self.working_dir = working_dir
+        self.reverse = reverse
 
-    def apply(self, stage: "spack.stage.Stage"):
+    def apply(self, stage: "spack.stage.Stage") -> None:
         """Apply a patch to source in a stage.
 
-        Arguments:
-            stage (spack.stage.Stage): stage where source code lives
+        Args:
+            stage: stage where source code lives
         """
         if not self.path or not os.path.isfile(self.path):
             raise NoSuchPatchError(f"No such patch: {self.path}")
 
-        apply_patch(stage, self.path, self.level, self.working_dir)
+        apply_patch(stage, self.path, self.level, self.working_dir, self.reverse)
 
-    @property
-    def stage(self):
-        return None
+    # TODO: Use TypedDict once Spack supports Python 3.8+ only
+    def to_dict(self) -> Dict[str, Any]:
+        """Dictionary representation of the patch.
 
-    def to_dict(self):
-        """Partial dictionary -- subclases should add to this."""
+        Returns:
+            A dictionary representation.
+        """
         return {
             "owner": self.owner,
             "sha256": self.sha256,
             "level": self.level,
             "working_dir": self.working_dir,
+            "reverse": self.reverse,
         }
 
-    def __eq__(self, other):
+    def __eq__(self, other: object) -> bool:
+        """Equality check.
+
+        Args:
+            other: another patch
+
+        Returns:
+            True if both patches have the same checksum, else False
+        """
+        if not isinstance(other, Patch):
+            return NotImplemented
         return self.sha256 == other.sha256
 
-    def __hash__(self):
+    def __hash__(self) -> int:
+        """Unique hash.
+
+        Returns:
+            A unique hash based on the sha256.
+        """
         return hash(self.sha256)
 
 
 class FilePatch(Patch):
-    """Describes a patch that is retrieved from a file in the repository.
+    """Describes a patch that is retrieved from a file in the repository."""
 
-    Arguments:
-        pkg (str): the class object for the package that owns the patch
-        relative_path (str): path to patch, relative to the repository
-            directory for a package.
-        level (int): level to pass to patch command
-        working_dir (str): path within the source directory where patch
-            should be applied
-    """
+    _sha256: Optional[str] = None
 
-    def __init__(self, pkg, relative_path, level, working_dir, ordering_key=None):
+    def __init__(
+        self,
+        pkg: "spack.package_base.PackageBase",
+        relative_path: str,
+        level: int,
+        working_dir: str,
+        reverse: bool = False,
+        ordering_key: Optional[Tuple[str, int]] = None,
+    ) -> None:
+        """Initialize a new FilePatch instance.
+
+        Args:
+            pkg: the class object for the package that owns the patch
+            relative_path: path to patch, relative to the repository directory for a package.
+            level: level to pass to patch command
+            working_dir: path within the source directory where patch should be applied
+            reverse: reverse the patch
+            ordering_key: key used to ensure patches are applied in a consistent order
+        """
         self.relative_path = relative_path
 
         # patches may be defined by relative paths to parent classes
         # search mro to look for the file
-        abs_path = None
+        abs_path: Optional[str] = None
         # At different times we call FilePatch on instances and classes
         pkg_cls = pkg if inspect.isclass(pkg) else pkg.__class__
         for cls in inspect.getmro(pkg_cls):
@@ -148,52 +201,94 @@ def __init__(self, pkg, relative_path, level, working_dir, ordering_key=None):
             msg += "package %s.%s does not exist." % (pkg.namespace, pkg.name)
             raise ValueError(msg)
 
-        super().__init__(pkg, abs_path, level, working_dir)
+        super().__init__(pkg, abs_path, level, working_dir, reverse)
         self.path = abs_path
-        self._sha256 = None
         self.ordering_key = ordering_key
 
     @property
-    def sha256(self):
-        if self._sha256 is None:
+    def sha256(self) -> str:
+        """Get the patch checksum.
+
+        Returns:
+            The sha256 of the patch file.
+        """
+        if self._sha256 is None and self.path is not None:
             self._sha256 = checksum(hashlib.sha256, self.path)
+        assert isinstance(self._sha256, str)
         return self._sha256
 
-    def to_dict(self):
-        return llnl.util.lang.union_dicts(super().to_dict(), {"relative_path": self.relative_path})
+    @sha256.setter
+    def sha256(self, value: str) -> None:
+        """Set the patch checksum.
+
+        Args:
+            value: the sha256
+        """
+        self._sha256 = value
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Dictionary representation of the patch.
+
+        Returns:
+            A dictionary representation.
+        """
+        data = super().to_dict()
+        data["relative_path"] = self.relative_path
+        return data
 
 
 class UrlPatch(Patch):
-    """Describes a patch that is retrieved from a URL.
+    """Describes a patch that is retrieved from a URL."""
 
-    Arguments:
-        pkg (str): the package that owns the patch
-        url (str): URL where the patch can be fetched
-        level (int): level to pass to patch command
-        working_dir (str): path within the source directory where patch
-            should be applied
-    """
+    def __init__(
+        self,
+        pkg: "spack.package_base.PackageBase",
+        url: str,
+        level: int = 1,
+        *,
+        working_dir: str = ".",
+        reverse: bool = False,
+        sha256: str,  # This is required for UrlPatch
+        ordering_key: Optional[Tuple[str, int]] = None,
+        archive_sha256: Optional[str] = None,
+    ) -> None:
+        """Initialize a new UrlPatch instance.
 
-    def __init__(self, pkg, url, level=1, working_dir=".", ordering_key=None, **kwargs):
-        super().__init__(pkg, url, level, working_dir)
+        Arguments:
+            pkg: the package that owns the patch
+            url: URL where the patch can be fetched
+            level: level to pass to patch command
+            working_dir: path within the source directory where patch should be applied
+            reverse: reverse the patch
+            ordering_key: key used to ensure patches are applied in a consistent order
+            sha256: sha256 sum of the patch, used to verify the patch
+            archive_sha256: sha256 sum of the *archive*, if the patch is compressed
+                (only required for compressed URL patches)
+        """
+        super().__init__(pkg, url, level, working_dir, reverse)
 
         self.url = url
-        self._stage = None
+        self._stage: Optional["spack.stage.Stage"] = None
 
         self.ordering_key = ordering_key
 
-        self.archive_sha256 = kwargs.get("archive_sha256")
-        if allowed_archive(self.url) and not self.archive_sha256:
+        if allowed_archive(self.url) and not archive_sha256:
             raise PatchDirectiveError(
                 "Compressed patches require 'archive_sha256' "
                 "and patch 'sha256' attributes: %s" % self.url
             )
+        self.archive_sha256 = archive_sha256
 
-        self.sha256 = kwargs.get("sha256")
-        if not self.sha256:
+        if not sha256:
             raise PatchDirectiveError("URL patches require a sha256 checksum")
+        self.sha256 = sha256
 
-    def apply(self, stage: "spack.stage.Stage"):
+    def apply(self, stage: "spack.stage.Stage") -> None:
+        """Apply a patch to source in a stage.
+
+        Args:
+            stage: stage where source code lives
+        """
         assert self.stage.expanded, "Stage must be expanded before applying patches"
 
         # Get the patch file.
@@ -204,15 +299,20 @@ def apply(self, stage: "spack.stage.Stage"):
         return super().apply(stage)
 
     @property
-    def stage(self):
+    def stage(self) -> "spack.stage.Stage":
+        """The stage in which to download (and unpack) the URL patch.
+
+        Returns:
+            The stage object.
+        """
         if self._stage:
             return self._stage
 
         fetch_digest = self.archive_sha256 or self.sha256
 
         # Two checksums, one for compressed file, one for its contents
-        if self.archive_sha256:
-            fetcher = fs.FetchAndVerifyExpandedFile(
+        if self.archive_sha256 and self.sha256:
+            fetcher: fs.FetchStrategy = fs.FetchAndVerifyExpandedFile(
                 self.url, archive_sha256=self.archive_sha256, expanded_sha256=self.sha256
             )
         else:
@@ -231,7 +331,12 @@ def stage(self):
         )
         return self._stage
 
-    def to_dict(self):
+    def to_dict(self) -> Dict[str, Any]:
+        """Dictionary representation of the patch.
+
+        Returns:
+            A dictionary representation.
+        """
         data = super().to_dict()
         data["url"] = self.url
         if self.archive_sha256:
@@ -239,8 +344,21 @@ def to_dict(self):
         return data
 
 
-def from_dict(dictionary, repository=None):
-    """Create a patch from json dictionary."""
+def from_dict(
+    dictionary: Dict[str, Any], repository: Optional["spack.repo.RepoPath"] = None
+) -> Patch:
+    """Create a patch from json dictionary.
+
+    Args:
+        dictionary: dictionary representation of a patch
+        repository: repository containing package
+
+    Returns:
+        A patch object.
+
+    Raises:
+        ValueError: If *owner* or *url*/*relative_path* are missing in the dictionary.
+    """
     repository = repository or spack.repo.PATH
     owner = dictionary.get("owner")
     if "owner" not in dictionary:
@@ -252,14 +370,21 @@ def from_dict(dictionary, repository=None):
             pkg_cls,
             dictionary["url"],
             dictionary["level"],
-            dictionary["working_dir"],
+            working_dir=dictionary["working_dir"],
+            # Added in v0.22, fallback required for backwards compatibility
+            reverse=dictionary.get("reverse", False),
             sha256=dictionary["sha256"],
             archive_sha256=dictionary.get("archive_sha256"),
         )
 
     elif "relative_path" in dictionary:
         patch = FilePatch(
-            pkg_cls, dictionary["relative_path"], dictionary["level"], dictionary["working_dir"]
+            pkg_cls,
+            dictionary["relative_path"],
+            dictionary["level"],
+            dictionary["working_dir"],
+            # Added in v0.22, fallback required for backwards compatibility
+            dictionary.get("reverse", False),
         )
 
         # If the patch in the repo changes, we cannot get it back, so we
@@ -267,7 +392,7 @@ def from_dict(dictionary, repository=None):
         # TODO: handle this more gracefully.
         sha256 = dictionary["sha256"]
         checker = Checker(sha256)
-        if not checker.check(patch.path):
+        if patch.path and not checker.check(patch.path):
             raise fs.ChecksumError(
                 "sha256 checksum failed for %s" % patch.path,
                 "Expected %s but got %s " % (sha256, checker.sum)
@@ -295,10 +420,17 @@ class PatchCache:
                 namespace2.package2:
                     <patch json>
                 ... etc. ...
-
     """
 
-    def __init__(self, repository, data=None):
+    def __init__(
+        self, repository: "spack.repo.RepoPath", data: Optional[Dict[str, Any]] = None
+    ) -> None:
+        """Initialize a new PatchCache instance.
+
+        Args:
+            repository: repository containing package
+            data: nested dictionary of patches
+        """
         if data is None:
             self.index = {}
         else:
@@ -309,21 +441,39 @@ def __init__(self, repository, data=None):
         self.repository = repository
 
     @classmethod
-    def from_json(cls, stream, repository):
+    def from_json(cls, stream: Any, repository: "spack.repo.RepoPath") -> "PatchCache":
+        """Initialize a new PatchCache instance from JSON.
+
+        Args:
+            stream: stream of data
+            repository: repository containing package
+
+        Returns:
+            A new PatchCache instance.
+        """
         return PatchCache(repository=repository, data=sjson.load(stream))
 
-    def to_json(self, stream):
+    def to_json(self, stream: Any) -> None:
+        """Dump a JSON representation to a stream.
+
+        Args:
+            stream: stream of data
+        """
         sjson.dump({"patches": self.index}, stream)
 
-    def patch_for_package(self, sha256: str, pkg):
+    def patch_for_package(self, sha256: str, pkg: "spack.package_base.PackageBase") -> Patch:
         """Look up a patch in the index and build a patch object for it.
 
-        Arguments:
-            sha256: sha256 hash to look up
-            pkg (spack.package_base.PackageBase): Package object to get patch for.
-
         We build patch objects lazily because building them requires that
-        we have information about the package's location in its repo."""
+        we have information about the package's location in its repo.
+
+        Args:
+            sha256: sha256 hash to look up
+            pkg: Package object to get patch for.
+
+        Returns:
+            The patch object.
+        """
         sha_index = self.index.get(sha256)
         if not sha_index:
             raise PatchLookupError(
@@ -346,7 +496,12 @@ def patch_for_package(self, sha256: str, pkg):
         patch_dict["sha256"] = sha256
         return from_dict(patch_dict, repository=self.repository)
 
-    def update_package(self, pkg_fullname):
+    def update_package(self, pkg_fullname: str) -> None:
+        """Update the patch cache.
+
+        Args:
+            pkg_fullname: package to update.
+        """
         # remove this package from any patch entries that reference it.
         empty = []
         for sha256, package_to_patch in self.index.items():
@@ -372,14 +527,29 @@ def update_package(self, pkg_fullname):
             p2p = self.index.setdefault(sha256, {})
             p2p.update(package_to_patch)
 
-    def update(self, other):
-        """Update this cache with the contents of another."""
+    def update(self, other: "PatchCache") -> None:
+        """Update this cache with the contents of another.
+
+        Args:
+            other: another patch cache to merge
+        """
         for sha256, package_to_patch in other.index.items():
             p2p = self.index.setdefault(sha256, {})
             p2p.update(package_to_patch)
 
     @staticmethod
-    def _index_patches(pkg_class, repository):
+    def _index_patches(
+        pkg_class: Type["spack.package_base.PackageBase"], repository: "spack.repo.RepoPath"
+    ) -> Dict[Any, Any]:
+        """Patch index for a specific patch.
+
+        Args:
+            pkg_class: package object to get patches for
+            repository: repository containing the package
+
+        Returns:
+            The patch index for that package.
+        """
         index = {}
 
         # Add patches from the class

lib/spack/spack/projections.py

@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import spack.util.path
+
 
 def get_projection(projections, spec):
     """
@@ -11,7 +13,7 @@ def get_projection(projections, spec):
     all_projection = None
     for spec_like, projection in projections.items():
         if spec.satisfies(spec_like):
-            return projection
+            return spack.util.path.substitute_path_variables(projection)
         elif spec_like == "all":
-            all_projection = projection
+            all_projection = spack.util.path.substitute_path_variables(projection)
     return all_projection

lib/spack/spack/repo.py

@@ -25,7 +25,7 @@
 import traceback
 import types
 import uuid
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Dict, List, Set, Tuple, Union
 
 import llnl.path
 import llnl.util.filesystem as fs
@@ -746,19 +746,17 @@ def all_package_paths(self):
         for name in self.all_package_names():
             yield self.package_path(name)
 
-    def packages_with_tags(self, *tags, full=False):
-        """Returns a list of packages matching any of the tags in input.
+    def packages_with_tags(self, *tags: str, full: bool = False) -> Set[str]:
+        """Returns a set of packages matching any of the tags in input.
 
         Args:
             full: if True the package names in the output are fully-qualified
         """
-        r = set()
-        for repo in self.repos:
-            current = repo.packages_with_tags(*tags)
-            if full:
-                current = [f"{repo.namespace}.{x}" for x in current]
-            r |= set(current)
-        return sorted(r)
+        return {
+            f"{repo.namespace}.{pkg}" if full else pkg
+            for repo in self.repos
+            for pkg in repo.packages_with_tags(*tags)
+        }
 
     def all_package_classes(self):
         for name in self.all_package_names():
@@ -1169,15 +1167,10 @@ def all_package_paths(self):
         for name in self.all_package_names():
             yield self.package_path(name)
 
-    def packages_with_tags(self, *tags):
+    def packages_with_tags(self, *tags: str) -> Set[str]:
         v = set(self.all_package_names())
-        index = self.tag_index
-
-        for t in tags:
-            t = t.lower()
-            v &= set(index[t])
-
-        return sorted(v)
+        v.intersection_update(*(self.tag_index[tag.lower()] for tag in tags))
+        return v
 
     def all_package_classes(self):
         """Iterator over all package *classes* in the repository.

lib/spack/spack/schema/__init__.py

@@ -6,7 +6,6 @@
 import warnings
 
 import llnl.util.lang
-import llnl.util.tty
 
 
 # jsonschema is imported lazily as it is heavy to import
@@ -62,25 +61,3 @@ def _deprecated_properties(validator, deprecated, instance, schema):
 
 
 Validator = llnl.util.lang.Singleton(_make_validator)
-
-spec_list_schema = {
-    "type": "array",
-    "default": [],
-    "items": {
-        "anyOf": [
-            {
-                "type": "object",
-                "additionalProperties": False,
-                "properties": {
-                    "matrix": {
-                        "type": "array",
-                        "items": {"type": "array", "items": {"type": "string"}},
-                    },
-                    "exclude": {"type": "array", "items": {"type": "string"}},
-                },
-            },
-            {"type": "string"},
-            {"type": "null"},
-        ]
-    },
-}

lib/spack/spack/schema/config.py

@@ -63,6 +63,7 @@
                 "oneOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}]
             },
             "stage_name": {"type": "string"},
+            "develop_stage_link": {"type": "string"},
             "test_stage": {"type": "string"},
             "extensions": {"type": "array", "items": {"type": "string"}},
             "template_dirs": {"type": "array", "items": {"type": "string"}},

lib/spack/spack/schema/definitions.py

@@ -10,7 +10,7 @@
 """
 from typing import Any, Dict
 
-import spack.schema
+from .spec_list import spec_list_schema
 
 #: Properties for inclusion in other schemas
 properties: Dict[str, Any] = {
@@ -20,7 +20,7 @@
         "items": {
             "type": "object",
             "properties": {"when": {"type": "string"}},
-            "patternProperties": {r"^(?!when$)\w*": spack.schema.spec_list_schema},
+            "patternProperties": {r"^(?!when$)\w*": spec_list_schema},
         },
     }
 }

lib/spack/spack/schema/env.py

@@ -16,11 +16,11 @@
 import spack.schema.merged
 import spack.schema.projections
 
+from .spec_list import spec_list_schema
+
 #: Top level key in a manifest file
 TOP_LEVEL_KEY = "spack"
 
-projections_scheme = spack.schema.projections.properties["projections"]
-
 properties: Dict[str, Any] = {
     "spack": {
         "type": "object",
@@ -34,7 +34,7 @@
             # extra environment schema properties
             {
                 "include": {"type": "array", "default": [], "items": {"type": "string"}},
-                "specs": spack.schema.spec_list_schema,
+                "specs": spec_list_schema,
             },
         ),
     }

lib/spack/spack/schema/modules.py

@@ -34,7 +34,7 @@
 
 dictionary_of_strings = {"type": "object", "patternProperties": {r"\w[\w-]*": {"type": "string"}}}
 
-dependency_selection = {"type": "string", "enum": ["none", "direct", "all"]}
+dependency_selection = {"type": "string", "enum": ["none", "run", "direct", "all"]}
 
 module_file_configuration = {
     "type": "object",

lib/spack/spack/schema/spack.py

@@ -1,46 +0,0 @@
-# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-"""Schema for spack environment
-
-.. literalinclude:: _spack_root/lib/spack/spack/schema/spack.py
-   :lines: 20-
-"""
-from typing import Any, Dict
-
-from llnl.util.lang import union_dicts
-
-import spack.schema
-import spack.schema.gitlab_ci as ci_schema  # DEPRECATED
-import spack.schema.merged as merged_schema
-
-#: Properties for inclusion in other schemas
-properties: Dict[str, Any] = {
-    "spack": {
-        "type": "object",
-        "default": {},
-        "additionalProperties": False,
-        "properties": union_dicts(
-            # Include deprecated "gitlab-ci" section
-            ci_schema.properties,
-            # merged configuration scope schemas
-            merged_schema.properties,
-            # extra environment schema properties
-            {
-                "include": {"type": "array", "default": [], "items": {"type": "string"}},
-                "specs": spack.schema.spec_list_schema,
-            },
-        ),
-    }
-}
-
-#: Full schema with metadata
-schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "Spack environment file schema",
-    "type": "object",
-    "additionalProperties": False,
-    "properties": properties,
-}