Set version to v0.22.3

This changes `Spec` serialization to include information about propagation for abstract specs.
This was previously not included in the JSON representation for abstract specs, and couldn't be
stored.

Now, there is a separate `propagate` dictionary alongside the `parameters` dictionary. This isn't
beautiful, but when we bump the spec version for Spack `v0.24`, we can clean up this and other
aspects of the schema.

.github/dependabot.yml

@@ -12,7 +12,6 @@ updates:
       interval: "daily"
   # Requirements to run style checks
   - package-ecosystem: "pip"
-    directories:
-     - "/.github/workflows/requirements/*"
+    directory: "/.github/workflows/style"
     schedule:
       interval: "daily"

.github/workflows/audit.yaml

@@ -28,8 +28,8 @@ jobs:
       run:
         shell: ${{ matrix.system.shell }}
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
       with:
         python-version: ${{inputs.python_version}}
     - name: Install Python packages
@@ -61,7 +61,7 @@ jobs:
           ./share/spack/qa/validate_last_exit.ps1
           spack -d audit externals
           ./share/spack/qa/validate_last_exit.ps1
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: codecov/codecov-action@5ecb98a3c6b747ed38dc09f787459979aebb39be
       if: ${{ inputs.with_coverage == 'true' }}
       with:
         flags: unittests,audits

.github/workflows/bootstrap.yml

@@ -37,7 +37,7 @@ jobs:
               make patch unzip which xz python3 python3-devel tree \
               cmake bison
       - name: Checkout
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
         with:
           fetch-depth: 0
       - name: Bootstrap clingo
@@ -53,33 +53,27 @@ jobs:
     runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        runner: ['macos-13', 'macos-14', "ubuntu-latest", "windows-latest"]
+        runner: ['macos-13', 'macos-14', "ubuntu-latest"]
     steps:
       - name: Setup macOS
-        if: ${{ matrix.runner != 'ubuntu-latest' && matrix.runner != 'windows-latest' }}
+        if: ${{ matrix.runner != 'ubuntu-latest' }}
         run: |
           brew install cmake bison tree
       - name: Checkout
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
         with:
           python-version: "3.12"
       - name: Bootstrap clingo
-        env:
-          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
-          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
-          USER_SCOPE_PARENT_DIR: ${{ matrix.runner == 'windows-latest' && '$env:userprofile' || '$HOME' }}
-          VALIDATE_LAST_EXIT: ${{ matrix.runner == 'windows-latest' && './share/spack/qa/validate_last_exit.ps1' || '' }}
         run: |
-          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
+          source share/spack/setup-env.sh
           spack bootstrap disable github-actions-v0.5
           spack bootstrap disable github-actions-v0.4
           spack external find --not-buildable cmake bison
           spack -d solve zlib
-          ${{ env.VALIDATE_LAST_EXIT }}
-          tree ${{ env.USER_SCOPE_PARENT_DIR }}/.spack/bootstrap/store/
+          tree ~/.spack/bootstrap/store/
 
   gnupg-sources:
     runs-on: ${{ matrix.runner }}
@@ -89,14 +83,14 @@ jobs:
     steps:
       - name: Setup macOS
         if: ${{ matrix.runner != 'ubuntu-latest' }}
+        run: brew install tree gawk
+      - name: Remove system executables
         run: |
-          brew install tree gawk
-          sudo rm -rf $(command -v gpg gpg2)
-      - name: Setup Ubuntu
-        if: ${{ matrix.runner == 'ubuntu-latest' }}
-        run: sudo rm -rf $(command -v gpg gpg2 patchelf)
+          while [ -n "$(command -v gpg gpg2 patchelf)" ]; do
+            sudo rm $(command -v gpg gpg2 patchelf)
+          done
       - name: Checkout
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
         with:
           fetch-depth: 0
       - name: Bootstrap GnuPG
@@ -116,19 +110,17 @@ jobs:
     steps:
       - name: Setup macOS
         if: ${{ matrix.runner != 'ubuntu-latest' }}
+        run: brew install tree
+      - name: Remove system executables
         run: |
-          brew install tree
-          # Remove GnuPG since we want to bootstrap it
-          sudo rm -rf /usr/local/bin/gpg
-      - name: Setup Ubuntu
-        if: ${{ matrix.runner == 'ubuntu-latest' }}
-        run: |
-          sudo rm -rf $(which gpg) $(which gpg2) $(which patchelf)
+          while [ -n "$(command -v gpg gpg2 patchelf)" ]; do
+            sudo rm $(command -v gpg gpg2 patchelf)
+          done
       - name: Checkout
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
         with:
           python-version: |
             3.8
@@ -154,7 +146,7 @@ jobs:
                 not_found=0
                 old_path="$PATH"
                 export PATH="$ver_dir:$PATH"
-                ./bin/spack-tmpconfig -b ./.github/workflows/bin/bootstrap-test.sh
+                ./bin/spack-tmpconfig -b ./.github/workflows/bootstrap-test.sh
                 export PATH="$old_path"
               fi
             fi
@@ -168,3 +160,4 @@ jobs:
           source share/spack/setup-env.sh
           spack -d gpg list
           tree ~/.spack/bootstrap/store/
+

.github/workflows/build-containers.yml

@@ -40,7 +40,8 @@ jobs:
         # 1: Platforms to build for
         # 2: Base image (e.g. ubuntu:22.04)
         dockerfile: [[amazon-linux, 'linux/amd64,linux/arm64', 'amazonlinux:2'],
-                     [centos-stream9, 'linux/amd64,linux/arm64,linux/ppc64le', 'centos:stream9'],
+                     [centos7, 'linux/amd64,linux/arm64,linux/ppc64le', 'centos:7'],
+                     [centos-stream, 'linux/amd64,linux/arm64,linux/ppc64le', 'centos:stream'],
                      [leap15, 'linux/amd64,linux/arm64,linux/ppc64le', 'opensuse/leap:15'],
                      [ubuntu-focal, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:20.04'],
                      [ubuntu-jammy, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:22.04'],
@@ -55,7 +56,7 @@ jobs:
     if: github.repository == 'spack/spack'
     steps:
       - name: Checkout
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
 
       - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
         id: docker_meta
@@ -76,7 +77,7 @@ jobs:
         env:
           SPACK_YAML_OS: "${{ matrix.dockerfile[2] }}"
         run: |
-          .github/workflows/bin/generate_spack_yaml_containerize.sh
+          .github/workflows/generate_spack_yaml_containerize.sh
           . share/spack/setup-env.sh
           mkdir -p dockerfiles/${{ matrix.dockerfile[0] }}
           spack containerize --last-stage=bootstrap | tee dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile
@@ -87,19 +88,19 @@ jobs:
           fi
 
       - name: Upload Dockerfile
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
         with:
           name: dockerfiles_${{ matrix.dockerfile[0] }}
           path: dockerfiles
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb
 
       - name: Log in to GitHub Container Registry
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -107,13 +108,13 @@ jobs:
 
       - name: Log in to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build & Deploy ${{ matrix.dockerfile[0] }}
-        uses: docker/build-push-action@1ca370b3a9802c92e886402e0dd88098a2533b12
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0
         with:
           context: dockerfiles/${{ matrix.dockerfile[0] }}
           platforms: ${{ matrix.dockerfile[1] }}
@@ -126,7 +127,7 @@ jobs:
     needs: deploy-images
     steps:
       - name: Merge Artifacts
-        uses: actions/upload-artifact/merge@0b2256b8c012f0828dc542b3febcab082c67f72b
+        uses: actions/upload-artifact/merge@65462800fd760344b1a7b4382951275a0abb4808
         with:
           name: dockerfiles
           pattern: dockerfiles_*

.github/workflows/ci.yaml

@@ -36,7 +36,7 @@ jobs:
       core: ${{ steps.filter.outputs.core }}
       packages: ${{ steps.filter.outputs.packages }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
         if: ${{ github.event_name == 'push' }}
         with:
           fetch-depth: 0
@@ -53,13 +53,6 @@ jobs:
             - 'var/spack/repos/builtin/packages/clingo/**'
             - 'var/spack/repos/builtin/packages/python/**'
             - 'var/spack/repos/builtin/packages/re2c/**'
-            - 'var/spack/repos/builtin/packages/gnupg/**'
-            - 'var/spack/repos/builtin/packages/libassuan/**'
-            - 'var/spack/repos/builtin/packages/libgcrypt/**'
-            - 'var/spack/repos/builtin/packages/libgpg-error/**'
-            - 'var/spack/repos/builtin/packages/libksba/**'
-            - 'var/spack/repos/builtin/packages/npth/**'
-            - 'var/spack/repos/builtin/packages/pinentry/**'
             - 'lib/spack/**'
             - 'share/spack/**'
             - '.github/workflows/bootstrap.yml'

.github/workflows/install_spack.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+. share/spack/setup-env.sh
+echo -e "config:\n  build_jobs: 2" > etc/spack/config.yaml
+spack config add "packages:all:target:[x86_64]"
+spack compiler find
+spack compiler info apple-clang
+spack debug report
+spack solve zlib

.github/workflows/nightly-win-builds.yml

@@ -14,10 +14,10 @@ jobs:
   build-paraview-deps:
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
       with:
         python-version: 3.9
     - name: Install Python packages

.github/workflows/style/requirements.txt

@@ -1,7 +1,7 @@
 black==24.4.2
 clingo==5.7.1
-flake8==7.1.0
+flake8==7.0.0
 isort==5.13.2
 mypy==1.8.0
-types-six==1.16.21.20240513
+types-six==1.16.21.9
 vermin==1.6.0

.github/workflows/unit_tests.yaml

@@ -51,10 +51,10 @@ jobs:
           on_develop: false
 
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install System packages
@@ -72,7 +72,7 @@ jobs:
       run: |
           # Need this for the git tests to succeed.
           git --version
-          . .github/workflows/bin/setup_git.sh
+          . .github/workflows/setup_git.sh
     - name: Bootstrap clingo
       if: ${{ matrix.concretizer == 'clingo' }}
       env:
@@ -91,7 +91,7 @@ jobs:
           UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }}
       run: |
           share/spack/qa/run-unit-tests
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: codecov/codecov-action@5ecb98a3c6b747ed38dc09f787459979aebb39be
       with:
         flags: unittests,linux,${{ matrix.concretizer }}
         token: ${{ secrets.CODECOV_TOKEN }}
@@ -100,10 +100,10 @@ jobs:
   shell:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -118,13 +118,13 @@ jobs:
       run: |
           # Need this for the git tests to succeed.
           git --version
-          . .github/workflows/bin/setup_git.sh
+          . .github/workflows/setup_git.sh
     - name: Run shell tests
       env:
           COVERAGE: true
       run: |
           share/spack/qa/run-shell-tests
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: codecov/codecov-action@5ecb98a3c6b747ed38dc09f787459979aebb39be
       with:
         flags: shelltests,linux
         token: ${{ secrets.CODECOV_TOKEN }}
@@ -141,13 +141,13 @@ jobs:
           dnf install -y \
               bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
               make patch tcl unzip which xz
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
     - name: Setup repo and non-root user
       run: |
           git --version
           git config --global --add safe.directory /__w/spack/spack
           git fetch --unshallow
-          . .github/workflows/bin/setup_git.sh
+          . .github/workflows/setup_git.sh
           useradd spack-test
           chown -R spack-test .
     - name: Run unit tests
@@ -160,10 +160,10 @@ jobs:
   clingo-cffi:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -178,14 +178,14 @@ jobs:
       run: |
           # Need this for the git tests to succeed.
           git --version
-          . .github/workflows/bin/setup_git.sh
+          . .github/workflows/setup_git.sh
     - name: Run unit tests (full suite with coverage)
       env:
           COVERAGE: true
           SPACK_TEST_SOLVER: clingo
       run: |
           share/spack/qa/run-unit-tests
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: codecov/codecov-action@5ecb98a3c6b747ed38dc09f787459979aebb39be
       with:
         flags: unittests,linux,clingo
         token: ${{ secrets.CODECOV_TOKEN }}
@@ -198,10 +198,10 @@ jobs:
         os: [macos-13, macos-14]
         python-version: ["3.11"]
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Python packages
@@ -217,13 +217,13 @@ jobs:
         SPACK_TEST_PARALLEL: 4
       run: |
         git --version
-        . .github/workflows/bin/setup_git.sh
+        . .github/workflows/setup_git.sh
         . share/spack/setup-env.sh
         $(which spack) bootstrap disable spack-install
         $(which spack) solve zlib
         common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
         $(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: codecov/codecov-action@5ecb98a3c6b747ed38dc09f787459979aebb39be
       with:
         flags: unittests,macos
         token: ${{ secrets.CODECOV_TOKEN }}
@@ -236,10 +236,10 @@ jobs:
           powershell Invoke-Expression -Command "./share/spack/qa/windows_test_setup.ps1"; {0}
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
       with:
         python-version: 3.9
     - name: Install Python packages
@@ -247,14 +247,14 @@ jobs:
           python -m pip install --upgrade pip pywin32 setuptools pytest-cov clingo
     - name: Create local develop
       run: |
-        ./.github/workflows/bin/setup_git.ps1
+        ./.github/workflows/setup_git.ps1
     - name: Unit Test
       run: |
         spack unit-test -x --verbose --cov --cov-config=pyproject.toml
         ./share/spack/qa/validate_last_exit.ps1
         coverage combine -a
         coverage xml
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c
       with:
         flags: unittests,windows
         token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/valid-style.yml

@@ -18,15 +18,15 @@ jobs:
   validate:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
       with:
         python-version: '3.11'
         cache: 'pip'
     - name: Install Python Packages
       run: |
         pip install --upgrade pip setuptools
-        pip install -r .github/workflows/requirements/style/requirements.txt
+        pip install -r .github/workflows/style/requirements.txt
     - name: vermin (Spack's Core)
       run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
     - name: vermin (Repositories)
@@ -35,22 +35,22 @@ jobs:
   style:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
       with:
         python-version: '3.11'
         cache: 'pip'
     - name: Install Python packages
       run: |
         pip install --upgrade pip setuptools
-        pip install -r .github/workflows/requirements/style/requirements.txt
+        pip install -r .github/workflows/style/requirements.txt
     - name: Setup git configuration
       run: |
         # Need this for the git tests to succeed.
         git --version
-        . .github/workflows/bin/setup_git.sh
+        . .github/workflows/setup_git.sh
     - name: Run style tests
       run: |
           share/spack/qa/run-style-tests
@@ -70,13 +70,13 @@ jobs:
           dnf install -y \
               bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
               make patch tcl unzip which xz
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
       - name: Setup repo and non-root user
         run: |
           git --version
           git config --global --add safe.directory /__w/spack/spack
           git fetch --unshallow
-          . .github/workflows/bin/setup_git.sh
+          . .github/workflows/setup_git.sh
           useradd spack-test
           chown -R spack-test .
       - name: Bootstrap Spack development environment

.readthedocs.yml

@@ -14,3 +14,26 @@ sphinx:
 python:
   install:
     - requirements: lib/spack/docs/requirements.txt
+
+search:
+  ranking:
+    spack.html: -10
+    spack.*.html: -10
+    llnl.html: -10
+    llnl.*.html: -10
+    _modules/*: -10
+    command_index.html: -9
+    basic_usage.html: 5
+    configuration.html: 5
+    config_yaml.html: 5
+    packages_yaml.html: 5
+    build_settings.html: 5
+    environments.html: 5
+    containers.html: 5
+    mirrors.html: 5
+    module_file_support.html: 5
+    repositories.html: 5
+    binary_caches.html: 5
+    chain.html: 5
+    pipelines.html: 5
+    packaging_guide.html: 5

CHANGELOG.md

@@ -1,3 +1,94 @@
+# v0.22.3 (2024-11-18)
+
+## Bugfixes
+- Forward compatibility with Python 3.13 (#46775, #46983, #47035, #47175)
+- `archspec` was updated to v0.2.5 (#46503, #46958)
+- Fix path to Spack in `spack env depfile` makefile (#46966)
+- Fix `glibc` detection in Chinese locales (#47434)
+- Fix pickle round-trip of specs propagating variants (#47351)
+- Fix a bug where concurrent spack install commands would not always update explicits correctly
+  (#47358)
+- Fix a bug where autopush would run before all post install hooks modifying the install prefix
+  had run (#47329)
+- Fix `spack find -u` (#47102)
+- Fix a bug where sometimes the wrong Python interpreter was used for build dependencies such as
+  `py-setuptools` (#46980)
+- Fix default config errors found by `spack audit externals` (#47308)
+- Fix duplicate printing of external roots in installer (#44917)
+- Fix modules schema in `compilers.yaml` (#47197)
+- Reduce the size of generated YAML for Gitlab CI (#44995)
+- Handle missing metadata file gracefully in bootstrap (#47278)
+- Show underlying errors on fetch failure (#45714)
+- Recognize `.` and `..` as paths instead of names in buildcache commands (#47105)
+- Documentation and style (#46991, #47107, #47110, #47111, #47346, #47307, #47309, #47328, #47160,
+  #47402, #47557, #46709, #47080)
+- Tests and CI fixes (#47165, #46711)
+
+## Package updates
+- ffmpeg: fix hash of patch (#45574)
+
+# v0.22.2 (2024-09-21)
+
+## Bugfixes
+- Forward compatibility with Spack 0.23 packages with language dependencies (#45205, #45191)
+- Forward compatibility with `urllib` from Python 3.12.6+ (#46453, #46483)
+- Bump vendored `archspec` for better aarch64 support (#45721, #46445)
+- Support macOS Sequoia (#45018, #45127)
+- Fix regression in `{variants.X}` and `{variants.X.value}` format strings (#46206)
+- Ensure shell escaping of environment variable values in load and activate commands (#42780)
+- Fix an issue where `spec[pkg]` considers specs outside the current DAG (#45090)
+- Do not halt concretization on unknown variants in externals (#45326)
+- Improve validation of `develop` config section (#46485)
+- Explicitly disable `ccache` if turned off in config, to avoid cache pollution (#45275)
+- Improve backwards compatibility in `include_concrete` (#45766)
+- Fix issue where package tags were sometimes repeated (#45160)
+- Make `setup-env.sh` "sourced only" by dropping execution bits (#45641)
+- Make certain source/binary fetch errors recoverable instead of a hard error (#45683)
+- Remove debug statements in package hash computation (#45235)
+- Remove redundant clingo warnings (#45269)
+- Remove hard-coded layout version (#45645)
+- Do not initialize previous store state in `use_store` (#45268)
+- Docs improvements (#46475)
+
+## Package updates
+- `chapel` major update (#42197, #44931, #45304)
+
+# v0.22.1 (2024-07-04)
+
+## Bugfixes
+- Fix reuse of externals on Linux (#44316)
+- Ensure parent gcc-runtime version >= child (#44834, #44870)
+- Ensure the latest gcc-runtime is rpath'ed when multiple exist among link deps (#44219)
+- Improve version detection of glibc (#44154)
+- Improve heuristics for solver (#44893, #44976, #45023)
+- Make strong preferences override reuse (#44373)
+- Reduce verbosity when C compiler is missing (#44182)
+- Make missing ccache executable an error when required (#44740)
+- Make every environment view containing `python` a `venv` (#44382)
+- Fix external detection for compilers with os but no target (#44156)
+- Fix version optimization for roots (#44272)
+- Handle common implementations of pagination of tags in OCI build caches (#43136)
+- Apply fetched patches to develop specs (#44950)
+- Avoid Windows wrappers for filesystem utilities on non-Windows (#44126)
+- Fix issue with long filenames in build caches on Windows (#43851)
+- Fix formatting issue in `spack audit` (#45045)
+- CI fixes (#44582, #43965, #43967, #44279, #44213)
+
+## Package updates
+- protobuf: fix 3.4:3.21 patch checksum (#44443)
+- protobuf: update hash for patch needed when="@3.4:3.21" (#44210)
+- git: bump v2.39 to 2.45; deprecate unsafe versions (#44248)
+- gcc: use -rpath {rpath_dir} not -rpath={rpath dir} (#44315)
+- Remove mesa18 and libosmesa (#44264)
+- Enforce consistency of `gl` providers (#44307)
+- Require libiconv for iconv (#44335, #45026).
+  Notice that glibc/musl also provide iconv, but are not guaranteed to be
+  complete. Set `packages:iconv:require:[glibc]` to restore the old behavior.
+- py-matplotlib: qualify when to do a post install (#44191)
+- rust: fix v1.78.0 instructions (#44127)
+- suite-sparse: improve setting of the `libs` property (#44214)
+- netlib-lapack: provide blas and lapack together (#44981)
+
 
 # v0.22.0 (2024-05-12)
 
@@ -319,6 +410,7 @@
     * 344 committers to packages
     * 45 committers to core
 
+
 # v0.21.2 (2024-03-01)
 
 ## Bugfixes
@@ -348,7 +440,7 @@
 - spack graph: fix coloring with environments (#41240)
 - spack info: sort variants in --variants-by-name (#41389)
 - Spec.format: error on old style format strings (#41934)
-- ASP-based solver: 
+- ASP-based solver:
   - fix infinite recursion when computing concretization errors (#41061)
   - don't error for type mismatch on preferences (#41138)
   - don't emit spurious debug output (#41218)

README.md

@@ -32,7 +32,7 @@
 
 Spack is a multi-platform package manager that builds and installs
 multiple versions and configurations of software. It works on Linux,
-macOS, Windows, and many supercomputers. Spack is non-destructive: installing a
+macOS, and many supercomputers. Spack is non-destructive: installing a
 new version of a package does not break existing installations, so many
 configurations of the same package can coexist.
 

bin/spack-python

@@ -22,4 +22,4 @@
 #
 # This is compatible across platforms.
 #
-exec spack python "$@"
+exec /usr/bin/env spack python "$@"

bin/spack.bat

@@ -188,27 +188,25 @@ if NOT "%_sp_args%"=="%_sp_args:--help=%" (
 goto :end_switch
 
 :case_load
-if NOT defined _sp_args (
-   exit /B 0
-)
-
-:: If args contain --bat, or -h/--help: just execute.
-if NOT "%_sp_args%"=="%_sp_args:--help=%" (
-    goto :default_case
-) else if NOT "%_sp_args%"=="%_sp_args:-h=%" (
-    goto :default_case
-) else if NOT "%_sp_args%"=="%_sp_args:--bat=%" (
-    goto :default_case
-) else if NOT "%_sp_args%"=="%_sp_args:--list=%" (
-    goto :default_case
+:: If args contain --sh, --csh, or -h/--help: just execute.
+if defined _sp_args (
+    if NOT "%_sp_args%"=="%_sp_args:--help=%" (
+        goto :default_case
+    ) else if NOT "%_sp_args%"=="%_sp_args:-h=%" (
+        goto :default_case
+    ) else if NOT "%_sp_args%"=="%_sp_args:--bat=%" (
+        goto :default_case
+    )
 )
 
 for /f "tokens=* USEBACKQ" %%I in (
-    `python "%spack%" %_sp_flags% %_sp_subcommand% --bat %_sp_args%`
-    ) do %%I
+    `python "%spack%" %_sp_flags% %_sp_subcommand% --bat %_sp_args%`) do %%I
 
 goto :end_switch
 
+:case_unload
+goto :case_load
+
 :default_case
 python "%spack%" %_sp_flags% %_sp_subcommand% %_sp_args%
 goto :end_switch

etc/spack/defaults/concretizer.yaml

@@ -42,8 +42,8 @@ concretizer:
     # "minimal": allows the duplication of 'build-tools' nodes only (e.g. py-setuptools, cmake etc.)
     # "full" (experimental): allows separation of the entire build-tool stack (e.g. the entire "cmake" subDAG)
     strategy: minimal
-  # Option to specify compatiblity between operating systems for reuse of compilers and packages
-  # Specified as a key: [list] where the key is the os that is being targeted, and the list contains the OS's 
-  # it can reuse. Note this is a directional compatibility so mutual compatibility between two OS's 
+  # Option to specify compatibility between operating systems for reuse of compilers and packages
+  # Specified as a key: [list] where the key is the os that is being targeted, and the list contains the OS's
+  # it can reuse. Note this is a directional compatibility so mutual compatibility between two OS's
   # requires two entries i.e. os_compatible: {sonoma: [monterey], monterey: [sonoma]}
   os_compatible: {}

etc/spack/defaults/cray/modules.yaml

@@ -0,0 +1,16 @@
+# -------------------------------------------------------------------------
+# This is the default configuration for Spack's module file generation.
+#
+# Settings here are versioned with Spack and are intended to provide
+# sensible defaults out of the box. Spack maintainers should edit this
+# file to keep it current.
+#
+# Users can override these settings by editing the following files.
+#
+# Per-spack-instance settings (overrides defaults):
+#   $SPACK_ROOT/etc/spack/modules.yaml
+#
+# Per-user settings (overrides default and site settings):
+#   ~/.spack/modules.yaml
+# -------------------------------------------------------------------------
+modules: {}

etc/spack/defaults/linux/packages.yaml

@@ -0,0 +1,3 @@
+packages:
+  iconv:
+    require: [libiconv]

etc/spack/defaults/packages.yaml

@@ -37,9 +37,9 @@ packages:
       jpeg: [libjpeg-turbo, libjpeg]
       lapack: [openblas, amdlibflame]
       libc: [glibc, musl]
-      libgfortran: [ gcc-runtime ]
+      libgfortran: [gcc-runtime]
       libglx: [mesa+glx]
-      libifcore: [ intel-oneapi-runtime ]
+      libifcore: [intel-oneapi-runtime]
       libllvm: [llvm]
       lua-lang: [lua, lua-luajit-openresty, lua-luajit]
       luajit: [lua-luajit-openresty, lua-luajit]

lib/spack/docs/basic_usage.rst

@@ -1433,12 +1433,22 @@ the reserved keywords ``platform``, ``os`` and ``target``:
    $ spack install libelf os=ubuntu18.04
    $ spack install libelf target=broadwell
 
+or together by using the reserved keyword ``arch``:
+
+.. code-block:: console
+
+   $ spack install libelf arch=cray-CNL10-haswell
+
 Normally users don't have to bother specifying the architecture if they
 are installing software for their current host, as in that case the
 values will be detected automatically.  If you need fine-grained control
 over which packages use which targets (or over *all* packages' default
 target), see :ref:`package-preferences`.
 
+.. admonition:: Cray machines
+
+  The situation is a little bit different for Cray machines and a detailed
+  explanation on how the architecture can be set on them can be found at :ref:`cray-support`
 
 .. _support-for-microarchitectures:
 

lib/spack/docs/build_systems/autotoolspackage.rst

@@ -147,15 +147,6 @@ example, the ``bash`` shell is used to run the ``autogen.sh`` script.
    def autoreconf(self, spec, prefix):
        which("bash")("autogen.sh")
 
-If the ``package.py`` has build instructions in a separate
-:ref:`builder class <multiple_build_systems>`, the signature for a phase changes slightly:
-
-.. code-block:: python
-
-   class AutotoolsBuilder(AutotoolsBuilder):
-      def autoreconf(self, pkg, spec, prefix):
-         which("bash")("autogen.sh")
-
 """""""""""""""""""""""""""""""""""""""
 patching configure or Makefile.in files
 """""""""""""""""""""""""""""""""""""""

lib/spack/docs/build_systems/inteloneapipackage.rst

@@ -25,7 +25,7 @@ use Spack to build packages with the tools.
 The Spack Python class ``IntelOneapiPackage`` is a base class that is
 used by ``IntelOneapiCompilers``, ``IntelOneapiMkl``,
 ``IntelOneapiTbb`` and other classes to implement the oneAPI
-packages. Search for ``oneAPI`` at `packages.spack.io <https://packages.spack.io>`_ for the full
+packages. Search for ``oneAPI`` at `<packages.spack.io>`_ for the full
 list of available oneAPI packages, or use::
 
   spack list -d oneAPI

lib/spack/docs/chain.rst

@@ -5,14 +5,13 @@
 
 .. chain:
 
-============================
-Chaining Spack Installations
-============================
+=============================================
+Chaining Spack Installations (upstreams.yaml)
+=============================================
 
 You can point your Spack installation to another installation to use any
 packages that are installed there. To register the other Spack instance,
-you can add it as an entry to ``upstreams.yaml`` at any of the
-:ref:`configuration-scopes`:
+you can add it as an entry to ``upstreams.yaml``:
 
 .. code-block:: yaml
 
@@ -23,8 +22,7 @@ you can add it as an entry to ``upstreams.yaml`` at any of the
       install_tree: /path/to/another/spack/opt/spack
 
 ``install_tree`` must point to the ``opt/spack`` directory inside of the
-Spack base directory, or the location of the ``install_tree`` defined
-in :ref:`config.yaml <config-yaml>`.
+Spack base directory.
 
 Once the upstream Spack instance has been added, ``spack find`` will
 automatically check the upstream instance when querying installed packages,

lib/spack/docs/containers.rst

@@ -203,9 +203,12 @@ The OS that are currently supported are summarized in the table below:
    * - Ubuntu 24.04
      - ``ubuntu:24.04``
      - ``spack/ubuntu-noble``
-   * - CentOS Stream9
-     - ``quay.io/centos/centos:stream9``
-     - ``spack/centos-stream9``
+   * - CentOS 7
+     - ``centos:7``
+     - ``spack/centos7``
+   * - CentOS Stream
+     - ``quay.io/centos/centos:stream``
+     - ``spack/centos-stream``
    * - openSUSE Leap
      - ``opensuse/leap``
      - ``spack/leap15``

lib/spack/docs/contribution_guide.rst

@@ -184,7 +184,7 @@ Style Tests
 
 Spack uses `Flake8 <http://flake8.pycqa.org/en/latest/>`_ to test for
 `PEP 8 <https://www.python.org/dev/peps/pep-0008/>`_ conformance and
-`mypy <https://mypy.readthedocs.io/en/stable/>` for type checking. PEP 8 is
+`mypy <https://mypy.readthedocs.io/en/stable/>`_ for type checking. PEP 8 is
 a series of style guides for Python that provide suggestions for everything
 from variable naming to indentation. In order to limit the number of PRs that
 were mostly style changes, we decided to enforce PEP 8 conformance. Your PR

lib/spack/docs/developer_guide.rst

@@ -716,27 +716,27 @@ Release branches
 ^^^^^^^^^^^^^^^^
 
 There are currently two types of Spack releases: :ref:`major releases
-<major-releases>` (``0.17.0``, ``0.18.0``, etc.) and :ref:`point releases
-<point-releases>` (``0.17.1``, ``0.17.2``, ``0.17.3``, etc.). Here is a
+<major-releases>` (``0.21.0``, ``0.22.0``, etc.) and :ref:`patch releases
+<patch-releases>` (``0.22.1``, ``0.22.2``, ``0.22.3``, etc.). Here is a
 diagram of how Spack release branches work::
 
-    o    branch: develop  (latest version, v0.19.0.dev0)
+    o    branch: develop  (latest version, v0.23.0.dev0)
     |
     o
-    | o  branch: releases/v0.18, tag: v0.18.1
+    | o  branch: releases/v0.22, tag: v0.22.1
     o |
-    | o  tag: v0.18.0
+    | o  tag: v0.22.0
     o |
     | o
     |/
     o
     |
     o
-    | o  branch: releases/v0.17, tag: v0.17.2
+    | o  branch: releases/v0.21, tag: v0.21.2
     o |
-    | o  tag: v0.17.1
+    | o  tag: v0.21.1
     o |
-    | o  tag: v0.17.0
+    | o  tag: v0.21.0
     o |
     | o
     |/
@@ -747,8 +747,8 @@ requests target ``develop``. The ``develop`` branch will report that its
 version is that of the next **major** release with a ``.dev0`` suffix.
 
 Each Spack release series also has a corresponding branch, e.g.
-``releases/v0.18`` has ``0.18.x`` versions of Spack, and
-``releases/v0.17`` has ``0.17.x`` versions. A major release is the first
+``releases/v0.22`` has ``v0.22.x`` versions of Spack, and
+``releases/v0.21`` has ``v0.21.x`` versions. A major release is the first
 tagged version on a release branch. Minor releases are back-ported from
 develop onto release branches. This is typically done by cherry-picking
 bugfix commits off of ``develop``.
@@ -778,27 +778,40 @@ for more details.
 Scheduling work for releases
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-We schedule work for releases by creating `GitHub projects
-<https://github.com/spack/spack/projects>`_. At any time, there may be
-several open release projects. For example, below are two releases (from
-some past version of the page linked above):
+We schedule work for **major releases** through `milestones
+<https://github.com/spack/spack/milestones>`_ and `GitHub Projects
+<https://github.com/spack/spack/projects>`_, while **patch releases** use `labels
+<https://github.com/spack/spack/labels>`_.
 
-.. image:: images/projects.png
+There is only one milestone open at a time. Its name corresponds to the next major version, for
+example ``v0.23``. Important issues and pull requests should be assigned to this milestone by
+core developers, so that they are not forgotten at the time of release. The milestone is closed
+when the release is made, and a new milestone is created for the next major release.
 
-This image shows one release in progress for ``0.15.1`` and another for
-``0.16.0``. Each of these releases has a project board containing issues
-and pull requests. GitHub shows a status bar with completed work in
-green, work in progress in purple, and work not started yet in gray, so
-it's fairly easy to see progress.
+Bug reports in GitHub issues are automatically labelled ``bug`` and ``triage``. Spack developers
+assign one of the labels ``impact-low``, ``impact-medium`` or ``impact-high``. This will make the
+issue appear in the `Triaged bugs <https://github.com/orgs/spack/projects/6>`_ project board.
+Important issues should be assigned to the next milestone as well, so they appear at the top of
+the project board.
 
-Spack's project boards are not firm commitments so we move work between
-releases frequently. If we need to make a release and some tasks are not
-yet done, we will simply move them to the next minor or major release, rather
-than delaying the release to complete them.
+Spack's milestones are not firm commitments so we move work between releases frequently. If we
+need to make a release and some tasks are not yet done, we will simply move them to the next major
+release milestone, rather than delaying the release to complete them.
 
-For more on using GitHub project boards, see `GitHub's documentation
-<https://docs.github.com/en/github/managing-your-work-on-github/about-project-boards>`_.
+^^^^^^^^^^^^^^^^^^^^^
+Backporting bug fixes
+^^^^^^^^^^^^^^^^^^^^^
 
+When a bug is fixed in the ``develop`` branch, it is often necessary to backport the fix to one
+(or more) of the ``release/vX.Y`` branches. Only the release manager is responsible for doing
+backports, but Spack maintainers are responsible for labelling pull requests (and issues if no bug
+fix is available yet) with ``vX.Y.Z`` labels. The label should correspond to the next patch version
+that the bug fix should be backported to.
+
+Backports are done publicly by the release manager using a pull request named ``Backports vX.Y.Z``.
+This pull request is opened from the ``backports/vX.Y.Z`` branch, targets the ``releases/vX.Y``
+branch and contains a (growing) list of cherry-picked commits from the ``develop`` branch.
+Typically there are one or two backport pull requests open at any given time.
 
 .. _major-releases:
 
@@ -806,25 +819,21 @@ For more on using GitHub project boards, see `GitHub's documentation
 Making major releases
 ^^^^^^^^^^^^^^^^^^^^^
 
-Assuming a project board has already been created and all required work
-completed, the steps to make the major release are:
+Assuming all required work from the milestone is completed, the steps to make the major release
+are:
 
-#. Create two new project boards:
+#. `Create a new milestone <https://github.com/spack/spack/milestones>`_ for the next major
+   release.
 
-   * One for the next major release
-   * One for the next point release
+#. `Create a new label <https://github.com/spack/spack/labels>`_ for the next patch release.
 
-#. Move any optional tasks that are not done to one of the new project boards.
-
-   In general, small bugfixes should go to the next point release. Major
-   features, refactors, and changes that could affect concretization should
-   go in the next major release.
+#. Move any optional tasks that are not done to the next milestone.
 
 #. Create a branch for the release, based on ``develop``:
 
    .. code-block:: console
 
-      $ git checkout -b releases/v0.15 develop
+      $ git checkout -b releases/v0.23 develop
 
    For a version ``vX.Y.Z``, the branch's name should be
    ``releases/vX.Y``. That is, you should create a ``releases/vX.Y``
@@ -860,8 +869,8 @@ completed, the steps to make the major release are:
 
    Create a pull request targeting the ``develop`` branch, bumping the major
    version in ``lib/spack/spack/__init__.py`` with a ``dev0`` release segment.
-   For instance when you have just released ``v0.15.0``, set the version
-   to ``(0, 16, 0, 'dev0')`` on ``develop``.
+   For instance when you have just released ``v0.23.0``, set the version
+   to ``(0, 24, 0, 'dev0')`` on ``develop``.
 
 #. Follow the steps in :ref:`publishing-releases`.
 
@@ -870,82 +879,52 @@ completed, the steps to make the major release are:
 #. Follow the steps in :ref:`announcing-releases`.
 
 
-.. _point-releases:
+.. _patch-releases:
 
 ^^^^^^^^^^^^^^^^^^^^^
-Making point releases
+Making patch releases
 ^^^^^^^^^^^^^^^^^^^^^
 
-Assuming a project board has already been created and all required work
-completed, the steps to make the point release are:
+To make the patch release process both efficient and transparent, we use a *backports pull request*
+which contains cherry-picked commits from the ``develop`` branch. The majority of the work is to
+cherry-pick the bug fixes, which ideally should be done as soon as they land on ``develop``:
+this ensures cherry-picking happens in order, and makes conflicts easier to resolve since the
+changes are fresh in the mind of the developer.
 
-#. Create a new project board for the next point release.
+The backports pull request is always titled ``Backports vX.Y.Z`` and is labelled ``backports``. It
+is opened from a branch named ``backports/vX.Y.Z`` and targets the ``releases/vX.Y`` branch.
 
-#. Move any optional tasks that are not done to the next project board.
+Whenever a pull request labelled ``vX.Y.Z`` is merged, cherry-pick the associated squashed commit
+on ``develop`` to the ``backports/vX.Y.Z`` branch. For pull requests that were rebased (or not
+squashed), cherry-pick each associated commit individually. Never force push to the
+``backports/vX.Y.Z`` branch.
 
-#. Check out the release branch (it should already exist).
+.. warning::
 
-    For the ``X.Y.Z`` release, the release branch is called ``releases/vX.Y``.
-    For ``v0.15.1``, you would check out ``releases/v0.15``:
+   Sometimes you may **still** get merge conflicts even if you have
+   cherry-picked all the commits in order. This generally means there
+   is some other intervening pull request that the one you're trying
+   to pick depends on. In these cases, you'll need to make a judgment
+   call regarding those pull requests.  Consider the number of affected
+   files and/or the resulting differences.
 
-   .. code-block:: console
+   1. If the changes are small, you might just cherry-pick it.
 
-      $ git checkout releases/v0.15
+   2. If the changes are large, then you may decide that this fix is not
+      worth including in a patch release, in which case you should remove
+      the label from the pull request. Remember that large, manual backports
+      are seldom the right choice for a patch release.
 
-#. If a pull request to the release branch named ``Backports vX.Y.Z`` is not already
-   in the project, create it. This pull request ought to be created as early as
-   possible when working on a release project, so that we can build the release
-   commits incrementally, and identify potential conflicts at an early stage.
+When all commits are cherry-picked in the ``backports/vX.Y.Z`` branch, make the patch
+release as follows:
 
-#. Cherry-pick each pull request in the ``Done`` column of the release
-   project board onto the ``Backports vX.Y.Z`` pull request.
+#. `Create a new label <https://github.com/spack/spack/labels>`_ ``vX.Y.{Z+1}`` for the next patch
+   release.
 
-   This is **usually** fairly simple since we squash the commits from the
-   vast majority of pull requests. That means there is only one commit
-   per pull request to cherry-pick. For example, `this pull request
-   <https://github.com/spack/spack/pull/15777>`_ has three commits, but
-   they were squashed into a single commit on merge. You can see the
-   commit that was created here:
+#. Replace the label ``vX.Y.Z`` with ``vX.Y.{Z+1}`` for all PRs and issues that are not done.
 
-   .. image:: images/pr-commit.png
-
-   You can easily cherry pick it like this (assuming you already have the
-   release branch checked out):
-
-   .. code-block:: console
-
-      $ git cherry-pick 7e46da7
-
-   For pull requests that were rebased (or not squashed), you'll need to
-   cherry-pick each associated commit individually.
-
-   .. warning::
-
-      It is important to cherry-pick commits in the order they happened,
-      otherwise you can get conflicts while cherry-picking. When
-      cherry-picking look at the merge date,
-      **not** the number of the pull request or the date it was opened.
-
-      Sometimes you may **still** get merge conflicts even if you have
-      cherry-picked all the commits in order. This generally means there
-      is some other intervening pull request that the one you're trying
-      to pick depends on. In these cases, you'll need to make a judgment
-      call regarding those pull requests.  Consider the number of affected
-      files and or the resulting differences.
-
-      1. If the dependency changes are small, you might just cherry-pick it,
-         too. If you do this, add the task to the release board.
-
-      2. If the changes are large, then you may decide that this fix is not
-         worth including in a point release, in which case you should remove
-         the task from the release project.
-
-      3. You can always decide to manually back-port the fix to the release
-         branch if neither of the above options makes sense, but this can
-         require a lot of work. It's seldom the right choice.
-
-#. When all the commits from the project board are cherry-picked into
-   the ``Backports vX.Y.Z`` pull request, you can push a commit to:
+#. Manually push a single commit with commit message ``Set version to vX.Y.Z`` to the
+   ``backports/vX.Y.Z`` branch, that both bumps the Spack version number and updates the changelog:
 
    1. Bump the version in ``lib/spack/spack/__init__.py``.
    2. Update ``CHANGELOG.md`` with a list of the changes.
@@ -954,20 +933,22 @@ completed, the steps to make the point release are:
    release branch. See `the changelog from 0.14.1
    <https://github.com/spack/spack/commit/ff0abb9838121522321df2a054d18e54b566b44a>`_.
 
-#. Merge the ``Backports vX.Y.Z`` PR with the **Rebase and merge** strategy. This
-   is needed to keep track in the release branch of all the commits that were
-   cherry-picked.
-
-#. Make sure CI passes on the release branch, including:
+#. Make sure CI passes on the **backports pull request**, including:
 
    * Regular unit tests
    * Build tests
    * The E4S pipeline at `gitlab.spack.io <https://gitlab.spack.io>`_
 
-   If CI does not pass, you'll need to figure out why, and make changes
-   to the release branch until it does. You can make more commits, modify
-   or remove cherry-picked commits, or cherry-pick **more** from
-   ``develop`` to make this happen.
+#. Merge the ``Backports vX.Y.Z`` PR with the **Rebase and merge** strategy. This
+   is needed to keep track in the release branch of all the commits that were
+   cherry-picked.
+
+#. Make sure CI passes on the last commit of the **release branch**.
+
+#. In the rare case you need to include additional commits in the patch release after the backports
+   PR is merged, it is best to delete the last commit ``Set version to vX.Y.Z`` from the release
+   branch with a single force push, open a new backports PR named ``Backports vX.Y.Z (2)``, and
+   repeat the process. Avoid repeated force pushes to the release branch.
 
 #. Follow the steps in :ref:`publishing-releases`.
 
@@ -1042,25 +1023,31 @@ Updating `releases/latest`
 
 If the new release is the **highest** Spack release yet, you should
 also tag it as ``releases/latest``. For example, suppose the highest
-release is currently ``0.15.3``:
+release is currently ``0.22.3``:
 
-* If you are releasing ``0.15.4`` or ``0.16.0``, then you should tag
-  it with ``releases/latest``, as these are higher than ``0.15.3``.
+* If you are releasing ``0.22.4`` or ``0.23.0``, then you should tag
+  it with ``releases/latest``, as these are higher than ``0.22.3``.
 
 * If you are making a new release of an **older** major version of
-  Spack, e.g. ``0.14.4``, then you should not tag it as
+  Spack, e.g. ``0.21.4``, then you should not tag it as
   ``releases/latest`` (as there are newer major versions).
 
-To tag ``releases/latest``, do this:
+To do so, first fetch the latest tag created on GitHub, since you may not have it locally:
 
 .. code-block:: console
 
-   $ git checkout releases/vX.Y     # vX.Y is the new release's branch
-   $ git tag --force releases/latest
-   $ git push --force --tags
+   $ git fetch --force git@github.com:spack/spack vX.Y.Z
 
-The ``--force`` argument to ``git tag`` makes ``git`` overwrite the existing
-``releases/latest`` tag with the new one.
+Then tag ``vX.Y.Z`` as ``releases/latest`` and push the individual tag to GitHub.
+
+.. code-block:: console
+
+   $ git tag --force releases/latest vX.Y.Z
+   $ git push --force git@github.com:spack/spack releases/latest
+
+The ``--force`` argument to ``git tag`` makes ``git`` overwrite the existing ``releases/latest``
+tag with the new one. Do **not** use the ``--tags`` flag when pushing, since this will push *all*
+local tags.
 
 
 .. _announcing-releases:

lib/spack/docs/environments.rst

@@ -931,84 +931,32 @@ This allows for a much-needed reduction in redundancy between packages
 and constraints.
 
 
------------------
-Environment Views
------------------
+----------------
+Filesystem Views
+----------------
 
-Spack Environments can have an associated filesystem view, which is a directory
-with a more traditional structure ``<view>/bin``, ``<view>/lib``, ``<view>/include``
-in which all files of the installed packages are linked.
-
-By default a view is created for each environment, thanks to the ``view: true``
-option in the ``spack.yaml`` manifest file:
-
-.. code-block:: yaml
-
-   spack:
-     specs: [perl, python]
-     view: true
-
-The view is created in a hidden directory ``.spack-env/view`` relative to the environment.
-If you've used ``spack env activate``, you may have already interacted with this view. Spack
-prepends its ``<view>/bin`` dir to ``PATH`` when the environment is activated, so that
-you can directly run executables from all installed packages in the environment.
-
-Views are highly customizable: you can control where they are put, modify their structure,
-include and exclude specs, change how files are linked, and you can even generate multiple
-views for a single environment.
+Spack Environments can define filesystem views, which provide a direct access point
+for software similar to the directory hierarchy that might exist under ``/usr/local``.
+Filesystem views are updated every time the environment is written out to the lock
+file ``spack.lock``, so the concrete environment and the view are always compatible.
+The files of the view's installed packages are brought into the view by symbolic or
+hard links, referencing the original Spack installation, or by copy.
 
 .. _configuring_environment_views:
 
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-Minimal view configuration
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Configuration in ``spack.yaml``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The minimal configuration
-
-.. code-block:: yaml
-
-   spack:
-     # ...
-     view: true
-
-lets Spack generate a single view with default settings under the
-``.spack-env/view`` directory of the environment.
-
-Another short way to configure a view is to specify just where to put it:
-
-.. code-block:: yaml
-
-   spack:
-     # ...
-     view: /path/to/view
-
-Views can also be disabled by setting ``view: false``.
-
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Advanced view configuration
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-One or more **view descriptors** can be defined under ``view``, keyed by a name.
-The example from the previous section with ``view: /path/to/view`` is equivalent
-to defining a view descriptor named ``default`` with a ``root`` attribute:
-
-.. code-block:: yaml
-
-   spack:
-     # ...
-     view:
-       default:  # name of the view
-         root: /path/to/view  # view descriptor attribute
-
-The ``default`` view descriptor name is special: when you ``spack env activate`` your
-environment, this view will be used to update (among other things) your ``PATH``
-variable.
-
-View descriptors must contain the root of the view, and optionally projections,
-``select`` and ``exclude`` lists and link information via ``link`` and
+The Spack Environment manifest file has a top-level keyword
+``view``. Each entry under that heading is a **view descriptor**, headed
+by a name. Any number of views may be defined under the ``view`` heading.
+The view descriptor contains the root of the view, and
+optionally the projections for the view, ``select`` and
+``exclude`` lists for the view and link information via ``link`` and
 ``link_type``.
 
-As a more advanced example, in the following manifest
+For example, in the following manifest
 file snippet we define a view named ``mpis``, rooted at
 ``/path/to/view`` in which all projections use the package name,
 version, and compiler name to determine the path for a given
@@ -1053,10 +1001,59 @@ of ``hardlink`` or ``copy``.
    when the environment is not activated, and linked libraries will be located
    *outside* of the view thanks to rpaths.
 
+
+There are two shorthands for environments with a single view. If the
+environment at ``/path/to/env`` has a single view, with a root at
+``/path/to/env/.spack-env/view``, with default selection and exclusion
+and the default projection, we can put ``view: True`` in the
+environment manifest. Similarly, if the environment has a view with a
+different root, but default selection, exclusion, and projections, the
+manifest can say ``view: /path/to/view``. These views are
+automatically named ``default``, so that
+
+.. code-block:: yaml
+
+   spack:
+     # ...
+     view: True
+
+is equivalent to
+
+.. code-block:: yaml
+
+   spack:
+     # ...
+     view:
+       default:
+         root: .spack-env/view
+
+and
+
+.. code-block:: yaml
+
+   spack:
+     # ...
+     view: /path/to/view
+
+is equivalent to
+
+.. code-block:: yaml
+
+   spack:
+     # ...
+     view:
+       default:
+         root: /path/to/view
+
+By default, Spack environments are configured with ``view: True`` in
+the manifest. Environments can be configured without views using
+``view: False``. For backwards compatibility reasons, environments
+with no ``view`` key are treated the same as ``view: True``.
+
 From the command line, the ``spack env create`` command takes an
 argument ``--with-view [PATH]`` that sets the path for a single, default
 view. If no path is specified, the default path is used (``view:
-true``). The argument ``--without-view`` can be used to create an
+True``). The argument ``--without-view`` can be used to create an
 environment without any view configured.
 
 The ``spack env view`` command can be used to change the manage views
@@ -1122,18 +1119,11 @@ the projection under ``all`` before reaching those entries.
 Activating environment views
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The ``spack env activate <env>`` has two effects:
-
-1. It activates the environment so that further Spack commands such
-   as ``spack install`` will run in the context of the environment.
-2. It activates the view so that environment variables such as
-   ``PATH`` are updated to include the view.
-
-Without further arguments, the ``default`` view of the environment is
-activated. If a view with a different name has to be activated,
-``spack env activate --with-view <name> <env>`` can be
-used instead. You can also activate the environment without modifying
-further environment variables using ``--without-view``.
+The ``spack env activate`` command will put the default view for the
+environment into the user's path, in addition to activating the
+environment for Spack commands. The arguments ``-v,--with-view`` and
+``-V,--without-view`` can be used to tune this behavior. The default
+behavior is to activate with the environment view if there is one.
 
 The environment variables affected by the ``spack env activate``
 command and the paths that are used to update them are determined by
@@ -1156,8 +1146,8 @@ relevant variable if the path exists. For this reason, it is not
 recommended to use non-default projections with the default view of an
 environment.
 
-The ``spack env deactivate`` command will remove the active view of
-the Spack environment from the user's environment variables.
+The ``spack env deactivate`` command will remove the default view of
+the environment from the user's path.
 
 
 .. _env-generate-depfile:
@@ -1316,7 +1306,7 @@ index once every package is pushed. Note how this target uses the generated
    example/push/%: example/install/%
    	@mkdir -p $(dir $@)
    	$(info About to push $(SPEC) to a buildcache)
-   	$(SPACK) -e . buildcache push --only=package $(BUILDCACHE_DIR) /$(HASH)
+   	$(SPACK) -e . buildcache push --allow-root --only=package $(BUILDCACHE_DIR) /$(HASH)
    	@touch $@
 
    push: $(addprefix example/push/,$(example/SPACK_PACKAGE_IDS))

lib/spack/docs/getting_started.rst

@@ -35,7 +35,7 @@ A build matrix showing which packages are working on which systems is shown belo
       .. code-block:: console
 
          apt update
-         apt install build-essential ca-certificates coreutils curl environment-modules gfortran git gpg lsb-release python3 python3-distutils python3-venv unzip zip
+         apt install bzip2 ca-certificates file g++ gcc gfortran git gzip lsb-release patch python3 tar unzip xz-utils zstd
 
    .. tab-item:: RHEL
 
@@ -43,14 +43,14 @@ A build matrix showing which packages are working on which systems is shown belo
 
          dnf install epel-release
          dnf group install "Development Tools"
-         dnf install curl findutils gcc-gfortran gnupg2 hostname iproute redhat-lsb-core python3 python3-pip python3-setuptools unzip python3-boto3
+         dnf install gcc-gfortran redhat-lsb-core python3 unzip
 
    .. tab-item:: macOS Brew
 
       .. code-block:: console
 
          brew update
-         brew install curl gcc git gnupg zip
+         brew install gcc git zip
 
 ------------
 Installation
@@ -1364,6 +1364,187 @@ This will write the private key to the file `dinosaur.priv`.
     or for help on an issue or the Spack slack.
 
 
+.. _cray-support:
+
+-------------
+Spack on Cray
+-------------
+
+Spack differs slightly when used on a Cray system. The architecture spec
+can differentiate between the front-end and back-end processor and operating system.
+For example, on Edison at NERSC, the back-end target processor
+is "Ivy Bridge", so you can specify to use the back-end this way:
+
+.. code-block:: console
+
+   $ spack install zlib target=ivybridge
+
+You can also use the operating system to build against the back-end:
+
+.. code-block:: console
+
+   $ spack install zlib os=CNL10
+
+Notice that the name includes both the operating system name and the major
+version number concatenated together.
+
+Alternatively, if you want to build something for the front-end,
+you can specify the front-end target processor. The processor for a login node
+on Edison is "Sandy bridge" so we specify on the command line like so:
+
+.. code-block:: console
+
+   $ spack install zlib target=sandybridge
+
+And the front-end operating system is:
+
+.. code-block:: console
+
+   $ spack install zlib os=SuSE11
+
+^^^^^^^^^^^^^^^^^^^^^^^
+Cray compiler detection
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Spack can detect compilers using two methods. For the front-end, we treat
+everything the same. The difference lies in back-end compiler detection.
+Back-end compiler detection is made via the Tcl module avail command.
+Once it detects the compiler it writes the appropriate PrgEnv and compiler
+module name to compilers.yaml and sets the paths to each compiler with Cray\'s
+compiler wrapper names (i.e. cc, CC, ftn). During build time, Spack will load
+the correct PrgEnv and compiler module and will call appropriate wrapper.
+
+The compilers.yaml config file will also differ. There is a
+modules section that is filled with the compiler's Programming Environment
+and module name. On other systems, this field is empty []:
+
+.. code-block:: yaml
+
+   - compiler:
+       modules:
+         - PrgEnv-intel
+         - intel/15.0.109
+
+As mentioned earlier, the compiler paths will look different on a Cray system.
+Since most compilers are invoked using cc, CC and ftn, the paths for each
+compiler are replaced with their respective Cray compiler wrapper names:
+
+.. code-block:: yaml
+
+     paths:
+       cc: cc
+       cxx: CC
+       f77: ftn
+       fc: ftn
+
+As opposed to an explicit path to the compiler executable. This allows Spack
+to call the Cray compiler wrappers during build time.
+
+For more on compiler configuration, check out :ref:`compiler-config`.
+
+Spack sets the default Cray link type to dynamic, to better match other
+other platforms. Individual packages can enable static linking (which is the
+default outside of Spack on cray systems) using the ``-static`` flag.
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Setting defaults and using Cray modules
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you want to use default compilers for each PrgEnv and also be able
+to load cray external modules, you will need to set up a ``packages.yaml``.
+
+Here's an example of an external configuration for cray modules:
+
+.. code-block:: yaml
+
+   packages:
+     mpich:
+       externals:
+       - spec: "mpich@7.3.1%gcc@5.2.0 arch=cray_xc-haswell-CNL10"
+         modules:
+         - cray-mpich
+       - spec: "mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-haswell-CNL10"
+         modules:
+         - cray-mpich
+     all:
+       providers:
+         mpi: [mpich]
+
+This tells Spack that for whatever package that depends on mpi, load the
+cray-mpich module into the environment. You can then be able to use whatever
+environment variables, libraries, etc, that are brought into the environment
+via module load.
+
+.. note::
+
+    For Cray-provided packages, it is best to use ``modules:`` instead of ``prefix:``
+    in ``packages.yaml``, because the Cray Programming Environment heavily relies on
+    modules (e.g., loading the ``cray-mpich`` module adds MPI libraries to the
+    compiler wrapper link line).
+
+You can set the default compiler that Spack can use for each compiler type.
+If you want to use the Cray defaults, then set them under ``all:`` in packages.yaml.
+In the compiler field, set the compiler specs in your order of preference.
+Whenever you build with that compiler type, Spack will concretize to that version.
+
+Here is an example of a full packages.yaml used at NERSC
+
+.. code-block:: yaml
+
+   packages:
+     mpich:
+       externals:
+       - spec: "mpich@7.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge"
+         modules:
+         - cray-mpich
+       - spec: "mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-SuSE11-ivybridge"
+         modules:
+         - cray-mpich
+       buildable: False
+     netcdf:
+       externals:
+       - spec: "netcdf@4.3.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge"
+         modules:
+         - cray-netcdf
+       - spec: "netcdf@4.3.3.1%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge"
+         modules:
+         - cray-netcdf
+       buildable: False
+     hdf5:
+       externals:
+       - spec: "hdf5@1.8.14%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge"
+         modules:
+         - cray-hdf5
+       - spec: "hdf5@1.8.14%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge"
+         modules:
+         - cray-hdf5
+       buildable: False
+     all:
+       compiler: [gcc@5.2.0, intel@16.0.0.109]
+       providers:
+         mpi: [mpich]
+
+Here we tell spack that whenever we want to build with gcc use version 5.2.0 or
+if we want to build with intel compilers, use version 16.0.0.109. We add a spec
+for each compiler type for each cray modules. This ensures that for each
+compiler on our system we can use that external module.
+
+For more on external packages check out the section :ref:`sec-external-packages`.
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Using Linux containers on Cray machines
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Spack uses environment variables particular to the Cray programming
+environment to determine which systems are Cray platforms. These
+environment variables may be propagated into containers that are not
+using the Cray programming environment.
+
+To ensure that Spack does not autodetect the Cray programming
+environment, unset the environment variable ``MODULEPATH``. This
+will cause Spack to treat a linux container on a Cray system as a base
+linux distro.
+
 .. _windows_support:
 
 ----------------

lib/spack/docs/index.rst

@@ -12,10 +12,6 @@
 Spack
 ===================
 
-.. epigraph::
-
-   `These are docs for the Spack package manager. For sphere packing, see` `pyspack <https://pyspack.readthedocs.io>`_.
-
 Spack is a package management tool designed to support multiple
 versions and configurations of software on a wide variety of platforms
 and environments.  It was designed for large supercomputing centers,

lib/spack/docs/pipelines.rst

@@ -253,6 +253,17 @@ can easily happen if it is not updated frequently, this behavior ensures that
 spack has a way to know for certain about the status of any concrete spec on
 the remote mirror, but can slow down pipeline generation significantly.
 
+The ``--optimize`` argument is experimental and runs the generated pipeline
+document through a series of optimization passes designed to reduce the size
+of the generated file.
+
+The ``--dependencies`` is also experimental and disables what in Gitlab is
+referred to as DAG scheduling, internally using the ``dependencies`` keyword
+rather than ``needs`` to list dependency jobs.  The drawback of using this option
+is that before any job can begin, all jobs in previous stages must first
+complete.  The benefit is that Gitlab allows more dependencies to be listed
+when using ``dependencies`` instead of ``needs``.
+
 The optional ``--output-file`` argument should be an absolute path (including
 file name) to the generated pipeline, and if not given, the default is
 ``./.gitlab-ci.yml``.

lib/spack/docs/repositories.rst

@@ -476,3 +476,9 @@ implemented using Python's built-in `sys.path
 :py:mod:`spack.repo` module implements a custom `Python importer
 <https://docs.python.org/2/library/imp.html>`_.
 
+.. warning::
+
+   The mechanism for extending packages is not yet extensively tested,
+   and extending packages across repositories imposes inter-repo
+   dependencies, which may be hard to manage.  Use this feature at your
+   own risk, but let us know if you have a use case for it.

lib/spack/docs/requirements.txt

@@ -1,13 +1,13 @@
-sphinx==7.4.6
+sphinx==7.2.6
 sphinxcontrib-programoutput==0.17
-sphinx_design==0.6.0
+sphinx_design==0.5.0
 sphinx-rtd-theme==2.0.0
 python-levenshtein==0.25.1
 docutils==0.20.1
-pygments==2.18.0
-urllib3==2.2.2
-pytest==8.2.2
+pygments==2.17.2
+urllib3==2.2.1
+pytest==8.2.0
 isort==5.13.2
 black==24.4.2
-flake8==7.1.0
-mypy==1.10.1
+flake8==7.0.0
+mypy==1.10.0

lib/spack/external/__init__.py

@@ -18,7 +18,7 @@
 
 * Homepage: https://pypi.python.org/pypi/archspec
 * Usage: Labeling, comparison and detection of microarchitectures
-* Version: 0.2.4 (commit 48b92512b9ce203ded0ebd1ac41b42593e931f7c)
+* Version: 0.2.5 (commit 38ce485258ffc4fc6dd6688f8dc90cb269478c47)
 
 astunparse
 ----------------

lib/spack/external/archspec/cpu/detect.py

@@ -47,7 +47,11 @@ def decorator(factory):
 
 
 def partial_uarch(
-    name: str = "", vendor: str = "", features: Optional[Set[str]] = None, generation: int = 0
+    name: str = "",
+    vendor: str = "",
+    features: Optional[Set[str]] = None,
+    generation: int = 0,
+    cpu_part: str = "",
 ) -> Microarchitecture:
     """Construct a partial microarchitecture, from information gathered during system scan."""
     return Microarchitecture(
@@ -57,6 +61,7 @@ def partial_uarch(
         features=features or set(),
         compilers={},
         generation=generation,
+        cpu_part=cpu_part,
     )
 
 
@@ -90,6 +95,7 @@ def proc_cpuinfo() -> Microarchitecture:
         return partial_uarch(
             vendor=_canonicalize_aarch64_vendor(data),
             features=_feature_set(data, key="Features"),
+            cpu_part=data.get("CPU part", ""),
         )
 
     if architecture in (PPC64LE, PPC64):
@@ -345,6 +351,10 @@ def sorting_fn(item):
     generic_candidates = [c for c in candidates if c.vendor == "generic"]
     best_generic = max(generic_candidates, key=sorting_fn)
 
+    # Relevant for AArch64. Filter on "cpu_part" if we have any match
+    if info.cpu_part != "" and any(c for c in candidates if info.cpu_part == c.cpu_part):
+        candidates = [c for c in candidates if info.cpu_part == c.cpu_part]
+
     # Filter the candidates to be descendant of the best generic candidate.
     # This is to avoid that the lack of a niche feature that can be disabled
     # from e.g. BIOS prevents detection of a reasonably performant architecture

lib/spack/external/archspec/cpu/microarchitecture.py

@@ -2,9 +2,7 @@
 # Archspec Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-"""Types and functions to manage information
-on CPU microarchitectures.
-"""
+"""Types and functions to manage information on CPU microarchitectures."""
 import functools
 import platform
 import re
@@ -65,23 +63,31 @@ class Microarchitecture:
                 passed in as argument above.
             * versions: versions that support this micro-architecture.
 
-        generation (int): generation of the micro-architecture, if
-            relevant.
+        generation (int): generation of the micro-architecture, if relevant.
+        cpu_part (str): cpu part of the architecture, if relevant.
     """
 
-    # pylint: disable=too-many-arguments
+    # pylint: disable=too-many-arguments,too-many-instance-attributes
     #: Aliases for micro-architecture's features
     feature_aliases = FEATURE_ALIASES
 
-    def __init__(self, name, parents, vendor, features, compilers, generation=0):
+    def __init__(self, name, parents, vendor, features, compilers, generation=0, cpu_part=""):
         self.name = name
         self.parents = parents
         self.vendor = vendor
         self.features = features
         self.compilers = compilers
+        # Only relevant for PowerPC
         self.generation = generation
-        # Cache the ancestor computation
+        # Only relevant for AArch64
+        self.cpu_part = cpu_part
+
+        # Cache the "ancestor" computation
         self._ancestors = None
+        # Cache the "generic" computation
+        self._generic = None
+        # Cache the "family" computation
+        self._family = None
 
     @property
     def ancestors(self):
@@ -111,8 +117,12 @@ def __eq__(self, other):
             and self.parents == other.parents  # avoid ancestors here
             and self.compilers == other.compilers
             and self.generation == other.generation
+            and self.cpu_part == other.cpu_part
         )
 
+    def __hash__(self):
+        return hash(self.name)
+
     @coerce_target_names
     def __ne__(self, other):
         return not self == other
@@ -143,7 +153,8 @@ def __repr__(self):
         cls_name = self.__class__.__name__
         fmt = (
             cls_name + "({0.name!r}, {0.parents!r}, {0.vendor!r}, "
-            "{0.features!r}, {0.compilers!r}, {0.generation!r})"
+            "{0.features!r}, {0.compilers!r}, generation={0.generation!r}, "
+            "cpu_part={0.cpu_part!r})"
         )
         return fmt.format(self)
 
@@ -168,18 +179,22 @@ def __contains__(self, feature):
     @property
     def family(self):
         """Returns the architecture family a given target belongs to"""
-        roots = [x for x in [self] + self.ancestors if not x.ancestors]
-        msg = "a target is expected to belong to just one architecture family"
-        msg += f"[found {', '.join(str(x) for x in roots)}]"
-        assert len(roots) == 1, msg
+        if self._family is None:
+            roots = [x for x in [self] + self.ancestors if not x.ancestors]
+            msg = "a target is expected to belong to just one architecture family"
+            msg += f"[found {', '.join(str(x) for x in roots)}]"
+            assert len(roots) == 1, msg
+            self._family = roots.pop()
 
-        return roots.pop()
+        return self._family
 
     @property
     def generic(self):
         """Returns the best generic architecture that is compatible with self"""
-        generics = [x for x in [self] + self.ancestors if x.vendor == "generic"]
-        return max(generics, key=lambda x: len(x.ancestors))
+        if self._generic is None:
+            generics = [x for x in [self] + self.ancestors if x.vendor == "generic"]
+            self._generic = max(generics, key=lambda x: len(x.ancestors))
+        return self._generic
 
     def to_dict(self):
         """Returns a dictionary representation of this object."""
@@ -190,6 +205,7 @@ def to_dict(self):
             "generation": self.generation,
             "parents": [str(x) for x in self.parents],
             "compilers": self.compilers,
+            "cpupart": self.cpu_part,
         }
 
     @staticmethod
@@ -202,6 +218,7 @@ def from_dict(data) -> "Microarchitecture":
             features=set(data["features"]),
             compilers=data.get("compilers", {}),
             generation=data.get("generation", 0),
+            cpu_part=data.get("cpupart", ""),
         )
 
     def optimization_flags(self, compiler, version):
@@ -360,8 +377,11 @@ def fill_target_from_dict(name, data, targets):
         features = set(values["features"])
         compilers = values.get("compilers", {})
         generation = values.get("generation", 0)
+        cpu_part = values.get("cpupart", "")
 
-        targets[name] = Microarchitecture(name, parents, vendor, features, compilers, generation)
+        targets[name] = Microarchitecture(
+            name, parents, vendor, features, compilers, generation=generation, cpu_part=cpu_part
+        )
 
     known_targets = {}
     data = archspec.cpu.schema.TARGETS_JSON["microarchitectures"]

lib/spack/external/archspec/json/cpu/microarchitectures.json

@@ -1482,7 +1482,6 @@
         "cldemote",
         "movdir64b",
         "movdiri",
-        "pdcm",
         "serialize",
         "waitpkg"
       ],
@@ -2225,14 +2224,96 @@
         ],
         "nvhpc": [
           {
-            "versions": "21.11:",
+            "versions": "21.11:23.8",
 	    "name": "zen3",
             "flags": "-tp {name}",
-	    "warnings": "zen4 is not fully supported by nvhpc yet, falling back to zen3"
+	    "warnings": "zen4 is not fully supported by nvhpc versions < 23.9, falling back to zen3"
+          },
+          {
+            "versions": "23.9:",
+            "flags": "-tp {name}"
           }
 	]
       }
     },
+    "zen5": {
+      "from": ["zen4"],
+      "vendor": "AuthenticAMD",
+      "features": [
+        "abm",
+        "aes",
+        "avx",
+        "avx2",
+        "avx512_bf16",
+        "avx512_bitalg",
+        "avx512bw",
+        "avx512cd",
+        "avx512dq",
+        "avx512f",
+        "avx512ifma",
+        "avx512vbmi",
+        "avx512_vbmi2",
+        "avx512vl",
+        "avx512_vnni",
+        "avx512_vp2intersect",
+        "avx512_vpopcntdq",
+        "avx_vnni",
+        "bmi1",
+        "bmi2",
+        "clflushopt",
+        "clwb",
+        "clzero",
+        "cppc",
+        "cx16",
+        "f16c",
+        "flush_l1d",
+        "fma",
+        "fsgsbase",
+        "gfni",
+        "ibrs_enhanced",
+        "mmx",
+        "movbe",
+        "movdir64b",
+        "movdiri",
+        "pclmulqdq",
+        "popcnt",
+        "rdseed",
+        "sse",
+        "sse2",
+        "sse4_1",
+        "sse4_2",
+        "sse4a",
+        "ssse3",
+        "tsc_adjust",
+        "vaes",
+        "vpclmulqdq",
+        "xsavec",
+        "xsaveopt"
+      ],
+      "compilers": {
+        "gcc": [
+          {
+            "versions": "14.1:",
+            "name": "znver5",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
+        "aocc": [
+          {
+            "versions": "5.0:",
+            "name": "znver5",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
+        "clang": [
+          {
+            "versions": "19.1:",
+            "name": "znver5",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ]
+      }
+    },
     "ppc64": {
       "from": [],
       "vendor": "generic",
@@ -2711,7 +2792,8 @@
             "flags": "-mcpu=thunderx2t99"
           }
         ]
-      }
+      },
+      "cpupart": "0x0af"
     },
     "a64fx": {
       "from": ["armv8.2a"],
@@ -2779,7 +2861,8 @@
             "flags": "-march=armv8.2-a+crc+crypto+fp16+sve"
           }
         ]
-      }
+      },
+      "cpupart": "0x001"
     },
     "cortex_a72": {
       "from": ["aarch64"],
@@ -2816,7 +2899,8 @@
                   "flags" : "-mcpu=cortex-a72"
               }
           ]
-      }
+      },
+      "cpupart": "0xd08"
     },
     "neoverse_n1": {
       "from": ["cortex_a72", "armv8.2a"],
@@ -2837,8 +2921,7 @@
           "asimdrdm",
           "lrcpc",
           "dcpop",
-          "asimddp",
-          "ssbs"
+          "asimddp"
       ],
       "compilers" : {
           "gcc": [
@@ -2902,7 +2985,8 @@
                   "flags": "-tp {name}"
               }
           ]
-      }
+      },
+      "cpupart": "0xd0c"
     },
     "neoverse_v1": {
       "from": ["neoverse_n1", "armv8.4a"],
@@ -2926,8 +3010,6 @@
           "lrcpc",
           "dcpop",
           "sha3",
-          "sm3",
-          "sm4",
           "asimddp",
           "sha512",
           "sve",
@@ -2936,7 +3018,6 @@
           "uscat",
           "ilrcpc",
           "flagm",
-          "ssbs",
           "dcpodp",
           "svei8mm",
           "svebf16",
@@ -3004,7 +3085,7 @@
               },
               {
                   "versions": "11:",
-                  "flags" : "-march=armv8.4-a+sve+ssbs+fp16+bf16+crypto+i8mm+rng"
+                  "flags" : "-march=armv8.4-a+sve+fp16+bf16+crypto+i8mm+rng"
               },
               {
                   "versions": "12:",
@@ -3028,7 +3109,8 @@
                   "flags": "-tp {name}"
               }
           ]
-      }
+      },
+      "cpupart": "0xd40"
     },
     "neoverse_v2": {
       "from": ["neoverse_n1", "armv9.0a"],
@@ -3052,32 +3134,22 @@
 	  "lrcpc",
 	  "dcpop",
 	  "sha3",
-	  "sm3",
-	  "sm4",
 	  "asimddp",
 	  "sha512",
 	  "sve",
 	  "asimdfhm",
-	  "dit",
 	  "uscat",
 	  "ilrcpc",
 	  "flagm",
-	  "ssbs",
 	  "sb",
 	  "dcpodp",
 	  "sve2",
-	  "sveaes",
-	  "svepmull",
-	  "svebitperm",
-	  "svesha3",
-	  "svesm4",
 	  "flagm2",
 	  "frint",
 	  "svei8mm",
 	  "svebf16",
 	  "i8mm",
-	  "bf16",
-	  "dgh"
+	  "bf16"
       ],
       "compilers" : {
           "gcc": [
@@ -3102,15 +3174,19 @@
                   "flags" : "-march=armv8.5-a+sve -mtune=cortex-a76"
               },
               {
-                  "versions": "10.0:11.99",
+                  "versions": "10.0:11.3.99",
                   "flags" : "-march=armv8.5-a+sve+sve2+i8mm+bf16 -mtune=cortex-a77"
               },
+	      {
+                  "versions": "11.4:11.99",
+                  "flags" : "-mcpu=neoverse-v2"
+              },
               {
-                  "versions": "12.0:12.99",
+                  "versions": "12.0:12.2.99",
                   "flags" : "-march=armv9-a+i8mm+bf16 -mtune=cortex-a710"
               },
 	      {
-                  "versions": "13.0:",
+                  "versions": "12.3:",
                   "flags" : "-mcpu=neoverse-v2"
               }
           ],
@@ -3145,7 +3221,112 @@
                   "flags": "-tp {name}"
               }
           ]
-      }
+      },
+      "cpupart": "0xd4f"
+    },
+    "neoverse_n2": {
+      "from": ["neoverse_n1", "armv9.0a"],
+      "vendor": "ARM",
+      "features": [
+          "fp",
+	  "asimd",
+	  "evtstrm",
+	  "aes",
+	  "pmull",
+	  "sha1",
+	  "sha2",
+	  "crc32",
+	  "atomics",
+	  "fphp",
+	  "asimdhp",
+	  "cpuid",
+	  "asimdrdm",
+	  "jscvt",
+	  "fcma",
+	  "lrcpc",
+	  "dcpop",
+	  "sha3",
+	  "asimddp",
+	  "sha512",
+	  "sve",
+	  "asimdfhm",
+	  "uscat",
+	  "ilrcpc",
+	  "flagm",
+	  "sb",
+	  "dcpodp",
+	  "sve2",
+	  "flagm2",
+	  "frint",
+	  "svei8mm",
+	  "svebf16",
+	  "i8mm",
+	  "bf16"
+      ],
+      "compilers" : {
+          "gcc": [
+              {
+                  "versions": "4.8:5.99",
+                  "flags": "-march=armv8-a"
+              },
+              {
+                  "versions": "6:6.99",
+                  "flags" : "-march=armv8.1-a"
+              },
+              {
+                  "versions": "7.0:7.99",
+                  "flags" : "-march=armv8.2-a -mtune=cortex-a72"
+              },
+              {
+                  "versions": "8.0:8.99",
+                  "flags" : "-march=armv8.4-a+sve -mtune=cortex-a72"
+              },
+              {
+                  "versions": "9.0:9.99",
+                  "flags" : "-march=armv8.5-a+sve -mtune=cortex-a76"
+              },
+              {
+                  "versions": "10.0:10.99",
+                  "flags" : "-march=armv8.5-a+sve+sve2+i8mm+bf16 -mtune=cortex-a77"
+              },
+              {
+                  "versions": "11.0:",
+                  "flags" : "-mcpu=neoverse-n2"
+              }
+          ],
+          "clang" : [
+              {
+                  "versions": "9.0:10.99",
+                  "flags" : "-march=armv8.5-a+sve"
+              },
+              {
+                  "versions": "11.0:13.99",
+                  "flags" : "-march=armv8.5-a+sve+sve2+i8mm+bf16"
+              },
+              {
+                  "versions": "14.0:15.99",
+                  "flags" : "-march=armv9-a+i8mm+bf16"
+              },
+              {
+                  "versions": "16.0:",
+                  "flags" : "-mcpu=neoverse-n2"
+              }
+          ],
+          "arm" : [
+              {
+                  "versions": "23.04.0:",
+                  "flags" : "-mcpu=neoverse-n2"
+              }
+          ],
+          "nvhpc" : [
+              {
+                  "versions": "23.3:",
+                  "name": "neoverse-n1",
+                  "flags": "-tp {name}"
+              }
+          ]
+      },
+      "cpupart": "0xd49"
     },
     "m1": {
       "from": ["armv8.4a"],
@@ -3211,7 +3392,8 @@
             "flags" : "-mcpu=apple-m1"
           }
         ]
-      }
+      },
+      "cpupart": "0x022"
     },
     "m2": {
       "from": ["m1", "armv8.5a"],
@@ -3289,7 +3471,8 @@
             "flags" : "-mcpu=apple-m2"
           }
         ]
-      }
+      },
+      "cpupart": "0x032"
     },
     "arm": {
       "from": [],

lib/spack/external/archspec/json/cpu/microarchitectures_schema.json

@@ -52,6 +52,9 @@
                   }
                 }
               }
+            },
+            "cpupart": {
+              "type": "string"
             }
           },
           "required": [
@@ -107,4 +110,4 @@
       "additionalProperties": false
     }
   }
-}
+}

lib/spack/llnl/util/filesystem.py

@@ -766,6 +766,7 @@ def copy_tree(
     src: str,
     dest: str,
     symlinks: bool = True,
+    allow_broken_symlinks: bool = sys.platform != "win32",
     ignore: Optional[Callable[[str], bool]] = None,
     _permissions: bool = False,
 ):
@@ -788,6 +789,8 @@ def copy_tree(
         src (str): the directory to copy
         dest (str): the destination directory
         symlinks (bool): whether or not to preserve symlinks
+        allow_broken_symlinks (bool): whether or not to allow broken (dangling) symlinks,
+            On Windows, setting this to True will raise an exception. Defaults to true on unix.
         ignore (typing.Callable): function indicating which files to ignore
         _permissions (bool): for internal use only
 
@@ -795,6 +798,8 @@ def copy_tree(
         IOError: if *src* does not match any files or directories
         ValueError: if *src* is a parent directory of *dest*
     """
+    if allow_broken_symlinks and sys.platform == "win32":
+        raise llnl.util.symlink.SymlinkError("Cannot allow broken symlinks on Windows!")
     if _permissions:
         tty.debug("Installing {0} to {1}".format(src, dest))
     else:
@@ -867,14 +872,16 @@ def escaped_path(path):
                 copy_mode(s, d)
 
     for target, d, s in links:
-        symlink(target, d)
+        symlink(target, d, allow_broken_symlinks=allow_broken_symlinks)
         if _permissions:
             set_install_permissions(d)
             copy_mode(s, d)
 
 
 @system_path_filter
-def install_tree(src, dest, symlinks=True, ignore=None):
+def install_tree(
+    src, dest, symlinks=True, ignore=None, allow_broken_symlinks=sys.platform != "win32"
+):
     """Recursively install an entire directory tree rooted at *src*.
 
     Same as :py:func:`copy_tree` with the addition of setting proper
@@ -885,12 +892,21 @@ def install_tree(src, dest, symlinks=True, ignore=None):
         dest (str): the destination directory
         symlinks (bool): whether or not to preserve symlinks
         ignore (typing.Callable): function indicating which files to ignore
+        allow_broken_symlinks (bool): whether or not to allow broken (dangling) symlinks,
+            On Windows, setting this to True will raise an exception.
 
     Raises:
         IOError: if *src* does not match any files or directories
         ValueError: if *src* is a parent directory of *dest*
     """
-    copy_tree(src, dest, symlinks=symlinks, ignore=ignore, _permissions=True)
+    copy_tree(
+        src,
+        dest,
+        symlinks=symlinks,
+        allow_broken_symlinks=allow_broken_symlinks,
+        ignore=ignore,
+        _permissions=True,
+    )
 
 
 @system_path_filter

lib/spack/llnl/util/symlink.py

@@ -8,7 +8,6 @@
 import subprocess
 import sys
 import tempfile
-from typing import Union
 
 from llnl.util import lang, tty
 
@@ -17,66 +16,92 @@
 if sys.platform == "win32":
     from win32file import CreateHardLink
 
+is_windows = sys.platform == "win32"
 
-def _windows_symlink(
-    src: str, dst: str, target_is_directory: bool = False, *, dir_fd: Union[int, None] = None
-):
-    """On Windows with System Administrator privileges this will be a normal symbolic link via
-    os.symlink. On Windows without privledges the link will be a junction for a directory and a
-    hardlink for a file. On Windows the various link types are:
 
-    Symbolic Link: A link to a file or directory on the same or different volume (drive letter) or
-    even to a remote file or directory (using UNC in its path). Need System Administrator
-    privileges to make these.
+def symlink(source_path: str, link_path: str, allow_broken_symlinks: bool = not is_windows):
+    """
+    Create a link.
 
-    Hard Link: A link to a file on the same volume (drive letter) only. Every file (file's data)
-    has at least 1 hard link (file's name). But when this method creates a new hard link there will
-    be 2. Deleting all hard links effectively deletes the file. Don't need System Administrator
-    privileges.
+    On non-Windows and Windows with System Administrator
+    privleges this will be a normal symbolic link via
+    os.symlink.
 
-    Junction: A link to a directory on the same or different volume (drive letter) but not to a
-    remote directory. Don't need System Administrator privileges."""
-    source_path = os.path.normpath(src)
+    On Windows without privledges the link will be a
+    junction for a directory and a hardlink for a file.
+    On Windows the various link types are:
+
+    Symbolic Link: A link to a file or directory on the
+    same or different volume (drive letter) or even to
+    a remote file or directory (using UNC in its path).
+    Need System Administrator privileges to make these.
+
+    Hard Link: A link to a file on the same volume (drive
+    letter) only. Every file (file's data) has at least 1
+    hard link (file's name). But when this method creates
+    a new hard link there will be 2. Deleting all hard
+    links effectively deletes the file. Don't need System
+    Administrator privileges.
+
+    Junction: A link to a directory on the same or different
+    volume (drive letter) but not to a remote directory. Don't
+    need System Administrator privileges.
+
+    Parameters:
+        source_path (str): The real file or directory that the link points to.
+            Must be absolute OR relative to the link.
+        link_path (str): The path where the link will exist.
+        allow_broken_symlinks (bool): On Linux or Mac, don't raise an exception if the source_path
+            doesn't exist. This will still raise an exception on Windows.
+    """
+    source_path = os.path.normpath(source_path)
     win_source_path = source_path
-    link_path = os.path.normpath(dst)
+    link_path = os.path.normpath(link_path)
 
-    # Perform basic checks to make sure symlinking will succeed
-    if os.path.lexists(link_path):
-        raise AlreadyExistsError(f"Link path ({link_path}) already exists. Cannot create link.")
+    # Never allow broken links on Windows.
+    if sys.platform == "win32" and allow_broken_symlinks:
+        raise ValueError("allow_broken_symlinks parameter cannot be True on Windows.")
 
-    if not os.path.exists(source_path):
-        if os.path.isabs(source_path):
-            # An absolute source path that does not exist will result in a broken link.
-            raise SymlinkError(
-                f"Source path ({source_path}) is absolute but does not exist. Resulting "
-                f"link would be broken so not making link."
+    if not allow_broken_symlinks:
+        # Perform basic checks to make sure symlinking will succeed
+        if os.path.lexists(link_path):
+            raise AlreadyExistsError(
+                f"Link path ({link_path}) already exists. Cannot create link."
             )
-        else:
-            # os.symlink can create a link when the given source path is relative to
-            # the link path. Emulate this behavior and check to see if the source exists
-            # relative to the link path ahead of link creation to prevent broken
-            # links from being made.
-            link_parent_dir = os.path.dirname(link_path)
-            relative_path = os.path.join(link_parent_dir, source_path)
-            if os.path.exists(relative_path):
-                # In order to work on windows, the source path needs to be modified to be
-                # relative because hardlink/junction dont resolve relative paths the same
-                # way as os.symlink. This is ignored on other operating systems.
-                win_source_path = relative_path
-            else:
+
+        if not os.path.exists(source_path):
+            if os.path.isabs(source_path) and not allow_broken_symlinks:
+                # An absolute source path that does not exist will result in a broken link.
                 raise SymlinkError(
-                    f"The source path ({source_path}) is not relative to the link path "
-                    f"({link_path}). Resulting link would be broken so not making link."
+                    f"Source path ({source_path}) is absolute but does not exist. Resulting "
+                    f"link would be broken so not making link."
                 )
+            else:
+                # os.symlink can create a link when the given source path is relative to
+                # the link path. Emulate this behavior and check to see if the source exists
+                # relative to the link path ahead of link creation to prevent broken
+                # links from being made.
+                link_parent_dir = os.path.dirname(link_path)
+                relative_path = os.path.join(link_parent_dir, source_path)
+                if os.path.exists(relative_path):
+                    # In order to work on windows, the source path needs to be modified to be
+                    # relative because hardlink/junction dont resolve relative paths the same
+                    # way as os.symlink. This is ignored on other operating systems.
+                    win_source_path = relative_path
+                elif not allow_broken_symlinks:
+                    raise SymlinkError(
+                        f"The source path ({source_path}) is not relative to the link path "
+                        f"({link_path}). Resulting link would be broken so not making link."
+                    )
 
     # Create the symlink
-    if not _windows_can_symlink():
+    if sys.platform == "win32" and not _windows_can_symlink():
         _windows_create_link(win_source_path, link_path)
     else:
         os.symlink(source_path, link_path, target_is_directory=os.path.isdir(source_path))
 
 
-def _windows_islink(path: str) -> bool:
+def islink(path: str) -> bool:
     """Override os.islink to give correct answer for spack logic.
 
     For Non-Windows: a link can be determined with the os.path.islink method.
@@ -244,7 +269,7 @@ def _windows_create_hard_link(path: str, link: str):
         CreateHardLink(link, path)
 
 
-def _windows_readlink(path: str, *, dir_fd=None):
+def readlink(path: str, *, dir_fd=None):
     """Spack utility to override of os.readlink method to work cross platform"""
     if _windows_is_hardlink(path):
         return _windows_read_hard_link(path)
@@ -313,16 +338,6 @@ def resolve_link_target_relative_to_the_link(link):
     return os.path.join(link_dir, target)
 
 
-if sys.platform == "win32":
-    symlink = _windows_symlink
-    readlink = _windows_readlink
-    islink = _windows_islink
-else:
-    symlink = os.symlink
-    readlink = os.readlink
-    islink = os.path.islink
-
-
 class SymlinkError(RuntimeError):
     """Exception class for errors raised while creating symlinks,
     junctions and hard links

lib/spack/llnl/util/tty/log.py

@@ -18,9 +18,10 @@
 import threading
 import traceback
 from contextlib import contextmanager
+from multiprocessing.connection import Connection
 from threading import Thread
 from types import ModuleType
-from typing import Optional
+from typing import Callable, Optional
 
 import llnl.util.tty as tty
 
@@ -33,23 +34,8 @@
     pass
 
 
-esc, bell, lbracket, bslash, newline = r"\x1b", r"\x07", r"\[", r"\\", r"\n"
-# Ansi Control Sequence Introducers (CSI) are a well-defined format
-# Standard ECMA-48: Control Functions for Character-Imaging I/O Devices, section 5.4
-# https://www.ecma-international.org/wp-content/uploads/ECMA-48_5th_edition_june_1991.pdf
-csi_pre = f"{esc}{lbracket}"
-csi_param, csi_inter, csi_post = r"[0-?]", r"[ -/]", r"[@-~]"
-ansi_csi = f"{csi_pre}{csi_param}*{csi_inter}*{csi_post}"
-# General ansi escape sequences have well-defined prefixes,
-#  but content and suffixes are less reliable.
-# Conservatively assume they end with either "<ESC>\" or "<BELL>",
-#  with no intervening "<ESC>"/"<BELL>" keys or newlines
-esc_pre = f"{esc}[@-_]"
-esc_content = f"[^{esc}{bell}{newline}]"
-esc_post = f"(?:{esc}{bslash}|{bell})"
-ansi_esc = f"{esc_pre}{esc_content}*{esc_post}"
 # Use this to strip escape sequences
-_escape = re.compile(f"{ansi_csi}|{ansi_esc}")
+_escape = re.compile(r"\x1b[^m]*m|\x1b\[?1034h|\x1b\][0-9]+;[^\x07]*\x07")
 
 # control characters for enabling/disabling echo
 #
@@ -344,49 +330,6 @@ def close(self):
             self.file.close()
 
 
-class MultiProcessFd:
-    """Return an object which stores a file descriptor and can be passed as an
-    argument to a function run with ``multiprocessing.Process``, such that
-    the file descriptor is available in the subprocess."""
-
-    def __init__(self, fd):
-        self._connection = None
-        self._fd = None
-        if sys.version_info >= (3, 8):
-            self._connection = multiprocessing.connection.Connection(fd)
-        else:
-            self._fd = fd
-
-    @property
-    def fd(self):
-        if self._connection:
-            return self._connection._handle
-        else:
-            return self._fd
-
-    def close(self):
-        if self._connection:
-            self._connection.close()
-        else:
-            os.close(self._fd)
-
-
-def close_connection_and_file(multiprocess_fd, file):
-    # MultiprocessFd is intended to transmit a FD
-    # to a child process, this FD is then opened to a Python File object
-    # (using fdopen). In >= 3.8, MultiprocessFd encapsulates a
-    # multiprocessing.connection.Connection; Connection closes the FD
-    # when it is deleted, and prints a warning about duplicate closure if
-    # it is not explicitly closed. In < 3.8, MultiprocessFd encapsulates a
-    # simple FD; closing the FD here appears to conflict with
-    # closure of the File object (in < 3.8 that is). Therefore this needs
-    # to choose whether to close the File or the Connection.
-    if sys.version_info >= (3, 8):
-        multiprocess_fd.close()
-    else:
-        file.close()
-
-
 @contextmanager
 def replace_environment(env):
     """Replace the current environment (`os.environ`) with `env`.
@@ -544,22 +487,20 @@ def __enter__(self):
         # forcing debug output.
         self._saved_debug = tty._debug
 
-        # OS-level pipe for redirecting output to logger
-        read_fd, write_fd = os.pipe()
+        # Pipe for redirecting output to logger
+        read_fd, self.write_fd = multiprocessing.Pipe(duplex=False)
 
-        read_multiprocess_fd = MultiProcessFd(read_fd)
-
-        # Multiprocessing pipe for communication back from the daemon
+        # Pipe for communication back from the daemon
         # Currently only used to save echo value between uses
-        self.parent_pipe, child_pipe = multiprocessing.Pipe()
+        self.parent_pipe, child_pipe = multiprocessing.Pipe(duplex=False)
 
         # Sets a daemon that writes to file what it reads from a pipe
         try:
             # need to pass this b/c multiprocessing closes stdin in child.
-            input_multiprocess_fd = None
+            input_fd = None
             try:
                 if sys.stdin.isatty():
-                    input_multiprocess_fd = MultiProcessFd(os.dup(sys.stdin.fileno()))
+                    input_fd = Connection(os.dup(sys.stdin.fileno()))
             except BaseException:
                 # just don't forward input if this fails
                 pass
@@ -568,9 +509,9 @@ def __enter__(self):
                 self.process = multiprocessing.Process(
                     target=_writer_daemon,
                     args=(
-                        input_multiprocess_fd,
-                        read_multiprocess_fd,
-                        write_fd,
+                        input_fd,
+                        read_fd,
+                        self.write_fd,
                         self.echo,
                         self.log_file,
                         child_pipe,
@@ -581,9 +522,9 @@ def __enter__(self):
                 self.process.start()
 
         finally:
-            if input_multiprocess_fd:
-                input_multiprocess_fd.close()
-            read_multiprocess_fd.close()
+            if input_fd:
+                input_fd.close()
+            read_fd.close()
 
         # Flush immediately before redirecting so that anything buffered
         # goes to the original stream
@@ -601,9 +542,9 @@ def __enter__(self):
             self._saved_stderr = os.dup(sys.stderr.fileno())
 
             # redirect to the pipe we created above
-            os.dup2(write_fd, sys.stdout.fileno())
-            os.dup2(write_fd, sys.stderr.fileno())
-            os.close(write_fd)
+            os.dup2(self.write_fd.fileno(), sys.stdout.fileno())
+            os.dup2(self.write_fd.fileno(), sys.stderr.fileno())
+            self.write_fd.close()
 
         else:
             # Handle I/O the Python way. This won't redirect lower-level
@@ -616,7 +557,7 @@ def __enter__(self):
             self._saved_stderr = sys.stderr
 
             # create a file object for the pipe; redirect to it.
-            pipe_fd_out = os.fdopen(write_fd, "w")
+            pipe_fd_out = os.fdopen(self.write_fd.fileno(), "w", closefd=False)
             sys.stdout = pipe_fd_out
             sys.stderr = pipe_fd_out
 
@@ -652,6 +593,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         else:
             sys.stdout = self._saved_stdout
             sys.stderr = self._saved_stderr
+            self.write_fd.close()
 
         # print log contents in parent if needed.
         if self.log_file.write_in_parent:
@@ -865,14 +807,14 @@ def force_echo(self):
 
 
 def _writer_daemon(
-    stdin_multiprocess_fd,
-    read_multiprocess_fd,
-    write_fd,
-    echo,
-    log_file_wrapper,
-    control_pipe,
-    filter_fn,
-):
+    stdin_fd: Optional[Connection],
+    read_fd: Connection,
+    write_fd: Connection,
+    echo: bool,
+    log_file_wrapper: FileWrapper,
+    control_fd: Connection,
+    filter_fn: Optional[Callable[[str], str]],
+) -> None:
     """Daemon used by ``log_output`` to write to a log file and to ``stdout``.
 
     The daemon receives output from the parent process and writes it both
@@ -909,43 +851,37 @@ def _writer_daemon(
     ``StringIO`` in the parent. This is mainly for testing.
 
     Arguments:
-        stdin_multiprocess_fd (int): input from the terminal
-        read_multiprocess_fd (int): pipe for reading from parent's redirected
-            stdout
-        echo (bool): initial echo setting -- controlled by user and
-            preserved across multiple writer daemons
-        log_file_wrapper (FileWrapper): file to log all output
-        control_pipe (Pipe): multiprocessing pipe on which to send control
-            information to the parent
-        filter_fn (callable, optional): function to filter each line of output
+        stdin_fd: optional input from the terminal
+        read_fd: pipe for reading from parent's redirected stdout
+        echo: initial echo setting -- controlled by user and preserved across multiple writer
+            daemons
+        log_file_wrapper: file to log all output
+        control_pipe: multiprocessing pipe on which to send control information to the parent
+        filter_fn: optional function to filter each line of output
 
     """
-    # If this process was forked, then it will inherit file descriptors from
-    # the parent process. This process depends on closing all instances of
-    # write_fd to terminate the reading loop, so we close the file descriptor
-    # here. Forking is the process spawning method everywhere except Mac OS
-    # for Python >= 3.8 and on Windows
-    if sys.version_info < (3, 8) or sys.platform != "darwin":
-        os.close(write_fd)
+    # This process depends on closing all instances of write_pipe to terminate the reading loop
+    write_fd.close()
 
     # 1. Use line buffering (3rd param = 1) since Python 3 has a bug
     # that prevents unbuffered text I/O.
     # 2. Python 3.x before 3.7 does not open with UTF-8 encoding by default
-    in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1, encoding="utf-8")
+    # 3. closefd=False because Connection has "ownership"
+    read_file = os.fdopen(read_fd.fileno(), "r", 1, encoding="utf-8", closefd=False)
 
-    if stdin_multiprocess_fd:
-        stdin = os.fdopen(stdin_multiprocess_fd.fd)
+    if stdin_fd:
+        stdin_file = os.fdopen(stdin_fd.fileno(), closefd=False)
     else:
-        stdin = None
+        stdin_file = None
 
     # list of streams to select from
-    istreams = [in_pipe, stdin] if stdin else [in_pipe]
+    istreams = [read_file, stdin_file] if stdin_file else [read_file]
     force_echo = False  # parent can force echo for certain output
 
     log_file = log_file_wrapper.unwrap()
 
     try:
-        with keyboard_input(stdin) as kb:
+        with keyboard_input(stdin_file) as kb:
             while True:
                 # fix the terminal settings if we recently came to
                 # the foreground
@@ -958,12 +894,12 @@ def _writer_daemon(
                 # Allow user to toggle echo with 'v' key.
                 # Currently ignores other chars.
                 # only read stdin if we're in the foreground
-                if stdin in rlist and not _is_background_tty(stdin):
+                if stdin_file and stdin_file in rlist and not _is_background_tty(stdin_file):
                     # it's possible to be backgrounded between the above
                     # check and the read, so we ignore SIGTTIN here.
                     with ignore_signal(signal.SIGTTIN):
                         try:
-                            if stdin.read(1) == "v":
+                            if stdin_file.read(1) == "v":
                                 echo = not echo
                         except IOError as e:
                             # If SIGTTIN is ignored, the system gives EIO
@@ -972,13 +908,13 @@ def _writer_daemon(
                             if e.errno != errno.EIO:
                                 raise
 
-                if in_pipe in rlist:
+                if read_file in rlist:
                     line_count = 0
                     try:
                         while line_count < 100:
                             # Handle output from the calling process.
                             try:
-                                line = _retry(in_pipe.readline)()
+                                line = _retry(read_file.readline)()
                             except UnicodeDecodeError:
                                 # installs like --test=root gpgme produce non-UTF8 logs
                                 line = "<line lost: output was not encoded as UTF-8>\n"
@@ -1007,7 +943,7 @@ def _writer_daemon(
                                 if xoff in controls:
                                     force_echo = False
 
-                            if not _input_available(in_pipe):
+                            if not _input_available(read_file):
                                 break
                     finally:
                         if line_count > 0:
@@ -1022,14 +958,14 @@ def _writer_daemon(
     finally:
         # send written data back to parent if we used a StringIO
         if isinstance(log_file, io.StringIO):
-            control_pipe.send(log_file.getvalue())
+            control_fd.send(log_file.getvalue())
         log_file_wrapper.close()
-        close_connection_and_file(read_multiprocess_fd, in_pipe)
-        if stdin_multiprocess_fd:
-            close_connection_and_file(stdin_multiprocess_fd, stdin)
+        read_fd.close()
+        if stdin_fd:
+            stdin_fd.close()
 
         # send echo value back to the parent so it can be preserved.
-        control_pipe.send(echo)
+        control_fd.send(echo)
 
 
 def _retry(function):

lib/spack/spack/__init__.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 #: PEP440 canonical <major>.<minor>.<micro>.<devN> string
-__version__ = "0.23.0.dev0"
+__version__ = "0.22.3"
 spack_version = __version__
 
 

lib/spack/spack/audit.py

@@ -421,10 +421,6 @@ def _check_patch_urls(pkgs, error_cls):
         r"^https?://(?:patch-diff\.)?github(?:usercontent)?\.com/"
         r".+/.+/(?:commit|pull)/[a-fA-F0-9]+\.(?:patch|diff)"
     )
-    github_pull_commits_re = (
-        r"^https?://(?:patch-diff\.)?github(?:usercontent)?\.com/"
-        r".+/.+/pull/\d+/commits/[a-fA-F0-9]+\.(?:patch|diff)"
-    )
     # Only .diff URLs have stable/full hashes:
     # https://forum.gitlab.com/t/patches-with-full-index/29313
     gitlab_patch_url_re = (
@@ -440,24 +436,14 @@ def _check_patch_urls(pkgs, error_cls):
                 if not isinstance(patch, spack.patch.UrlPatch):
                     continue
 
-                if re.match(github_pull_commits_re, patch.url):
-                    url = re.sub(r"/pull/\d+/commits/", r"/commit/", patch.url)
-                    url = re.sub(r"^(.*)(?<!full_index=1)$", r"\1?full_index=1", url)
-                    errors.append(
-                        error_cls(
-                            f"patch URL in package {pkg_cls.name} "
-                            + "must not be a pull request commit; "
-                            + f"instead use {url}",
-                            [patch.url],
-                        )
-                    )
-                elif re.match(github_patch_url_re, patch.url):
+                if re.match(github_patch_url_re, patch.url):
                     full_index_arg = "?full_index=1"
                     if not patch.url.endswith(full_index_arg):
                         errors.append(
                             error_cls(
-                                f"patch URL in package {pkg_cls.name} "
-                                + f"must end with {full_index_arg}",
+                                "patch URL in package {0} must end with {1}".format(
+                                    pkg_cls.name, full_index_arg
+                                ),
                                 [patch.url],
                             )
                         )
@@ -465,7 +451,9 @@ def _check_patch_urls(pkgs, error_cls):
                     if not patch.url.endswith(".diff"):
                         errors.append(
                             error_cls(
-                                f"patch URL in package {pkg_cls.name} must end with .diff",
+                                "patch URL in package {0} must end with .diff".format(
+                                    pkg_cls.name
+                                ),
                                 [patch.url],
                             )
                         )

lib/spack/spack/binary_distribution.py

@@ -23,7 +23,6 @@
 import warnings
 from contextlib import closing
 from typing import Dict, Iterable, List, NamedTuple, Optional, Set, Tuple
-from urllib.error import HTTPError, URLError
 
 import llnl.util.filesystem as fsys
 import llnl.util.lang
@@ -899,9 +898,8 @@ def url_read_method(url):
         try:
             _, _, spec_file = web_util.read_from_url(url)
             contents = codecs.getreader("utf-8")(spec_file).read()
-        except (URLError, web_util.SpackWebError) as url_err:
-            tty.error("Error reading specfile: {0}".format(url))
-            tty.error(url_err)
+        except web_util.SpackWebError as e:
+            tty.error(f"Error reading specfile: {url}: {e}")
         return contents
 
     try:
@@ -2041,21 +2039,17 @@ def try_direct_fetch(spec, mirrors=None):
         try:
             _, _, fs = web_util.read_from_url(buildcache_fetch_url_signed_json)
             specfile_is_signed = True
-        except (URLError, web_util.SpackWebError, HTTPError) as url_err:
+        except web_util.SpackWebError as e1:
             try:
                 _, _, fs = web_util.read_from_url(buildcache_fetch_url_json)
-            except (URLError, web_util.SpackWebError, HTTPError) as url_err_x:
+            except web_util.SpackWebError as e2:
                 tty.debug(
-                    "Did not find {0} on {1}".format(
-                        specfile_name, buildcache_fetch_url_signed_json
-                    ),
-                    url_err,
+                    f"Did not find {specfile_name} on {buildcache_fetch_url_signed_json}",
+                    e1,
                     level=2,
                 )
                 tty.debug(
-                    "Did not find {0} on {1}".format(specfile_name, buildcache_fetch_url_json),
-                    url_err_x,
-                    level=2,
+                    f"Did not find {specfile_name} on {buildcache_fetch_url_json}", e2, level=2
                 )
                 continue
         specfile_contents = codecs.getreader("utf-8")(fs).read()
@@ -2140,6 +2134,9 @@ def get_keys(install=False, trust=False, force=False, mirrors=None):
 
     for mirror in mirror_collection.values():
         fetch_url = mirror.fetch_url
+        # TODO: oci:// does not support signing.
+        if fetch_url.startswith("oci://"):
+            continue
         keys_url = url_util.join(
             fetch_url, BUILD_CACHE_RELATIVE_PATH, BUILD_CACHE_KEYS_RELATIVE_PATH
         )
@@ -2150,19 +2147,12 @@ def get_keys(install=False, trust=False, force=False, mirrors=None):
         try:
             _, _, json_file = web_util.read_from_url(keys_index)
             json_index = sjson.load(codecs.getreader("utf-8")(json_file))
-        except (URLError, web_util.SpackWebError) as url_err:
+        except web_util.SpackWebError as url_err:
             if web_util.url_exists(keys_index):
-                err_msg = [
-                    "Unable to find public keys in {0},",
-                    " caught exception attempting to read from {1}.",
-                ]
-
                 tty.error(
-                    "".join(err_msg).format(
-                        url_util.format(fetch_url), url_util.format(keys_index)
-                    )
+                    f"Unable to find public keys in {url_util.format(fetch_url)},"
+                    f" caught exception attempting to read from {url_util.format(keys_index)}."
                 )
-
                 tty.debug(url_err)
 
             continue
@@ -2442,7 +2432,7 @@ def get_remote_hash(self):
         url_index_hash = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json.hash")
         try:
             response = self.urlopen(urllib.request.Request(url_index_hash, headers=self.headers))
-        except urllib.error.URLError:
+        except (TimeoutError, urllib.error.URLError):
             return None
 
         # Validate the hash
@@ -2464,7 +2454,7 @@ def conditional_fetch(self) -> FetchIndexResult:
 
         try:
             response = self.urlopen(urllib.request.Request(url_index, headers=self.headers))
-        except urllib.error.URLError as e:
+        except (TimeoutError, urllib.error.URLError) as e:
             raise FetchIndexError("Could not fetch index from {}".format(url_index), e) from e
 
         try:
@@ -2505,10 +2495,7 @@ def __init__(self, url, etag, urlopen=web_util.urlopen):
     def conditional_fetch(self) -> FetchIndexResult:
         # Just do a conditional fetch immediately
         url = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json")
-        headers = {
-            "User-Agent": web_util.SPACK_USER_AGENT,
-            "If-None-Match": '"{}"'.format(self.etag),
-        }
+        headers = {"User-Agent": web_util.SPACK_USER_AGENT, "If-None-Match": f'"{self.etag}"'}
 
         try:
             response = self.urlopen(urllib.request.Request(url, headers=headers))
@@ -2516,14 +2503,14 @@ def conditional_fetch(self) -> FetchIndexResult:
             if e.getcode() == 304:
                 # Not modified; that means fresh.
                 return FetchIndexResult(etag=None, hash=None, data=None, fresh=True)
-            raise FetchIndexError("Could not fetch index {}".format(url), e) from e
-        except urllib.error.URLError as e:
-            raise FetchIndexError("Could not fetch index {}".format(url), e) from e
+            raise FetchIndexError(f"Could not fetch index {url}", e) from e
+        except (TimeoutError, urllib.error.URLError) as e:
+            raise FetchIndexError(f"Could not fetch index {url}", e) from e
 
         try:
             result = codecs.getreader("utf-8")(response).read()
         except ValueError as e:
-            raise FetchIndexError("Remote index {} is invalid".format(url), e) from e
+            raise FetchIndexError(f"Remote index {url} is invalid", e) from e
 
         headers = response.headers
         etag_header_value = headers.get("Etag", None) or headers.get("etag", None)
@@ -2554,21 +2541,19 @@ def conditional_fetch(self) -> FetchIndexResult:
                     headers={"Accept": "application/vnd.oci.image.manifest.v1+json"},
                 )
             )
-        except urllib.error.URLError as e:
-            raise FetchIndexError(
-                "Could not fetch manifest from {}".format(url_manifest), e
-            ) from e
+        except (TimeoutError, urllib.error.URLError) as e:
+            raise FetchIndexError(f"Could not fetch manifest from {url_manifest}", e) from e
 
         try:
             manifest = json.loads(response.read())
         except Exception as e:
-            raise FetchIndexError("Remote index {} is invalid".format(url_manifest), e) from e
+            raise FetchIndexError(f"Remote index {url_manifest} is invalid", e) from e
 
         # Get first blob hash, which should be the index.json
         try:
             index_digest = spack.oci.image.Digest.from_string(manifest["layers"][0]["digest"])
         except Exception as e:
-            raise FetchIndexError("Remote index {} is invalid".format(url_manifest), e) from e
+            raise FetchIndexError(f"Remote index {url_manifest} is invalid", e) from e
 
         # Fresh?
         if index_digest.digest == self.local_hash:

lib/spack/spack/bootstrap/_common.py

@@ -213,18 +213,15 @@ def _root_spec(spec_str: str) -> str:
     Args:
         spec_str: spec to be bootstrapped. Must be without compiler and target.
     """
-    # Add a compiler and platform requirement to the root spec.
+    # Add a compiler requirement to the root spec.
     platform = str(spack.platforms.host())
-
     if platform == "darwin":
         spec_str += " %apple-clang"
-    elif platform == "windows":
-        spec_str += " %msvc"
     elif platform == "linux":
         spec_str += " %gcc"
     elif platform == "freebsd":
         spec_str += " %clang"
-    spec_str += f" platform={platform}"
+
     target = archspec.cpu.host().family
     spec_str += f" target={target}"
 

lib/spack/spack/bootstrap/config.py

@@ -129,10 +129,10 @@ def _bootstrap_config_scopes() -> Sequence["spack.config.ConfigScope"]:
     configuration_paths = (spack.config.CONFIGURATION_DEFAULTS_PATH, ("bootstrap", _config_path()))
     for name, path in configuration_paths:
         platform = spack.platforms.host().name
-        platform_scope = spack.config.DirectoryConfigScope(
-            f"{name}/{platform}", os.path.join(path, platform)
+        platform_scope = spack.config.ConfigScope(
+            "/".join([name, platform]), os.path.join(path, platform)
         )
-        generic_scope = spack.config.DirectoryConfigScope(name, path)
+        generic_scope = spack.config.ConfigScope(name, path)
         config_scopes.extend([generic_scope, platform_scope])
         msg = "[BOOTSTRAP CONFIG SCOPE] name={0}, path={1}"
         tty.debug(msg.format(generic_scope.name, generic_scope.path))

lib/spack/spack/bootstrap/core.py

@@ -597,7 +597,10 @@ def bootstrapping_sources(scope: Optional[str] = None):
         current = copy.copy(entry)
         metadata_dir = spack.util.path.canonicalize_path(entry["metadata"])
         metadata_yaml = os.path.join(metadata_dir, METADATA_YAML_FILENAME)
-        with open(metadata_yaml, encoding="utf-8") as stream:
-            current.update(spack.util.spack_yaml.load(stream))
-        list_of_sources.append(current)
+        try:
+            with open(metadata_yaml, encoding="utf-8") as stream:
+                current.update(spack.util.spack_yaml.load(stream))
+            list_of_sources.append(current)
+        except OSError:
+            pass
     return list_of_sources

lib/spack/spack/build_environment.py

@@ -43,7 +43,8 @@
 from collections import defaultdict
 from enum import Flag, auto
 from itertools import chain
-from typing import Dict, List, Set, Tuple
+from multiprocessing.connection import Connection
+from typing import Callable, Dict, List, Optional, Set, Tuple
 
 import llnl.util.tty as tty
 from llnl.string import plural
@@ -51,7 +52,6 @@
 from llnl.util.lang import dedupe, stable_partition
 from llnl.util.symlink import symlink
 from llnl.util.tty.color import cescape, colorize
-from llnl.util.tty.log import MultiProcessFd
 
 import spack.build_systems.cmake
 import spack.build_systems.meson
@@ -92,7 +92,7 @@
 )
 from spack.util.executable import Executable
 from spack.util.log_parse import make_log_context, parse_log_events
-from spack.util.module_cmd import load_module, path_from_modules
+from spack.util.module_cmd import load_module, module, path_from_modules
 
 #
 # This can be set by the user to globally disable parallel builds.
@@ -191,6 +191,14 @@ def __call__(self, *args, **kwargs):
         return super().__call__(*args, **kwargs)
 
 
+def _on_cray():
+    host_platform = spack.platforms.host()
+    host_os = host_platform.operating_system("default_os")
+    on_cray = str(host_platform) == "cray"
+    using_cnl = re.match(r"cnl\d+", str(host_os))
+    return on_cray, using_cnl
+
+
 def clean_environment():
     # Stuff in here sanitizes the build environment to eliminate
     # anything the user has set that may interfere. We apply it immediately
@@ -234,6 +242,17 @@ def clean_environment():
         if varname.endswith("_ROOT") and varname != "SPACK_ROOT":
             env.unset(varname)
 
+    # On Cray "cluster" systems, unset CRAY_LD_LIBRARY_PATH to avoid
+    # interference with Spack dependencies.
+    # CNL requires these variables to be set (or at least some of them,
+    # depending on the CNL version).
+    on_cray, using_cnl = _on_cray()
+    if on_cray and not using_cnl:
+        env.unset("CRAY_LD_LIBRARY_PATH")
+        for varname in os.environ.keys():
+            if "PKGCONF" in varname:
+                env.unset(varname)
+
     # Unset the following variables because they can affect installation of
     # Autotools and CMake packages.
     build_system_vars = [
@@ -363,7 +382,11 @@ def set_compiler_environment_variables(pkg, env):
     _add_werror_handling(keep_werror, env)
 
     # Set the target parameters that the compiler will add
-    isa_arg = spec.architecture.target.optimization_flags(compiler)
+    # Don't set on cray platform because the targeting module handles this
+    if spec.satisfies("platform=cray"):
+        isa_arg = ""
+    else:
+        isa_arg = spec.architecture.target.optimization_flags(compiler)
     env.set("SPACK_TARGET_ARGS", isa_arg)
 
     # Trap spack-tracked compiler flags as appropriate.
@@ -457,9 +480,12 @@ def set_wrapper_variables(pkg, env):
     env.set(SPACK_DEBUG_LOG_ID, pkg.spec.format("{name}-{hash:7}"))
     env.set(SPACK_DEBUG_LOG_DIR, spack.main.spack_working_dir)
 
-    # Find ccache binary and hand it to build environment
     if spack.config.get("config:ccache"):
+        # Enable ccache in the compiler wrapper
         env.set(SPACK_CCACHE_BINARY, spack.util.executable.which_string("ccache", required=True))
+    else:
+        # Avoid cache pollution if a build system forces `ccache <compiler wrapper invocation>`.
+        env.set("CCACHE_DISABLE", "1")
 
     # Gather information about various types of dependencies
     link_deps = set(pkg.spec.traverse(root=False, deptype=("link")))
@@ -738,9 +764,7 @@ def get_rpaths(pkg):
     # Second module is our compiler mod name. We use that to get rpaths from
     # module show output.
     if pkg.compiler.modules and len(pkg.compiler.modules) > 1:
-        mod_rpath = path_from_modules([pkg.compiler.modules[1]])
-        if mod_rpath:
-            rpaths.append(mod_rpath)
+        rpaths.append(path_from_modules([pkg.compiler.modules[1]]))
     return list(dedupe(filter_system_paths(rpaths)))
 
 
@@ -810,6 +834,14 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD):
         for mod in pkg.compiler.modules:
             load_module(mod)
 
+    # kludge to handle cray mpich and libsci being automatically loaded by
+    # PrgEnv modules on cray platform. Module unload does no damage when
+    # unnecessary
+    on_cray, _ = _on_cray()
+    if on_cray and not dirty:
+        for mod in ["cray-mpich", "cray-libsci"]:
+            module("unload", mod)
+
     if target and target.module_name:
         load_module(target.module_name)
 
@@ -1113,18 +1145,60 @@ def get_cmake_prefix_path(pkg):
 
 
 def _setup_pkg_and_run(
-    serialized_pkg, function, kwargs, write_pipe, input_multiprocess_fd, jsfd1, jsfd2
+    serialized_pkg: "spack.subprocess_context.PackageInstallContext",
+    function: Callable,
+    kwargs: Dict,
+    write_pipe: Connection,
+    input_pipe: Optional[Connection],
+    jsfd1: Optional[Connection],
+    jsfd2: Optional[Connection],
 ):
+    """Main entry point in the child process for Spack builds.
+
+    ``_setup_pkg_and_run`` is called by the child process created in
+    ``start_build_process()``, and its main job is to run ``function()`` on behalf of
+    some Spack installation (see :ref:`spack.installer.PackageInstaller._install_task`).
+
+    The child process is passed a ``write_pipe``, on which it's expected to send one of
+    the following:
+
+    * ``StopPhase``: error raised by a build process indicating it's stopping at a
+      particular build phase.
+
+    * ``BaseException``: any exception raised by a child build process, which will be
+      wrapped in ``ChildError`` (which adds a bunch of debug info and log context) and
+      raised in the parent.
+
+    * The return value of ``function()``, which can be anything (except an exception).
+      This is returned to the caller.
+
+    Note: ``jsfd1`` and ``jsfd2`` are passed solely to ensure that the child process
+    does not close these file descriptors. Some ``multiprocessing`` backends will close
+    them automatically in the child if they are not passed at process creation time.
+
+    Arguments:
+        serialized_pkg: Spack package install context object (serialized form of the
+            package that we'll build in the child process).
+        function: function to call in the child process; serialized_pkg is passed to
+            this as the first argument.
+        kwargs: additional keyword arguments to pass to ``function()``.
+        write_pipe: multiprocessing ``Connection`` to the parent process, to which the
+            child *must* send a result (or an error) back to parent on.
+        input_multiprocess_fd: stdin from the parent (not passed currently on Windows)
+        jsfd1: gmake Jobserver file descriptor 1.
+        jsfd2: gmake Jobserver file descriptor 2.
+
+    """
+
     context: str = kwargs.get("context", "build")
 
     try:
-        # We are in the child process. Python sets sys.stdin to
-        # open(os.devnull) to prevent our process and its parent from
-        # simultaneously reading from the original stdin. But, we assume
-        # that the parent process is not going to read from it till we
-        # are done with the child, so we undo Python's precaution.
-        if input_multiprocess_fd is not None:
-            sys.stdin = os.fdopen(input_multiprocess_fd.fd)
+        # We are in the child process. Python sets sys.stdin to open(os.devnull) to prevent our
+        # process and its parent from simultaneously reading from the original stdin. But, we
+        # assume that the parent process is not going to read from it till we are done with the
+        # child, so we undo Python's precaution. closefd=False since Connection has ownership.
+        if input_pipe is not None:
+            sys.stdin = os.fdopen(input_pipe.fileno(), closefd=False)
 
         pkg = serialized_pkg.restore()
 
@@ -1140,13 +1214,14 @@ def _setup_pkg_and_run(
         # Do not create a full ChildError from this, it's not an error
         # it's a control statement.
         write_pipe.send(e)
-    except BaseException:
+    except BaseException as e:
         # catch ANYTHING that goes wrong in the child process
-        exc_type, exc, tb = sys.exc_info()
 
         # Need to unwind the traceback in the child because traceback
         # objects can't be sent to the parent.
-        tb_string = traceback.format_exc()
+        exc_type = type(e)
+        tb = e.__traceback__
+        tb_string = "".join(traceback.format_exception(exc_type, e, tb))
 
         # build up some context from the offending package so we can
         # show that, too.
@@ -1163,8 +1238,8 @@ def _setup_pkg_and_run(
         elif context == "test":
             logfile = os.path.join(pkg.test_suite.stage, pkg.test_suite.test_log_name(pkg.spec))
 
-        error_msg = str(exc)
-        if isinstance(exc, (spack.multimethod.NoSuchMethodError, AttributeError)):
+        error_msg = str(e)
+        if isinstance(e, (spack.multimethod.NoSuchMethodError, AttributeError)):
             process = "test the installation" if context == "test" else "build from sources"
             error_msg = (
                 "The '{}' package cannot find an attribute while trying to {}. "
@@ -1174,7 +1249,7 @@ def _setup_pkg_and_run(
                 "More information at https://spack.readthedocs.io/en/latest/packaging_guide.html#installation-procedure"
             ).format(pkg.name, process, context)
             error_msg = colorize("@*R{{{}}}".format(error_msg))
-            error_msg = "{}\n\n{}".format(str(exc), error_msg)
+            error_msg = "{}\n\n{}".format(str(e), error_msg)
 
         # make a pickleable exception to send to parent.
         msg = "%s: %s" % (exc_type.__name__, error_msg)
@@ -1192,8 +1267,8 @@ def _setup_pkg_and_run(
 
     finally:
         write_pipe.close()
-        if input_multiprocess_fd is not None:
-            input_multiprocess_fd.close()
+        if input_pipe is not None:
+            input_pipe.close()
 
 
 def start_build_process(pkg, function, kwargs):
@@ -1220,23 +1295,9 @@ def child_fun():
     If something goes wrong, the child process catches the error and
     passes it to the parent wrapped in a ChildError.  The parent is
     expected to handle (or re-raise) the ChildError.
-
-    This uses `multiprocessing.Process` to create the child process. The
-    mechanism used to create the process differs on different operating
-    systems and for different versions of Python. In some cases "fork"
-    is used (i.e. the "fork" system call) and some cases it starts an
-    entirely new Python interpreter process (in the docs this is referred
-    to as the "spawn" start method). Breaking it down by OS:
-
-    - Linux always uses fork.
-    - Mac OS uses fork before Python 3.8 and "spawn" for 3.8 and after.
-    - Windows always uses the "spawn" start method.
-
-    For more information on `multiprocessing` child process creation
-    mechanisms, see https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
     """
     read_pipe, write_pipe = multiprocessing.Pipe(duplex=False)
-    input_multiprocess_fd = None
+    input_fd = None
     jobserver_fd1 = None
     jobserver_fd2 = None
 
@@ -1245,14 +1306,13 @@ def child_fun():
     try:
         # Forward sys.stdin when appropriate, to allow toggling verbosity
         if sys.platform != "win32" and sys.stdin.isatty() and hasattr(sys.stdin, "fileno"):
-            input_fd = os.dup(sys.stdin.fileno())
-            input_multiprocess_fd = MultiProcessFd(input_fd)
+            input_fd = Connection(os.dup(sys.stdin.fileno()))
         mflags = os.environ.get("MAKEFLAGS", False)
         if mflags:
             m = re.search(r"--jobserver-[^=]*=(\d),(\d)", mflags)
             if m:
-                jobserver_fd1 = MultiProcessFd(int(m.group(1)))
-                jobserver_fd2 = MultiProcessFd(int(m.group(2)))
+                jobserver_fd1 = Connection(int(m.group(1)))
+                jobserver_fd2 = Connection(int(m.group(2)))
 
         p = multiprocessing.Process(
             target=_setup_pkg_and_run,
@@ -1261,7 +1321,7 @@ def child_fun():
                 function,
                 kwargs,
                 write_pipe,
-                input_multiprocess_fd,
+                input_fd,
                 jobserver_fd1,
                 jobserver_fd2,
             ),
@@ -1281,8 +1341,8 @@ def child_fun():
 
     finally:
         # Close the input stream in the parent process
-        if input_multiprocess_fd is not None:
-            input_multiprocess_fd.close()
+        if input_fd is not None:
+            input_fd.close()
 
     def exitcode_msg(p):
         typ = "exit" if p.exitcode >= 0 else "signal"
@@ -1473,7 +1533,7 @@ def long_message(self):
             out.write("  {0}\n".format(self.log_name))
 
         # Also output the test log path IF it exists
-        if self.context != "test" and have_log:
+        if self.context != "test":
             test_log = join_path(os.path.dirname(self.log_name), spack_install_test_log)
             if os.path.isfile(test_log):
                 out.write("\nSee test log for details:\n")

lib/spack/spack/build_systems/cached_cmake.py

@@ -162,9 +162,7 @@ def initconfig_compiler_entries(self):
             ld_flags = " ".join(flags["ldflags"])
             ld_format_string = "CMAKE_{0}_LINKER_FLAGS"
             # CMake has separate linker arguments for types of builds.
-            # 'ldflags' should not be used with CMAKE_STATIC_LINKER_FLAGS which
-            # is used by the archiver, so don't include "STATIC" in this loop:
-            for ld_type in ["EXE", "MODULE", "SHARED"]:
+            for ld_type in ["EXE", "MODULE", "SHARED", "STATIC"]:
                 ld_string = ld_format_string.format(ld_type)
                 entries.append(cmake_cache_string(ld_string, ld_flags))
 

lib/spack/spack/build_systems/cuda.py

@@ -110,8 +110,9 @@ def cuda_flags(arch_list):
     # From the NVIDIA install guide we know of conflicts for particular
     # platforms (linux, darwin), architectures (x86, powerpc) and compilers
     # (gcc, clang). We don't restrict %gcc and %clang conflicts to
-    # platform=linux, since they may apply to platform=darwin. We currently
-    # do not provide conflicts for platform=darwin with %apple-clang.
+    # platform=linux, since they should also apply to platform=cray, and may
+    # apply to platform=darwin. We currently do not provide conflicts for
+    # platform=darwin with %apple-clang.
 
     # Linux x86_64 compiler conflicts from here:
     # https://gist.github.com/ax3l/9489132
@@ -124,8 +125,6 @@ def cuda_flags(arch_list):
         # minimum supported versions
         conflicts("%gcc@:4", when="+cuda ^cuda@11.0:")
         conflicts("%gcc@:5", when="+cuda ^cuda@11.4:")
-        conflicts("%gcc@:7.2", when="+cuda ^cuda@12.4:")
-        conflicts("%clang@:6", when="+cuda ^cuda@12.2:")
 
         # maximum supported version
         # NOTE:
@@ -138,14 +137,11 @@ def cuda_flags(arch_list):
         conflicts("%gcc@11.2:", when="+cuda ^cuda@:11.5")
         conflicts("%gcc@12:", when="+cuda ^cuda@:11.8")
         conflicts("%gcc@13:", when="+cuda ^cuda@:12.3")
-        conflicts("%gcc@14:", when="+cuda ^cuda@:12.5")
         conflicts("%clang@12:", when="+cuda ^cuda@:11.4.0")
         conflicts("%clang@13:", when="+cuda ^cuda@:11.5")
         conflicts("%clang@14:", when="+cuda ^cuda@:11.7")
         conflicts("%clang@15:", when="+cuda ^cuda@:12.0")
-        conflicts("%clang@16:", when="+cuda ^cuda@:12.1")
-        conflicts("%clang@17:", when="+cuda ^cuda@:12.3")
-        conflicts("%clang@18:", when="+cuda ^cuda@:12.5")
+        conflicts("%clang@16:", when="+cuda ^cuda@:12.3")
 
         # https://gist.github.com/ax3l/9489132#gistcomment-3860114
         conflicts("%gcc@10", when="+cuda ^cuda@:11.4.0")
@@ -213,16 +209,12 @@ def cuda_flags(arch_list):
         conflicts("%intel@19.0:", when="+cuda ^cuda@:10.0")
         conflicts("%intel@19.1:", when="+cuda ^cuda@:10.1")
         conflicts("%intel@19.2:", when="+cuda ^cuda@:11.1.0")
-        conflicts("%intel@2021:", when="+cuda ^cuda@:11.4.0")
 
         # XL is mostly relevant for ppc64le Linux
         conflicts("%xl@:12,14:", when="+cuda ^cuda@:9.1")
         conflicts("%xl@:12,14:15,17:", when="+cuda ^cuda@9.2")
         conflicts("%xl@:12,17:", when="+cuda ^cuda@:11.1.0")
 
-        # PowerPC.
-        conflicts("target=ppc64le", when="+cuda ^cuda@12.5:")
-
         # Darwin.
         # TODO: add missing conflicts for %apple-clang cuda@:10
-        conflicts("platform=darwin", when="+cuda ^cuda@11.0.2:")
+        conflicts("platform=darwin", when="+cuda ^cuda@11.0.2: ")

lib/spack/spack/build_systems/intel.py

@@ -846,7 +846,6 @@ def scalapack_libs(self):
             "^mpich@2:" in spec_root
             or "^cray-mpich" in spec_root
             or "^mvapich2" in spec_root
-            or "^mvapich" in spec_root
             or "^intel-mpi" in spec_root
             or "^intel-oneapi-mpi" in spec_root
             or "^intel-parallel-studio" in spec_root
@@ -937,15 +936,32 @@ def mpi_setup_dependent_build_environment(self, env, dependent_spec, compilers_o
             "I_MPI_ROOT": self.normalize_path("mpi"),
         }
 
-        compiler_wrapper_commands = self.mpi_compiler_wrappers
-        wrapper_vars.update(
-            {
-                "MPICC": compiler_wrapper_commands["MPICC"],
-                "MPICXX": compiler_wrapper_commands["MPICXX"],
-                "MPIF77": compiler_wrapper_commands["MPIF77"],
-                "MPIF90": compiler_wrapper_commands["MPIF90"],
-            }
-        )
+        # CAUTION - SIMILAR code in:
+        #   var/spack/repos/builtin/packages/mpich/package.py
+        #   var/spack/repos/builtin/packages/openmpi/package.py
+        #   var/spack/repos/builtin/packages/mvapich2/package.py
+        #
+        # On Cray, the regular compiler wrappers *are* the MPI wrappers.
+        if "platform=cray" in self.spec:
+            # TODO: Confirm
+            wrapper_vars.update(
+                {
+                    "MPICC": compilers_of_client["CC"],
+                    "MPICXX": compilers_of_client["CXX"],
+                    "MPIF77": compilers_of_client["F77"],
+                    "MPIF90": compilers_of_client["F90"],
+                }
+            )
+        else:
+            compiler_wrapper_commands = self.mpi_compiler_wrappers
+            wrapper_vars.update(
+                {
+                    "MPICC": compiler_wrapper_commands["MPICC"],
+                    "MPICXX": compiler_wrapper_commands["MPICXX"],
+                    "MPIF77": compiler_wrapper_commands["MPIF77"],
+                    "MPIF90": compiler_wrapper_commands["MPIF90"],
+                }
+            )
 
         # Ensure that the directory containing the compiler wrappers is in the
         # PATH. Spack packages add `prefix.bin` to their dependents' paths,

lib/spack/spack/build_systems/msbuild.py

@@ -24,6 +24,7 @@ class MSBuildPackage(spack.package_base.PackageBase):
     build_system("msbuild")
     conflicts("platform=linux", when="build_system=msbuild")
     conflicts("platform=darwin", when="build_system=msbuild")
+    conflicts("platform=cray", when="build_system=msbuild")
 
 
 @spack.builder.builder("msbuild")

lib/spack/spack/build_systems/nmake.py

@@ -24,6 +24,7 @@ class NMakePackage(spack.package_base.PackageBase):
     build_system("nmake")
     conflicts("platform=linux", when="build_system=nmake")
     conflicts("platform=darwin", when="build_system=nmake")
+    conflicts("platform=cray", when="build_system=nmake")
 
 
 @spack.builder.builder("nmake")

lib/spack/spack/build_systems/oneapi.py

@@ -36,8 +36,9 @@ class IntelOneApiPackage(Package):
         "target=ppc64:",
         "target=ppc64le:",
         "target=aarch64:",
-        "platform=darwin",
-        "platform=windows",
+        "platform=darwin:",
+        "platform=cray:",
+        "platform=windows:",
     ]:
         conflicts(c, msg="This package in only available for x86_64 and Linux")
 

lib/spack/spack/caches.py

@@ -34,8 +34,6 @@ def _misc_cache():
     return spack.util.file_cache.FileCache(path)
 
 
-FileCacheType = Union[spack.util.file_cache.FileCache, llnl.util.lang.Singleton]
-
 #: Spack's cache for small data
 MISC_CACHE: Union[spack.util.file_cache.FileCache, llnl.util.lang.Singleton] = (
     llnl.util.lang.Singleton(_misc_cache)

lib/spack/spack/ci.py

@@ -553,9 +553,10 @@ def generate_gitlab_ci_yaml(
     env,
     print_summary,
     output_file,
-    *,
     prune_dag=False,
     check_index_only=False,
+    run_optimizer=False,
+    use_dependencies=False,
     artifacts_root=None,
     remote_mirror_override=None,
 ):
@@ -576,6 +577,12 @@ def generate_gitlab_ci_yaml(
             this mode results in faster yaml generation time). Otherwise, also
             check each spec directly by url (useful if there is no index or it
             might be out of date).
+        run_optimizer (bool): If True, post-process the generated yaml to try
+            try to reduce the size (attempts to collect repeated configuration
+            and replace with definitions).)
+        use_dependencies (bool): If true, use "dependencies" rather than "needs"
+            ("needs" allows DAG scheduling).  Useful if gitlab instance cannot
+            be configured to handle more than a few "needs" per job.
         artifacts_root (str): Path where artifacts like logs, environment
             files (spack.yaml, spack.lock), etc should be written.  GitLab
             requires this to be within the project directory.
@@ -809,8 +816,7 @@ def ensure_expected_target_path(path):
         cli_scopes = [
             os.path.relpath(s.path, concrete_env_dir)
             for s in cfg.scopes().values()
-            if not s.writable
-            and isinstance(s, (cfg.DirectoryConfigScope))
+            if isinstance(s, cfg.ImmutableConfigScope)
             and s.path not in env_includes
             and os.path.exists(s.path)
         ]
@@ -1107,7 +1113,7 @@ def main_script_replacements(cmd):
     if cdash_handler and cdash_handler.auth_token:
         try:
             cdash_handler.populate_buildgroup(all_job_names)
-        except (SpackError, HTTPError, URLError) as err:
+        except (SpackError, HTTPError, URLError, TimeoutError) as err:
             tty.warn(f"Problem populating buildgroup: {err}")
     else:
         tty.warn("Unable to populate buildgroup without CDash credentials")
@@ -1267,6 +1273,17 @@ def main_script_replacements(cmd):
             with open(copy_specs_file, "w") as fd:
                 fd.write(json.dumps(buildcache_copies))
 
+        # TODO(opadron): remove this or refactor
+        if run_optimizer:
+            import spack.ci_optimization as ci_opt
+
+            output_object = ci_opt.optimizer(output_object)
+
+        # TODO(opadron): remove this or refactor
+        if use_dependencies:
+            import spack.ci_needs_workaround as cinw
+
+            output_object = cinw.needs_to_dependencies(output_object)
     else:
         # No jobs were generated
         noop_job = spack_ci_ir["jobs"]["noop"]["attributes"]
@@ -2083,7 +2100,7 @@ def read_broken_spec(broken_spec_url):
     """
     try:
         _, _, fs = web_util.read_from_url(broken_spec_url)
-    except (URLError, web_util.SpackWebError, HTTPError):
+    except web_util.SpackWebError:
         tty.warn(f"Unable to read broken spec from {broken_spec_url}")
         return None
 

lib/spack/spack/ci_needs_workaround.py

@@ -0,0 +1,34 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import collections.abc
+
+get_job_name = lambda needs_entry: (
+    needs_entry.get("job")
+    if (isinstance(needs_entry, collections.abc.Mapping) and needs_entry.get("artifacts", True))
+    else needs_entry if isinstance(needs_entry, str) else None
+)
+
+
+def convert_job(job_entry):
+    if not isinstance(job_entry, collections.abc.Mapping):
+        return job_entry
+
+    needs = job_entry.get("needs")
+    if needs is None:
+        return job_entry
+
+    new_job = {}
+    new_job.update(job_entry)
+    del new_job["needs"]
+
+    new_job["dependencies"] = list(
+        filter((lambda x: x is not None), (get_job_name(needs_entry) for needs_entry in needs))
+    )
+
+    return new_job
+
+
+def needs_to_dependencies(yaml):
+    return dict((k, convert_job(v)) for k, v in yaml.items())

lib/spack/spack/ci_optimization.py

@@ -0,0 +1,363 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import collections
+import collections.abc
+import copy
+import hashlib
+
+import spack.util.spack_yaml as syaml
+
+
+def sort_yaml_obj(obj):
+    if isinstance(obj, collections.abc.Mapping):
+        return syaml.syaml_dict(
+            (k, sort_yaml_obj(v)) for k, v in sorted(obj.items(), key=(lambda item: str(item[0])))
+        )
+
+    if isinstance(obj, collections.abc.Sequence) and not isinstance(obj, str):
+        return syaml.syaml_list(sort_yaml_obj(x) for x in obj)
+
+    return obj
+
+
+def matches(obj, proto):
+    """Returns True if the test object "obj" matches the prototype object
+    "proto".
+
+    If obj and proto are mappings, obj matches proto if (key in obj) and
+    (obj[key] matches proto[key]) for every key in proto.
+
+    If obj and proto are sequences, obj matches proto if they are of the same
+    length and (a matches b) for every (a,b) in zip(obj, proto).
+
+    Otherwise, obj matches proto if obj == proto.
+
+    Precondition: proto must not have any reference cycles
+    """
+    if isinstance(obj, collections.abc.Mapping):
+        if not isinstance(proto, collections.abc.Mapping):
+            return False
+
+        return all((key in obj and matches(obj[key], val)) for key, val in proto.items())
+
+    if isinstance(obj, collections.abc.Sequence) and not isinstance(obj, str):
+        if not (isinstance(proto, collections.abc.Sequence) and not isinstance(proto, str)):
+            return False
+
+        if len(obj) != len(proto):
+            return False
+
+        return all(matches(obj[index], val) for index, val in enumerate(proto))
+
+    return obj == proto
+
+
+def subkeys(obj, proto):
+    """Returns the test mapping "obj" after factoring out the items it has in
+    common with the prototype mapping "proto".
+
+    Consider a recursive merge operation, merge(a, b) on mappings a and b, that
+    returns a mapping, m, whose keys are the union of the keys of a and b, and
+    for every such key, "k", its corresponding value is:
+
+      - merge(a[key], b[key])  if a[key] and b[key] are mappings, or
+      - b[key]                 if (key in b) and not matches(a[key], b[key]),
+                               or
+      - a[key]                 otherwise
+
+
+    If obj and proto are mappings, the returned object is the smallest object,
+    "a", such that merge(a, proto) matches obj.
+
+    Otherwise, obj is returned.
+    """
+    if not (
+        isinstance(obj, collections.abc.Mapping) and isinstance(proto, collections.abc.Mapping)
+    ):
+        return obj
+
+    new_obj = {}
+    for key, value in obj.items():
+        if key not in proto:
+            new_obj[key] = value
+            continue
+
+        if matches(value, proto[key]) and matches(proto[key], value):
+            continue
+
+        if isinstance(value, collections.abc.Mapping):
+            new_obj[key] = subkeys(value, proto[key])
+            continue
+
+        new_obj[key] = value
+
+    return new_obj
+
+
+def add_extends(yaml, key):
+    """Modifies the given object "yaml" so that it includes an "extends" key
+    whose value features "key".
+
+    If "extends" is not in yaml, then yaml is modified such that
+    yaml["extends"] == key.
+
+    If yaml["extends"] is a str, then yaml is modified such that
+    yaml["extends"] == [yaml["extends"], key]
+
+    If yaml["extends"] is a list that does not include key, then key is
+    appended to the list.
+
+    Otherwise, yaml is left unchanged.
+    """
+
+    has_key = "extends" in yaml
+    extends = yaml.get("extends")
+
+    if has_key and not isinstance(extends, (str, collections.abc.Sequence)):
+        return
+
+    if extends is None:
+        yaml["extends"] = key
+        return
+
+    if isinstance(extends, str):
+        if extends != key:
+            yaml["extends"] = [extends, key]
+        return
+
+    if key not in extends:
+        extends.append(key)
+
+
+def common_subobject(yaml, sub):
+    """Factor prototype object "sub" out of the values of mapping "yaml".
+
+    Consider a modified copy of yaml, "new", where for each key, "key" in yaml:
+
+      - If yaml[key] matches sub, then new[key] = subkeys(yaml[key], sub).
+      - Otherwise, new[key] = yaml[key].
+
+    If the above match criteria is not satisfied for any such key, then (yaml,
+    None) is returned. The yaml object is returned unchanged.
+
+    Otherwise, each matching value in new is modified as in
+    add_extends(new[key], common_key), and then new[common_key] is set to sub.
+    The common_key value is chosen such that it does not match any preexisting
+    key in new. In this case, (new, common_key) is returned.
+    """
+    match_list = set(k for k, v in yaml.items() if matches(v, sub))
+
+    if not match_list:
+        return yaml, None
+
+    common_prefix = ".c"
+    common_index = 0
+
+    while True:
+        common_key = "".join((common_prefix, str(common_index)))
+        if common_key not in yaml:
+            break
+        common_index += 1
+
+    new_yaml = {}
+
+    for key, val in yaml.items():
+        new_yaml[key] = copy.deepcopy(val)
+
+        if not matches(val, sub):
+            continue
+
+        new_yaml[key] = subkeys(new_yaml[key], sub)
+        add_extends(new_yaml[key], common_key)
+
+    new_yaml[common_key] = sub
+
+    return new_yaml, common_key
+
+
+def print_delta(name, old, new, applied=None):
+    delta = new - old
+    reldelta = (1000 * delta) // old
+    reldelta = (reldelta // 10, reldelta % 10)
+
+    if applied is None:
+        applied = new <= old
+
+    print(
+        "\n".join(
+            (
+                "{0} {1}:",
+                "  before: {2: 10d}",
+                "  after : {3: 10d}",
+                "  delta : {4:+10d} ({5:=+3d}.{6}%)",
+            )
+        ).format(name, ("+" if applied else "x"), old, new, delta, reldelta[0], reldelta[1])
+    )
+
+
+def try_optimization_pass(name, yaml, optimization_pass, *args, **kwargs):
+    """Try applying an optimization pass and return information about the
+    result
+
+    "name" is a string describing the nature of the pass. If it is a non-empty
+    string, summary statistics are also printed to stdout.
+
+    "yaml" is the object to apply the pass to.
+
+    "optimization_pass" is the function implementing the pass to be applied.
+
+    "args" and "kwargs" are the additional arguments to pass to optimization
+    pass. The pass is applied as
+
+    >>> (new_yaml, *other_results) = optimization_pass(yaml, *args, **kwargs)
+
+    The pass's results are greedily rejected if it does not modify the original
+    yaml document, or if it produces a yaml document that serializes to a
+    larger string.
+
+    Returns (new_yaml, yaml, applied, other_results) if applied, or
+    (yaml, new_yaml, applied, other_results) otherwise.
+    """
+    result = optimization_pass(yaml, *args, **kwargs)
+    new_yaml, other_results = result[0], result[1:]
+
+    if new_yaml is yaml:
+        # pass was not applied
+        return (yaml, new_yaml, False, other_results)
+
+    pre_size = len(syaml.dump_config(sort_yaml_obj(yaml), default_flow_style=True))
+    post_size = len(syaml.dump_config(sort_yaml_obj(new_yaml), default_flow_style=True))
+
+    # pass makes the size worse: not applying
+    applied = post_size <= pre_size
+    if applied:
+        yaml, new_yaml = new_yaml, yaml
+
+    if name:
+        print_delta(name, pre_size, post_size, applied)
+
+    return (yaml, new_yaml, applied, other_results)
+
+
+def build_histogram(iterator, key):
+    """Builds a histogram of values given an iterable of mappings and a key.
+
+    For each mapping "m" with key "key" in iterator, the value m[key] is
+    considered.
+
+    Returns a list of tuples (hash, count, proportion, value), where
+
+      - "hash" is a sha1sum hash of the value.
+      - "count" is the number of occurences of values that hash to "hash".
+      - "proportion" is the proportion of all values considered above that
+        hash to "hash".
+      - "value" is one of the values considered above that hash to "hash".
+        Which value is chosen when multiple values hash to the same "hash" is
+        undefined.
+
+    The list is sorted in descending order by count, yielding the most
+    frequently occuring hashes first.
+    """
+    buckets = collections.defaultdict(int)
+    values = {}
+
+    num_objects = 0
+    for obj in iterator:
+        num_objects += 1
+
+        try:
+            val = obj[key]
+        except (KeyError, TypeError):
+            continue
+
+        value_hash = hashlib.sha1()
+        value_hash.update(syaml.dump_config(sort_yaml_obj(val)).encode())
+        value_hash = value_hash.hexdigest()
+
+        buckets[value_hash] += 1
+        values[value_hash] = val
+
+    return [
+        (h, buckets[h], float(buckets[h]) / num_objects, values[h])
+        for h in sorted(buckets.keys(), key=lambda k: -buckets[k])
+    ]
+
+
+def optimizer(yaml):
+    original_size = len(syaml.dump_config(sort_yaml_obj(yaml), default_flow_style=True))
+
+    # try factoring out commonly repeated portions
+    common_job = {
+        "variables": {"SPACK_COMPILER_ACTION": "NONE"},
+        "after_script": ['rm -rf "./spack"'],
+        "artifacts": {"paths": ["jobs_scratch_dir", "cdash_report"], "when": "always"},
+    }
+
+    # look for a list of tags that appear frequently
+    _, count, proportion, tags = next(iter(build_histogram(yaml.values(), "tags")), (None,) * 4)
+
+    # If a list of tags is found, and there are more than one job that uses it,
+    # *and* the jobs that do use it represent at least 70% of all jobs, then
+    # add the list to the prototype object.
+    if tags and count > 1 and proportion >= 0.70:
+        common_job["tags"] = tags
+
+    # apply common object factorization
+    yaml, other, applied, rest = try_optimization_pass(
+        "general common object factorization", yaml, common_subobject, common_job
+    )
+
+    # look for a common script, and try factoring that out
+    _, count, proportion, script = next(
+        iter(build_histogram(yaml.values(), "script")), (None,) * 4
+    )
+
+    if script and count > 1 and proportion >= 0.70:
+        yaml, other, applied, rest = try_optimization_pass(
+            "script factorization", yaml, common_subobject, {"script": script}
+        )
+
+    # look for a common before_script, and try factoring that out
+    _, count, proportion, script = next(
+        iter(build_histogram(yaml.values(), "before_script")), (None,) * 4
+    )
+
+    if script and count > 1 and proportion >= 0.70:
+        yaml, other, applied, rest = try_optimization_pass(
+            "before_script factorization", yaml, common_subobject, {"before_script": script}
+        )
+
+    # Look specifically for the SPACK_ROOT_SPEC environment variables.
+    # Try to factor them out.
+    h = build_histogram(
+        (getattr(val, "get", lambda *args: {})("variables") for val in yaml.values()),
+        "SPACK_ROOT_SPEC",
+    )
+
+    # In this case, we try to factor out *all* instances of the SPACK_ROOT_SPEC
+    # environment variable; not just the one that appears with the greatest
+    # frequency. We only require that more than 1 job uses a given instance's
+    # value, because we expect the value to be very large, and so expect even
+    # few-to-one factorizations to yield large space savings.
+    counter = 0
+    for _, count, proportion, spec in h:
+        if count <= 1:
+            continue
+
+        counter += 1
+
+        yaml, other, applied, rest = try_optimization_pass(
+            "SPACK_ROOT_SPEC factorization ({count})".format(count=counter),
+            yaml,
+            common_subobject,
+            {"variables": {"SPACK_ROOT_SPEC": spec}},
+        )
+
+    new_size = len(syaml.dump_config(sort_yaml_obj(yaml), default_flow_style=True))
+
+    print("\n")
+    print_delta("overall summary", original_size, new_size)
+    print("\n")
+    return yaml

lib/spack/spack/cmd/__init__.py

@@ -336,7 +336,6 @@ def display_specs(specs, args=None, **kwargs):
         groups (bool): display specs grouped by arch/compiler (default True)
         decorator (typing.Callable): function to call to decorate specs
         all_headers (bool): show headers even when arch/compiler aren't defined
-        status_fn (typing.Callable): if provided, prepend install-status info
         output (typing.IO): A file object to write to. Default is ``sys.stdout``
 
     """
@@ -360,7 +359,6 @@ def get_arg(name, default=None):
     groups = get_arg("groups", True)
     all_headers = get_arg("all_headers", False)
     output = get_arg("output", sys.stdout)
-    status_fn = get_arg("status_fn", None)
 
     decorator = get_arg("decorator", None)
     if decorator is None:
@@ -388,13 +386,6 @@ def get_arg(name, default=None):
     def fmt(s, depth=0):
         """Formatter function for all output specs"""
         string = ""
-
-        if status_fn:
-            # This was copied from spec.tree's colorization logic
-            # then shortened because it seems like status_fn should
-            # always return an InstallStatus
-            string += colorize(status_fn(s).value)
-
         if hashes:
             string += gray_hash(s, hlen) + " "
         string += depth * "    "
@@ -453,7 +444,7 @@ def format_list(specs):
 def filter_loaded_specs(specs):
     """Filter a list of specs returning only those that are
     currently loaded."""
-    hashes = os.environ.get(uenv.spack_loaded_hashes_var, "").split(os.pathsep)
+    hashes = os.environ.get(uenv.spack_loaded_hashes_var, "").split(":")
     return [x for x in specs if x.dag_hash() in hashes]
 
 

lib/spack/spack/cmd/bootstrap.py

@@ -165,7 +165,7 @@ def _reset(args):
         if not ok_to_continue:
             raise RuntimeError("Aborting")
 
-    for scope in spack.config.CONFIG.writable_scopes:
+    for scope in spack.config.CONFIG.file_scopes:
         # The default scope should stay untouched
         if scope.name == "defaults":
             continue

lib/spack/spack/cmd/buildcache.py

@@ -70,6 +70,12 @@ def setup_parser(subparser: argparse.ArgumentParser):
 
     push = subparsers.add_parser("push", aliases=["create"], help=push_fn.__doc__)
     push.add_argument("-f", "--force", action="store_true", help="overwrite tarball if it exists")
+    push.add_argument(
+        "--allow-root",
+        "-a",
+        action="store_true",
+        help="allow install root string in binary files after RPATH substitution",
+    )
     push_sign = push.add_mutually_exclusive_group(required=False)
     push_sign.add_argument(
         "--unsigned",
@@ -184,6 +190,10 @@ def setup_parser(subparser: argparse.ArgumentParser):
     keys.add_argument("-f", "--force", action="store_true", help="force new download of keys")
     keys.set_defaults(func=keys_fn)
 
+    preview = subparsers.add_parser("preview", help=preview_fn.__doc__)
+    arguments.add_common_arguments(preview, ["installed_specs"])
+    preview.set_defaults(func=preview_fn)
+
     # Check if binaries need to be rebuilt on remote mirror
     check = subparsers.add_parser("check", help=check_fn.__doc__)
     check.add_argument(
@@ -394,6 +404,11 @@ def push_fn(args):
     else:
         roots = spack.cmd.require_active_env(cmd_name="buildcache push").concrete_roots()
 
+    if args.allow_root:
+        tty.warn(
+            "The flag `--allow-root` is the default in Spack 0.21, will be removed in Spack 0.22"
+        )
+
     mirror: spack.mirror.Mirror = args.mirror
 
     # Check if this is an OCI image.
@@ -798,7 +813,7 @@ def _push_oci(
 
     def extra_config(spec: Spec):
         spec_dict = spec.to_dict(hash=ht.dag_hash)
-        spec_dict["buildcache_layout_version"] = 1
+        spec_dict["buildcache_layout_version"] = bindist.CURRENT_BUILD_CACHE_LAYOUT_VERSION
         spec_dict["binary_cache_checksum"] = {
             "hash_algorithm": "sha256",
             "hash": checksums[spec.dag_hash()].compressed_digest.digest,
@@ -945,6 +960,14 @@ def keys_fn(args):
     bindist.get_keys(args.install, args.trust, args.force)
 
 
+def preview_fn(args):
+    """analyze an installed spec and reports whether executables and libraries are relocatable"""
+    tty.warn(
+        "`spack buildcache preview` is deprecated since `spack buildcache push --allow-root` is "
+        "now the default. This command will be removed in Spack 0.22"
+    )
+
+
 def check_fn(args: argparse.Namespace):
     """check specs against remote binary mirror(s) to see if any need to be rebuilt
 

lib/spack/spack/cmd/ci.py

@@ -6,7 +6,6 @@
 import json
 import os
 import shutil
-import warnings
 from urllib.parse import urlparse, urlunparse
 
 import llnl.util.filesystem as fs
@@ -74,7 +73,7 @@ def setup_parser(subparser):
         "--optimize",
         action="store_true",
         default=False,
-        help="(DEPRECATED) optimize the gitlab yaml file for size\n\n"
+        help="(experimental) optimize the gitlab yaml file for size\n\n"
         "run the generated document through a series of optimization passes "
         "designed to reduce the size of the generated file",
     )
@@ -82,7 +81,7 @@ def setup_parser(subparser):
         "--dependencies",
         action="store_true",
         default=False,
-        help="(DEPRECATED) disable DAG scheduling (use 'plain' dependencies)",
+        help="(experimental) disable DAG scheduling (use 'plain' dependencies)",
     )
     generate.add_argument(
         "--buildcache-destination",
@@ -201,18 +200,6 @@ def ci_generate(args):
     before invoking this command. the value must be the CDash authorization token needed to create
     a build group and register all generated jobs under it
     """
-    if args.optimize:
-        warnings.warn(
-            "The --optimize option has been deprecated, and currently has no effect. "
-            "It will be removed in Spack v0.24."
-        )
-
-    if args.dependencies:
-        warnings.warn(
-            "The --dependencies option has been deprecated, and currently has no effect. "
-            "It will be removed in Spack v0.24."
-        )
-
     env = spack.cmd.require_active_env(cmd_name="ci generate")
 
     if args.copy_to:
@@ -225,6 +212,8 @@ def ci_generate(args):
 
     output_file = args.output_file
     copy_yaml_to = args.copy_to
+    run_optimizer = args.optimize
+    use_dependencies = args.dependencies
     prune_dag = args.prune_dag
     index_only = args.index_only
     artifacts_root = args.artifacts_root
@@ -245,6 +234,8 @@ def ci_generate(args):
         output_file,
         prune_dag=prune_dag,
         check_index_only=index_only,
+        run_optimizer=run_optimizer,
+        use_dependencies=use_dependencies,
         artifacts_root=artifacts_root,
         remote_mirror_override=buildcache_destination,
     )

lib/spack/spack/cmd/clean.py

@@ -106,8 +106,7 @@ def clean(parser, args):
 
     # Then do the cleaning falling through the cases
     if args.specs:
-        specs = spack.cmd.parse_specs(args.specs, concretize=False)
-        specs = list(spack.cmd.matching_spec_from_env(x) for x in specs)
+        specs = spack.cmd.parse_specs(args.specs, concretize=True)
         for spec in specs:
             msg = "Cleaning build stage [{0}]"
             tty.msg(msg.format(spec.short_spec))

lib/spack/spack/cmd/commands.py

@@ -11,7 +11,6 @@
 from argparse import ArgumentParser, Namespace
 from typing import IO, Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
 
-import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 from llnl.util.argparsewriter import ArgparseRstWriter, ArgparseWriter, Command
 from llnl.util.tty.colify import colify
@@ -867,9 +866,6 @@ def _commands(parser: ArgumentParser, args: Namespace) -> None:
             prepend_header(args, f)
             formatter(args, f)
 
-        if args.update_completion:
-            fs.set_executable(args.update)
-
     else:
         prepend_header(args, sys.stdout)
         formatter(args, sys.stdout)

lib/spack/spack/cmd/common/arguments.py

@@ -661,34 +661,32 @@ def mirror_name_or_url(m):
     # accidentally to a dir in the current working directory.
 
     # If there's a \ or / in the name, it's interpreted as a path or url.
-    if "/" in m or "\\" in m:
+    if "/" in m or "\\" in m or m in (".", ".."):
         return spack.mirror.Mirror(m)
 
     # Otherwise, the named mirror is required to exist.
     try:
         return spack.mirror.require_mirror_name(m)
     except ValueError as e:
-        raise argparse.ArgumentTypeError(
-            str(e) + ". Did you mean {}?".format(os.path.join(".", m))
-        )
+        raise argparse.ArgumentTypeError(f"{e}. Did you mean {os.path.join('.', m)}?") from e
 
 
 def mirror_url(url):
     try:
         return spack.mirror.Mirror.from_url(url)
     except ValueError as e:
-        raise argparse.ArgumentTypeError(str(e))
+        raise argparse.ArgumentTypeError(str(e)) from e
 
 
 def mirror_directory(path):
     try:
         return spack.mirror.Mirror.from_local_path(path)
     except ValueError as e:
-        raise argparse.ArgumentTypeError(str(e))
+        raise argparse.ArgumentTypeError(str(e)) from e
 
 
 def mirror_name(name):
     try:
         return spack.mirror.require_mirror_name(name)
     except ValueError as e:
-        raise argparse.ArgumentTypeError(str(e))
+        raise argparse.ArgumentTypeError(str(e)) from e

lib/spack/spack/cmd/concretize.py

@@ -3,9 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import llnl.util.tty as tty
-from llnl.string import plural
-
 import spack.cmd
 import spack.cmd.common.arguments
 import spack.environment as ev
@@ -46,9 +43,5 @@ def concretize(parser, args):
     with env.write_transaction():
         concretized_specs = env.concretize(force=args.force, tests=tests)
         if not args.quiet:
-            if concretized_specs:
-                tty.msg(f"Concretized {plural(len(concretized_specs), 'spec')}:")
-                ev.display_specs([concrete for _, concrete in concretized_specs])
-            else:
-                tty.msg("No new specs to concretize.")
+            ev.display_specs(concretized_specs)
         env.write()

lib/spack/spack/cmd/config.py

@@ -156,7 +156,7 @@ def print_flattened_configuration(*, blame: bool) -> None:
     """
     env = ev.active_environment()
     if env is not None:
-        pristine = env.manifest.yaml_content
+        pristine = env.manifest.pristine_yaml_content
         flattened = pristine.copy()
         flattened[spack.schema.env.TOP_LEVEL_KEY] = pristine[spack.schema.env.TOP_LEVEL_KEY].copy()
     else:
@@ -264,9 +264,7 @@ def config_remove(args):
 def _can_update_config_file(scope: spack.config.ConfigScope, cfg_file):
     if isinstance(scope, spack.config.SingleFileScope):
         return fs.can_access(cfg_file)
-    elif isinstance(scope, spack.config.DirectoryConfigScope):
-        return fs.can_write_to_dir(scope.path) and fs.can_access(cfg_file)
-    return False
+    return fs.can_write_to_dir(scope.path) and fs.can_access(cfg_file)
 
 
 def _config_change_requires_scope(path, spec, scope, match_spec=None):
@@ -364,11 +362,14 @@ def config_change(args):
 def config_update(args):
     # Read the configuration files
     spack.config.CONFIG.get_config(args.section, scope=args.scope)
-    updates: List[spack.config.ConfigScope] = [
-        x
-        for x in spack.config.CONFIG.format_updates[args.section]
-        if not isinstance(x, spack.config.InternalConfigScope) and x.writable
-    ]
+    updates: List[spack.config.ConfigScope] = list(
+        filter(
+            lambda s: not isinstance(
+                s, (spack.config.InternalConfigScope, spack.config.ImmutableConfigScope)
+            ),
+            spack.config.CONFIG.format_updates[args.section],
+        )
+    )
 
     cannot_overwrite, skip_system_scope = [], False
     for scope in updates:
@@ -446,7 +447,7 @@ def _can_revert_update(scope_dir, cfg_file, bkp_file):
 
 
 def config_revert(args):
-    scopes = [args.scope] if args.scope else [x.name for x in spack.config.CONFIG.writable_scopes]
+    scopes = [args.scope] if args.scope else [x.name for x in spack.config.CONFIG.file_scopes]
 
     # Search for backup files in the configuration scopes
     Entry = collections.namedtuple("Entry", ["scope", "cfg", "bkp"])

lib/spack/spack/cmd/create.py

@@ -2,6 +2,7 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
 import os
 import re
 import sys
@@ -933,7 +934,7 @@ def get_repository(args, name):
     # Figure out where the new package should live
     repo_path = args.repo
     if repo_path is not None:
-        repo = spack.repo.from_path(repo_path)
+        repo = spack.repo.Repo(repo_path)
         if spec.namespace and spec.namespace != repo.namespace:
             tty.die(
                 "Can't create package with namespace {0} in repo with "
@@ -941,7 +942,9 @@ def get_repository(args, name):
             )
     else:
         if spec.namespace:
-            repo = spack.repo.PATH.get_repo(spec.namespace)
+            repo = spack.repo.PATH.get_repo(spec.namespace, None)
+            if not repo:
+                tty.die("Unknown namespace: '{0}'".format(spec.namespace))
         else:
             repo = spack.repo.PATH.first_repo()
 

lib/spack/spack/cmd/dependents.py

@@ -47,6 +47,16 @@ def inverted_dependencies():
     dependents of, e.g., `mpi`, but virtuals are not included as
     actual dependents.
     """
+    dag = {}
+    for pkg_cls in spack.repo.PATH.all_package_classes():
+        dag.setdefault(pkg_cls.name, set())
+        for dep in pkg_cls.dependencies_by_name():
+            deps = [dep]
+
+            # expand virtuals if necessary
+            if spack.repo.PATH.is_virtual(dep):
+                deps += [s.name for s in spack.repo.PATH.providers_for(dep)]
+
     dag = collections.defaultdict(set)
     for pkg_cls in spack.repo.PATH.all_package_classes():
         for _, deps_by_name in pkg_cls.dependencies.items():

lib/spack/spack/cmd/develop.py

@@ -9,8 +9,6 @@
 
 import spack.cmd
 import spack.config
-import spack.fetch_strategy
-import spack.repo
 import spack.spec
 import spack.util.path
 import spack.version
@@ -71,15 +69,13 @@ def _retrieve_develop_source(spec, abspath):
     # We construct a package class ourselves, rather than asking for
     # Spec.package, since Spec only allows this when it is concrete
     package = pkg_cls(spec)
-    source_stage = package.stage[0]
-    if isinstance(source_stage.fetcher, spack.fetch_strategy.GitFetchStrategy):
-        source_stage.fetcher.get_full_repo = True
+    if isinstance(package.stage[0].fetcher, spack.fetch_strategy.GitFetchStrategy):
+        package.stage[0].fetcher.get_full_repo = True
         # If we retrieved this version before and cached it, we may have
         # done so without cloning the full git repo; likewise, any
         # mirror might store an instance with truncated history.
-        source_stage.disable_mirrors()
+        package.stage[0].disable_mirrors()
 
-    source_stage.fetcher.set_package(package)
     package.stage.steal_source(abspath)
 
 

lib/spack/spack/cmd/edit.py

@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import errno
 import glob
 import os
 
@@ -12,13 +11,43 @@
 import spack.cmd
 import spack.paths
 import spack.repo
-import spack.util.editor
+from spack.spec import Spec
+from spack.util.editor import editor
 
 description = "open package files in $EDITOR"
 section = "packaging"
 level = "short"
 
 
+def edit_package(name, repo_path, namespace):
+    """Opens the requested package file in your favorite $EDITOR.
+
+    Args:
+        name (str): The name of the package
+        repo_path (str): The path to the repository containing this package
+        namespace (str): A valid namespace registered with Spack
+    """
+    # Find the location of the package
+    if repo_path:
+        repo = spack.repo.Repo(repo_path)
+    elif namespace:
+        repo = spack.repo.PATH.get_repo(namespace)
+    else:
+        repo = spack.repo.PATH
+    path = repo.filename_for_package_name(name)
+
+    spec = Spec(name)
+    if os.path.exists(path):
+        if not os.path.isfile(path):
+            tty.die("Something is wrong. '{0}' is not a file!".format(path))
+        if not os.access(path, os.R_OK):
+            tty.die("Insufficient permissions on '%s'!" % path)
+    else:
+        raise spack.repo.UnknownPackageError(spec.name)
+
+    editor(path)
+
+
 def setup_parser(subparser):
     excl_args = subparser.add_mutually_exclusive_group()
 
@@ -69,67 +98,41 @@ def setup_parser(subparser):
     excl_args.add_argument("-r", "--repo", default=None, help="path to repo to edit package in")
     excl_args.add_argument("-N", "--namespace", default=None, help="namespace of package to edit")
 
-    subparser.add_argument("package", nargs="*", default=None, help="package name")
-
-
-def locate_package(name: str, repo: spack.repo.Repo) -> str:
-    path = repo.filename_for_package_name(name)
-
-    try:
-        with open(path, "r"):
-            return path
-    except OSError as e:
-        if e.errno == errno.ENOENT:
-            raise spack.repo.UnknownPackageError(name) from e
-        tty.die(f"Cannot edit package: {e}")
-
-
-def locate_file(name: str, path: str) -> str:
-    # convert command names to python module name
-    if path == spack.paths.command_path:
-        name = spack.cmd.python_name(name)
-
-    file_path = os.path.join(path, name)
-
-    # Try to open direct match.
-    try:
-        with open(file_path, "r"):
-            return file_path
-    except OSError as e:
-        if e.errno != errno.ENOENT:
-            tty.die(f"Cannot edit file: {e}")
-        pass
-
-    # Otherwise try to find a file that starts with the name
-    candidates = glob.glob(file_path + "*")
-    exclude_list = [".pyc", "~"]  # exclude binaries and backups
-    files = [f for f in candidates if not any(f.endswith(ext) for ext in exclude_list)]
-    if len(files) > 1:
-        tty.die(
-            f"Multiple files start with `{name}`:\n"
-            + "\n".join(f"        {os.path.basename(f)}" for f in files)
-        )
-    elif not files:
-        tty.die(f"No file for '{name}' was found in {path}")
-    return files[0]
+    subparser.add_argument("package", nargs="?", default=None, help="package name")
 
 
 def edit(parser, args):
-    names = args.package
+    name = args.package
+
+    # By default, edit package files
+    path = spack.paths.packages_path
 
     # If `--command`, `--test`, or `--module` is chosen, edit those instead
     if args.path:
-        paths = [locate_file(name, args.path) for name in names] if names else [args.path]
-        spack.util.editor.editor(*paths)
-    elif names:
-        if args.repo:
-            repo = spack.repo.from_path(args.repo)
-        elif args.namespace:
-            repo = spack.repo.PATH.get_repo(args.namespace)
-        else:
-            repo = spack.repo.PATH
-        paths = [locate_package(name, repo) for name in names]
-        spack.util.editor.editor(*paths)
+        path = args.path
+        if name:
+            # convert command names to python module name
+            if path == spack.paths.command_path:
+                name = spack.cmd.python_name(name)
+
+            path = os.path.join(path, name)
+            if not os.path.exists(path):
+                files = glob.glob(path + "*")
+                exclude_list = [".pyc", "~"]  # exclude binaries and backups
+                files = list(filter(lambda x: all(s not in x for s in exclude_list), files))
+                if len(files) > 1:
+                    m = "Multiple files exist with the name {0}.".format(name)
+                    m += " Please specify a suffix. Files are:\n\n"
+                    for f in files:
+                        m += "        " + os.path.basename(f) + "\n"
+                    tty.die(m)
+                if not files:
+                    tty.die("No file for '{0}' was found in {1}".format(name, path))
+                path = files[0]  # already confirmed only one entry in files
+
+        editor(path)
+    elif name:
+        edit_package(name, args.repo, args.namespace)
     else:
         # By default open the directory where packages live
-        spack.util.editor.editor(spack.paths.packages_path)
+        editor(path)

lib/spack/spack/cmd/env.py

@@ -468,32 +468,30 @@ def env_remove(args):
     This removes an environment managed by Spack. Directory environments
     and manifests embedded in repositories should be removed manually.
     """
-    read_envs = []
+    remove_envs = []
     valid_envs = []
     bad_envs = []
-    invalid_envs = []
 
     for env_name in ev.all_environment_names():
         try:
             env = ev.read(env_name)
-            valid_envs.append(env_name)
+            valid_envs.append(env)
 
             if env_name in args.rm_env:
-                read_envs.append(env)
+                remove_envs.append(env)
         except (spack.config.ConfigFormatError, ev.SpackEnvironmentConfigError):
-            invalid_envs.append(env_name)
-
             if env_name in args.rm_env:
                 bad_envs.append(env_name)
 
-        # Check if env is linked to another before trying to remove
-        for name in valid_envs:
+    # Check if remove_env is included from another env before trying to remove
+    for env in valid_envs:
+        for remove_env in remove_envs:
             # don't check if environment is included to itself
-            if name == env_name:
+            if env.name == remove_env.name:
                 continue
-            environ = ev.Environment(ev.root(name))
-            if ev.root(env_name) in environ.included_concrete_envs:
-                msg = f'Environment "{env_name}" is being used by environment "{name}"'
+
+            if remove_env.path in env.included_concrete_envs:
+                msg = f'Environment "{remove_env.name}" is being used by environment "{env.name}"'
                 if args.force:
                     tty.warn(msg)
                 else:
@@ -506,7 +504,7 @@ def env_remove(args):
         if not answer:
             tty.die("Will not remove any environments")
 
-    for env in read_envs:
+    for env in remove_envs:
         name = env.name
         if env.active:
             tty.die(f"Environment {name} can't be removed while activated.")

lib/spack/spack/cmd/external.py

@@ -7,7 +7,7 @@
 import os
 import re
 import sys
-from typing import List, Optional, Set
+from typing import List, Optional
 
 import llnl.util.tty as tty
 import llnl.util.tty.colify as colify
@@ -19,7 +19,6 @@
 import spack.detection
 import spack.error
 import spack.repo
-import spack.spec
 import spack.util.environment
 from spack.cmd.common import arguments
 
@@ -139,26 +138,14 @@ def external_find(args):
         candidate_packages, path_hints=args.path, max_workers=args.jobs
     )
 
-    new_specs = spack.detection.update_configuration(
+    new_entries = spack.detection.update_configuration(
         detected_packages, scope=args.scope, buildable=not args.not_buildable
     )
-
-    # If the user runs `spack external find --not-buildable mpich` we also mark `mpi` non-buildable
-    # to avoid that the concretizer picks a different mpi provider.
-    if new_specs and args.not_buildable:
-        virtuals: Set[str] = {
-            virtual.name
-            for new_spec in new_specs
-            for virtual_specs in spack.repo.PATH.get_pkg_class(new_spec.name).provided.values()
-            for virtual in virtual_specs
-        }
-        new_virtuals = spack.detection.set_virtuals_nonbuildable(virtuals, scope=args.scope)
-        new_specs.extend(spack.spec.Spec(name) for name in new_virtuals)
-
-    if new_specs:
+    if new_entries:
         path = spack.config.CONFIG.get_config_filename(args.scope, "packages")
-        tty.msg(f"The following specs have been detected on this system and added to {path}")
-        spack.cmd.display_specs(new_specs)
+        msg = "The following specs have been detected on this system and added to {0}"
+        tty.msg(msg.format(path))
+        spack.cmd.display_specs(new_entries)
     else:
         tty.msg("No new external packages detected")
 

lib/spack/spack/cmd/find.py

@@ -46,10 +46,6 @@ def setup_parser(subparser):
         help="output specs as machine-readable json records",
     )
 
-    subparser.add_argument(
-        "-I", "--install-status", action="store_true", help="show install status of packages"
-    )
-
     subparser.add_argument(
         "-d", "--deps", action="store_true", help="output dependencies along with found specs"
     )
@@ -173,9 +169,9 @@ def query_arguments(args):
     if (args.missing or args.only_missing) and not args.only_deprecated:
         installed.append(InstallStatuses.MISSING)
 
-    known = any
+    predicate_fn = None
     if args.unknown:
-        known = False
+        predicate_fn = lambda x: not spack.repo.PATH.exists(x.spec.name)
 
     explicit = any
     if args.explicit:
@@ -183,7 +179,7 @@ def query_arguments(args):
     if args.implicit:
         explicit = False
 
-    q_args = {"installed": installed, "known": known, "explicit": explicit}
+    q_args = {"installed": installed, "predicate_fn": predicate_fn, "explicit": explicit}
 
     install_tree = args.install_tree
     upstreams = spack.config.get("upstreams", {})
@@ -297,24 +293,25 @@ def root_decorator(spec, string):
         )
         print()
 
+    if args.show_concretized:
+        tty.msg("Concretized roots")
+        cmd.display_specs(env.specs_by_hash.values(), args, decorator=decorator)
+        print()
+
+    # Display a header for the installed packages section IF there are installed
+    # packages. If there aren't any, we'll just end up printing "0 installed packages"
+    # later.
+    if results and not args.only_roots:
+        tty.msg("Installed packages")
+
 
 def find(parser, args):
-    env = ev.active_environment()
+    q_args = query_arguments(args)
+    results = args.specs(**q_args)
 
+    env = ev.active_environment()
     if not env and args.only_roots:
         tty.die("-r / --only-roots requires an active environment")
-    if not env and args.show_concretized:
-        tty.die("-c / --show-concretized requires an active environment")
-
-    if env:
-        if args.constraint:
-            init_specs = spack.cmd.parse_specs(args.constraint)
-            results = env.all_matching_specs(*init_specs)
-        else:
-            results = env.all_specs()
-    else:
-        q_args = query_arguments(args)
-        results = args.specs(**q_args)
 
     decorator = make_env_decorator(env) if env else lambda s, f: f
 
@@ -335,11 +332,6 @@ def find(parser, args):
     if args.loaded:
         results = spack.cmd.filter_loaded_specs(results)
 
-    if args.install_status or args.show_concretized:
-        status_fn = spack.spec.Spec.install_status
-    else:
-        status_fn = None
-
     # Display the result
     if args.json:
         cmd.display_specs_as_json(results, deps=args.deps)
@@ -348,34 +340,12 @@ def find(parser, args):
             if env:
                 display_env(env, args, decorator, results)
 
+        count_suffix = " (not shown)"
         if not args.only_roots:
-            display_results = results
-            if not args.show_concretized:
-                display_results = list(x for x in results if x.installed)
-            cmd.display_specs(
-                display_results, args, decorator=decorator, all_headers=True, status_fn=status_fn
-            )
+            cmd.display_specs(results, args, decorator=decorator, all_headers=True)
+            count_suffix = ""
 
         # print number of installed packages last (as the list may be long)
         if sys.stdout.isatty() and args.groups:
-            installed_suffix = ""
-            concretized_suffix = " to be installed"
-
-            if args.only_roots:
-                installed_suffix += " (not shown)"
-                concretized_suffix += " (not shown)"
-            else:
-                if env and not args.show_concretized:
-                    concretized_suffix += " (show with `spack find -c`)"
-
             pkg_type = "loaded" if args.loaded else "installed"
-            spack.cmd.print_how_many_pkgs(
-                list(x for x in results if x.installed), pkg_type, suffix=installed_suffix
-            )
-
-            if env:
-                spack.cmd.print_how_many_pkgs(
-                    list(x for x in results if not x.installed),
-                    "concretized",
-                    suffix=concretized_suffix,
-                )
+            spack.cmd.print_how_many_pkgs(results, pkg_type, suffix=count_suffix)

lib/spack/spack/cmd/gc.py

@@ -56,6 +56,7 @@ def roots_from_environments(args, active_env):
 
     # -e says "also preserve things needed by this particular env"
     for env_name_or_dir in args.except_environment:
+        print("HMM", env_name_or_dir)
         if ev.exists(env_name_or_dir):
             env = ev.read(env_name_or_dir)
         elif ev.is_env_dir(env_name_or_dir):

lib/spack/spack/cmd/help.py

@@ -50,7 +50,7 @@
       @B{++}, @r{--}, @r{~~}, @B{==}                propagate variants to package dependencies
 
     architecture variants:
-      @m{platform=platform}             linux, darwin, freebsd, windows
+      @m{platform=platform}             linux, darwin, cray, etc.
       @m{os=operating_system}           specific <operating_system>
       @m{target=target}                 specific <target> processor
       @m{arch=platform-os-target}       shortcut for all three above

lib/spack/spack/cmd/install.py

@@ -10,7 +10,6 @@
 from typing import List
 
 import llnl.util.filesystem as fs
-from llnl.string import plural
 from llnl.util import lang, tty
 
 import spack.build_environment
@@ -376,9 +375,7 @@ def _maybe_add_and_concretize(args, env, specs):
         # `spack concretize`
         tests = compute_tests_install_kwargs(env.user_specs, args.test)
         concretized_specs = env.concretize(tests=tests)
-        if concretized_specs:
-            tty.msg(f"Concretized {plural(len(concretized_specs), 'spec')}")
-            ev.display_specs([concrete for _, concrete in concretized_specs])
+        ev.display_specs(concretized_specs)
 
         # save view regeneration for later, so that we only do it
         # once, as it can be slow.

lib/spack/spack/cmd/mark.py

@@ -101,8 +101,9 @@ def do_mark(specs, explicit):
         specs (list): list of specs to be marked
         explicit (bool): whether to mark specs as explicitly installed
     """
-    for spec in specs:
-        spack.store.STORE.db.update_explicit(spec, explicit)
+    with spack.store.STORE.db.write_transaction():
+        for spec in specs:
+            spack.store.STORE.db.mark(spec, "explicit", explicit)
 
 
 def mark_specs(args, specs):

lib/spack/spack/cmd/modules/__init__.py

@@ -377,7 +377,10 @@ def refresh(module_type, specs, args):
 def modules_cmd(parser, args, module_type, callbacks=callbacks):
     # Qualifiers to be used when querying the db for specs
     constraint_qualifiers = {
-        "refresh": {"installed": True, "known": lambda x: not spack.repo.PATH.exists(x)}
+        "refresh": {
+            "installed": True,
+            "predicate_fn": lambda x: spack.repo.PATH.exists(x.spec.name),
+        }
     }
     query_args = constraint_qualifiers.get(args.subparser_name, {})
 

lib/spack/spack/cmd/repo.py

@@ -91,7 +91,7 @@ def repo_add(args):
         tty.die("Not a Spack repository: %s" % path)
 
     # Make sure it's actually a spack repository by constructing it.
-    repo = spack.repo.from_path(canon_path)
+    repo = spack.repo.Repo(canon_path)
 
     # If that succeeds, finally add it to the configuration.
     repos = spack.config.get("repos", scope=args.scope)
@@ -124,7 +124,7 @@ def repo_remove(args):
     # If it is a namespace, remove corresponding repo
     for path in repos:
         try:
-            repo = spack.repo.from_path(path)
+            repo = spack.repo.Repo(path)
             if repo.namespace == namespace_or_path:
                 repos.remove(path)
                 spack.config.set("repos", repos, args.scope)
@@ -142,7 +142,7 @@ def repo_list(args):
     repos = []
     for r in roots:
         try:
-            repos.append(spack.repo.from_path(r))
+            repos.append(spack.repo.Repo(r))
         except spack.repo.RepoError:
             continue
 

lib/spack/spack/cmd/solve.py

@@ -114,16 +114,15 @@ def _process_result(result, show, required_format, kwargs):
 
     # dump the solutions as concretized specs
     if "solutions" in show:
-        if required_format:
-            for spec in result.specs:
-                # With -y, just print YAML to output.
-                if required_format == "yaml":
-                    # use write because to_yaml already has a newline.
-                    sys.stdout.write(spec.to_yaml(hash=ht.dag_hash))
-                elif required_format == "json":
-                    sys.stdout.write(spec.to_json(hash=ht.dag_hash))
-        else:
-            sys.stdout.write(spack.spec.tree(result.specs, color=sys.stdout.isatty(), **kwargs))
+        for spec in result.specs:
+            # With -y, just print YAML to output.
+            if required_format == "yaml":
+                # use write because to_yaml already has a newline.
+                sys.stdout.write(spec.to_yaml(hash=ht.dag_hash))
+            elif required_format == "json":
+                sys.stdout.write(spec.to_json(hash=ht.dag_hash))
+            else:
+                sys.stdout.write(spec.tree(color=sys.stdout.isatty(), **kwargs))
         print()
 
     if result.unsolved_specs and "solutions" in show:

lib/spack/spack/cmd/spec.py

@@ -105,19 +105,11 @@ def spec(parser, args):
         if env:
             env.concretize()
             specs = env.concretized_specs()
-
-            # environments are printed together in a combined tree() invocation,
-            # except when using --yaml or --json, which we print spec by spec below.
-            if not args.format:
-                tree_kwargs["key"] = spack.traverse.by_dag_hash
-                tree_kwargs["hashes"] = args.long or args.very_long
-                print(spack.spec.tree([concrete for _, concrete in specs], **tree_kwargs))
-                return
         else:
             tty.die("spack spec requires at least one spec or an active environment")
 
     for input, output in specs:
-        # With --yaml or --json, just print the raw specs to output
+        # With -y, just print YAML to output.
         if args.format:
             if args.format == "yaml":
                 # use write because to_yaml already has a newline.

lib/spack/spack/cmd/uninstall.py

@@ -151,8 +151,7 @@ def is_installed(spec):
         key=lambda s: s.dag_hash(),
     )
 
-    with spack.store.STORE.db.read_transaction():
-        return [spec for spec in specs if is_installed(spec)]
+    return [spec for spec in specs if is_installed(spec)]
 
 
 def dependent_environments(
@@ -240,8 +239,6 @@ def get_uninstall_list(args, specs: List[spack.spec.Spec], env: Optional[ev.Envi
             print()
             tty.info("The following environments still reference these specs:")
             colify([e.name for e in other_dependent_envs.keys()], indent=4)
-            if env:
-                msgs.append("use `spack remove` to remove the spec from the current environment")
             msgs.append("use `spack env remove` to remove environments")
         msgs.append("use `spack uninstall --force` to override")
         print()

lib/spack/spack/cmd/unload.py

@@ -71,7 +71,7 @@ def unload(parser, args):
             "Cannot specify specs on command line when unloading all specs with '--all'"
         )
 
-    hashes = os.environ.get(uenv.spack_loaded_hashes_var, "").split(os.pathsep)
+    hashes = os.environ.get(uenv.spack_loaded_hashes_var, "").split(":")
     if args.specs:
         specs = [
             spack.cmd.disambiguate_spec_from_hashes(spec, hashes)

lib/spack/spack/compiler.py

@@ -18,6 +18,7 @@
 import llnl.util.tty as tty
 from llnl.util.filesystem import path_contains_subdirectory, paths_containing_libs
 
+import spack.compilers
 import spack.error
 import spack.schema.environment
 import spack.spec
@@ -289,7 +290,7 @@ def __init__(
         operating_system,
         target,
         paths,
-        modules=None,
+        modules: Optional[List[str]] = None,
         alias=None,
         environment=None,
         extra_rpaths=None,
@@ -694,6 +695,10 @@ def compiler_environment(self):
         try:
             # load modules and set env variables
             for module in self.modules:
+                # On cray, mic-knl module cannot be loaded without cce module
+                # See: https://github.com/spack/spack/issues/3153
+                if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
+                    spack.util.module_cmd.load_module("cce")
                 spack.util.module_cmd.load_module(module)
 
             # apply other compiler environment changes

lib/spack/spack/compilers/__init__.py

@@ -260,7 +260,7 @@ def _init_compiler_config(
 def compiler_config_files():
     config_files = list()
     config = spack.config.CONFIG
-    for scope in config.writable_scopes:
+    for scope in config.file_scopes:
         name = scope.name
         compiler_config = config.get("compilers", scope=name)
         if compiler_config:
@@ -488,7 +488,7 @@ def supported_compilers_for_host_platform() -> List[str]:
     return supported_compilers_for_platform(host_plat)
 
 
-def supported_compilers_for_platform(platform: "spack.platforms.Platform") -> List[str]:
+def supported_compilers_for_platform(platform: spack.platforms.Platform) -> List[str]:
     """Return a set of compiler class objects supported by Spack
     that are also supported by the provided platform
 

lib/spack/spack/compilers/clang.py

@@ -96,8 +96,6 @@ def verbose_flag(self):
 
     openmp_flag = "-fopenmp"
 
-    # C++ flags based on CMake Modules/Compiler/Clang.cmake
-
     @property
     def cxx11_flag(self):
         if self.real_version < Version("3.3"):
@@ -122,24 +120,6 @@ def cxx17_flag(self):
 
         return "-std=c++17"
 
-    @property
-    def cxx20_flag(self):
-        if self.real_version < Version("5.0"):
-            raise UnsupportedCompilerFlag(self, "the C++20 standard", "cxx20_flag", "< 5.0")
-        elif self.real_version < Version("11.0"):
-            return "-std=c++2a"
-        else:
-            return "-std=c++20"
-
-    @property
-    def cxx23_flag(self):
-        if self.real_version < Version("12.0"):
-            raise UnsupportedCompilerFlag(self, "the C++23 standard", "cxx23_flag", "< 12.0")
-        elif self.real_version < Version("17.0"):
-            return "-std=c++2b"
-        else:
-            return "-std=c++23"
-
     @property
     def c99_flag(self):
         return "-std=c99"
@@ -162,10 +142,7 @@ def c17_flag(self):
     def c23_flag(self):
         if self.real_version < Version("9.0"):
             raise UnsupportedCompilerFlag(self, "the C23 standard", "c23_flag", "< 9.0")
-        elif self.real_version < Version("18.0"):
-            return "-std=c2x"
-        else:
-            return "-std=c23"
+        return "-std=c2x"
 
     @property
     def cc_pic_flag(self):

lib/spack/spack/compilers/oneapi.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import os
-from os.path import dirname, join
+from os.path import dirname
 
 from llnl.util import tty
 
@@ -135,12 +135,8 @@ def setup_custom_environment(self, pkg, env):
         # It is located in the same directory as the driver. Error message:
         #   clang++: error: unable to execute command:
         #   Executable "sycl-post-link" doesn't exist!
-        # also ensures that shared objects and libraries required by the compiler,
-        # e.g. libonnx, can be found succesfully
-        # due to a fix, this is no longer required for OneAPI versions >= 2024.2
-        if self.cxx and pkg.spec.satisfies("%oneapi@:2024.1"):
+        if self.cxx:
             env.prepend_path("PATH", dirname(self.cxx))
-            env.prepend_path("LD_LIBRARY_PATH", join(dirname(dirname(self.cxx)), "lib"))
 
         # 2024 release bumped the libsycl version because of an ABI
         # change, 2024 compilers are required.  You will see this

lib/spack/spack/config.py

@@ -35,10 +35,11 @@
 import os
 import re
 import sys
-from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Type, Union
 
 from llnl.util import filesystem, lang, tty
 
+import spack.compilers
 import spack.paths
 import spack.platforms
 import spack.schema
@@ -116,39 +117,21 @@
 
 
 class ConfigScope:
-    def __init__(self, name: str) -> None:
-        self.name = name
-        self.writable = False
-        self.sections = syaml.syaml_dict()
+    """This class represents a configuration scope.
 
-    def get_section_filename(self, section: str) -> str:
-        raise NotImplementedError
+    A scope is one directory containing named configuration files.
+    Each file is a config "section" (e.g., mirrors, compilers, etc.).
+    """
 
-    def get_section(self, section: str) -> Optional[YamlConfigDict]:
-        raise NotImplementedError
-
-    def _write_section(self, section: str) -> None:
-        raise NotImplementedError
+    def __init__(self, name, path) -> None:
+        self.name = name  # scope name.
+        self.path = path  # path to directory containing configs.
+        self.sections = syaml.syaml_dict()  # sections read from config files.
 
     @property
     def is_platform_dependent(self) -> bool:
-        return False
-
-    def clear(self) -> None:
-        """Empty cached config information."""
-        self.sections = syaml.syaml_dict()
-
-    def __repr__(self) -> str:
-        return f"<ConfigScope: {self.name}>"
-
-
-class DirectoryConfigScope(ConfigScope):
-    """Config scope backed by a directory containing one file per section."""
-
-    def __init__(self, name: str, path: str, *, writable: bool = True) -> None:
-        super().__init__(name)
-        self.path = path
-        self.writable = writable
+        """Returns true if the scope name is platform specific"""
+        return os.sep in self.name
 
     def get_section_filename(self, section: str) -> str:
         """Returns the filename associated with a given section"""
@@ -165,15 +148,14 @@ def get_section(self, section: str) -> Optional[YamlConfigDict]:
         return self.sections[section]
 
     def _write_section(self, section: str) -> None:
-        if not self.writable:
-            raise ConfigError(f"Cannot write to immutable scope {self}")
-
         filename = self.get_section_filename(section)
         data = self.get_section(section)
         if data is None:
             return
 
-        validate(data, SECTION_SCHEMAS[section])
+        # We copy data here to avoid adding defaults at write time
+        validate_data = copy.deepcopy(data)
+        validate(validate_data, SECTION_SCHEMAS[section])
 
         try:
             filesystem.mkdirp(self.path)
@@ -182,23 +164,19 @@ def _write_section(self, section: str) -> None:
         except (syaml.SpackYAMLError, OSError) as e:
             raise ConfigFileError(f"cannot write to '{filename}'") from e
 
-    @property
-    def is_platform_dependent(self) -> bool:
-        """Returns true if the scope name is platform specific"""
-        return "/" in self.name
+    def clear(self) -> None:
+        """Empty cached config information."""
+        self.sections = syaml.syaml_dict()
+
+    def __repr__(self) -> str:
+        return f"<ConfigScope: {self.name}: {self.path}>"
 
 
 class SingleFileScope(ConfigScope):
     """This class represents a configuration scope in a single YAML file."""
 
     def __init__(
-        self,
-        name: str,
-        path: str,
-        schema: YamlConfigDict,
-        *,
-        yaml_path: Optional[List[str]] = None,
-        writable: bool = True,
+        self, name: str, path: str, schema: YamlConfigDict, yaml_path: Optional[List[str]] = None
     ) -> None:
         """Similar to ``ConfigScope`` but can be embedded in another schema.
 
@@ -217,13 +195,15 @@ def __init__(
                        config:
                          install_tree: $spack/opt/spack
         """
-        super().__init__(name)
+        super().__init__(name, path)
         self._raw_data: Optional[YamlConfigDict] = None
         self.schema = schema
-        self.path = path
-        self.writable = writable
         self.yaml_path = yaml_path or []
 
+    @property
+    def is_platform_dependent(self) -> bool:
+        return False
+
     def get_section_filename(self, section) -> str:
         return self.path
 
@@ -277,8 +257,6 @@ def get_section(self, section: str) -> Optional[YamlConfigDict]:
         return self.sections.get(section, None)
 
     def _write_section(self, section: str) -> None:
-        if not self.writable:
-            raise ConfigError(f"Cannot write to immutable scope {self}")
         data_to_write: Optional[YamlConfigDict] = self._raw_data
 
         # If there is no existing data, this section SingleFileScope has never
@@ -323,6 +301,19 @@ def __repr__(self) -> str:
         return f"<SingleFileScope: {self.name}: {self.path}>"
 
 
+class ImmutableConfigScope(ConfigScope):
+    """A configuration scope that cannot be written to.
+
+    This is used for ConfigScopes passed on the command line.
+    """
+
+    def _write_section(self, section) -> None:
+        raise ConfigError(f"Cannot write to immutable scope {self}")
+
+    def __repr__(self) -> str:
+        return f"<ImmutableConfigScope: {self.name}: {self.path}>"
+
+
 class InternalConfigScope(ConfigScope):
     """An internal configuration scope that is not persisted to a file.
 
@@ -332,7 +323,7 @@ class InternalConfigScope(ConfigScope):
     """
 
     def __init__(self, name: str, data: Optional[YamlConfigDict] = None) -> None:
-        super().__init__(name)
+        super().__init__(name, None)
         self.sections = syaml.syaml_dict()
 
         if data is not None:
@@ -342,6 +333,9 @@ def __init__(self, name: str, data: Optional[YamlConfigDict] = None) -> None:
                 validate({section: dsec}, SECTION_SCHEMAS[section])
                 self.sections[section] = _mark_internal(syaml.syaml_dict({section: dsec}), name)
 
+    def get_section_filename(self, section: str) -> str:
+        raise NotImplementedError("Cannot get filename for InternalConfigScope.")
+
     def get_section(self, section: str) -> Optional[YamlConfigDict]:
         """Just reads from an internal dictionary."""
         if section not in self.sections:
@@ -446,21 +440,27 @@ def remove_scope(self, scope_name: str) -> Optional[ConfigScope]:
         return scope
 
     @property
-    def writable_scopes(self) -> Generator[ConfigScope, None, None]:
-        """Generator of writable scopes with an associated file."""
-        return (s for s in self.scopes.values() if s.writable)
+    def file_scopes(self) -> List[ConfigScope]:
+        """List of writable scopes with an associated file."""
+        return [
+            s
+            for s in self.scopes.values()
+            if (type(s) is ConfigScope or type(s) is SingleFileScope)
+        ]
 
     def highest_precedence_scope(self) -> ConfigScope:
-        """Writable scope with highest precedence."""
-        return next(s for s in reversed(self.scopes.values()) if s.writable)  # type: ignore
+        """Non-internal scope with highest precedence."""
+        return next(reversed(self.file_scopes))
 
     def highest_precedence_non_platform_scope(self) -> ConfigScope:
-        """Writable non-platform scope with highest precedence"""
-        return next(
-            s
-            for s in reversed(self.scopes.values())  # type: ignore
-            if s.writable and not s.is_platform_dependent
-        )
+        """Non-internal non-platform scope with highest precedence
+
+        Platform-specific scopes are of the form scope/platform"""
+        generator = reversed(self.file_scopes)
+        highest = next(generator)
+        while highest and highest.is_platform_dependent:
+            highest = next(generator)
+        return highest
 
     def matching_scopes(self, reg_expr) -> List[ConfigScope]:
         """
@@ -755,14 +755,13 @@ def override(
 
 
 def _add_platform_scope(
-    cfg: Union[Configuration, lang.Singleton], name: str, path: str, writable: bool = True
+    cfg: Union[Configuration, lang.Singleton], scope_type: Type[ConfigScope], name: str, path: str
 ) -> None:
     """Add a platform-specific subdirectory for the current platform."""
     platform = spack.platforms.host().name
-    scope = DirectoryConfigScope(
-        f"{name}/{platform}", os.path.join(path, platform), writable=writable
-    )
-    cfg.push_scope(scope)
+    plat_name = os.path.join(name, platform)
+    plat_path = os.path.join(path, platform)
+    cfg.push_scope(scope_type(plat_name, plat_path))
 
 
 def config_paths_from_entry_points() -> List[Tuple[str, str]]:
@@ -793,27 +792,22 @@ def config_paths_from_entry_points() -> List[Tuple[str, str]]:
 def _add_command_line_scopes(
     cfg: Union[Configuration, lang.Singleton], command_line_scopes: List[str]
 ) -> None:
-    """Add additional scopes from the --config-scope argument, either envs or dirs."""
-    import spack.environment.environment as env  # circular import
+    """Add additional scopes from the --config-scope argument.
 
+    Command line scopes are named after their position in the arg list.
+    """
     for i, path in enumerate(command_line_scopes):
-        name = f"cmd_scope_{i}"
+        # We ensure that these scopes exist and are readable, as they are
+        # provided on the command line by the user.
+        if not os.path.isdir(path):
+            raise ConfigError(f"config scope is not a directory: '{path}'")
+        elif not os.access(path, os.R_OK):
+            raise ConfigError(f"config scope is not readable: '{path}'")
 
-        if env.exists(path):  # managed environment
-            manifest = env.EnvironmentManifestFile(env.root(path))
-        elif env.is_env_dir(path):  # anonymous environment
-            manifest = env.EnvironmentManifestFile(path)
-        elif os.path.isdir(path):  # directory with config files
-            cfg.push_scope(DirectoryConfigScope(name, path, writable=False))
-            _add_platform_scope(cfg, name, path, writable=False)
-            continue
-        else:
-            raise ConfigError(f"Invalid configuration scope: {path}")
-
-        for scope in manifest.env_config_scopes:
-            scope.name = f"{name}:{scope.name}"
-            scope.writable = False
-            cfg.push_scope(scope)
+        # name based on order on the command line
+        name = f"cmd_scope_{i:d}"
+        cfg.push_scope(ImmutableConfigScope(name, path))
+        _add_platform_scope(cfg, ImmutableConfigScope, name, path)
 
 
 def create() -> Configuration:
@@ -857,10 +851,10 @@ def create() -> Configuration:
 
     # add each scope and its platform-specific directory
     for name, path in configuration_paths:
-        cfg.push_scope(DirectoryConfigScope(name, path))
+        cfg.push_scope(ConfigScope(name, path))
 
         # Each scope can have per-platfom overrides in subdirectories
-        _add_platform_scope(cfg, name, path)
+        _add_platform_scope(cfg, ConfigScope, name, path)
 
     # add command-line scopes
     _add_command_line_scopes(cfg, COMMAND_LINE_SCOPES)
@@ -975,7 +969,7 @@ def set(path: str, value: Any, scope: Optional[str] = None) -> None:
 def add_default_platform_scope(platform: str) -> None:
     plat_name = os.path.join("defaults", platform)
     plat_path = os.path.join(CONFIGURATION_DEFAULTS_PATH[1], platform)
-    CONFIG.push_scope(DirectoryConfigScope(plat_name, plat_path))
+    CONFIG.push_scope(ConfigScope(plat_name, plat_path))
 
 
 def scopes() -> Dict[str, ConfigScope]:
@@ -984,10 +978,19 @@ def scopes() -> Dict[str, ConfigScope]:
 
 
 def writable_scopes() -> List[ConfigScope]:
-    """Return list of writable scopes. Higher-priority scopes come first in the list."""
-    scopes = [x for x in CONFIG.scopes.values() if x.writable]
-    scopes.reverse()
-    return scopes
+    """
+    Return list of writable scopes. Higher-priority scopes come first in the
+    list.
+    """
+    return list(
+        reversed(
+            list(
+                x
+                for x in CONFIG.scopes.values()
+                if not isinstance(x, (InternalConfigScope, ImmutableConfigScope))
+            )
+        )
+    )
 
 
 def writable_scope_names() -> List[str]:
@@ -1077,8 +1080,11 @@ def validate(
     """
     import jsonschema
 
+    # Validate a copy to avoid adding defaults
+    # This allows us to round-trip data without adding to it.
+    test_data = syaml.deepcopy(data)
     try:
-        spack.schema.Validator(schema).validate(data)
+        spack.schema.Validator(schema).validate(test_data)
     except jsonschema.ValidationError as e:
         if hasattr(e.instance, "lc"):
             line_number = e.instance.lc.line + 1
@@ -1087,7 +1093,7 @@ def validate(
         raise ConfigFormatError(e, data, filename, line_number) from e
     # return the validated data so that we can access the raw data
     # mostly relevant for environments
-    return data
+    return test_data
 
 
 def read_config_file(
@@ -1593,7 +1599,7 @@ def _config_from(scopes_or_paths: List[Union[ConfigScope, str]]) -> Configuratio
         path = os.path.normpath(scope_or_path)
         assert os.path.isdir(path), f'"{path}" must be a directory'
         name = os.path.basename(path)
-        scopes.append(DirectoryConfigScope(name, path))
+        scopes.append(ConfigScope(name, path))
 
     configuration = Configuration(*scopes)
     return configuration

lib/spack/spack/container/images.json

@@ -78,17 +78,24 @@
         "image": "quay.io/almalinuxorg/almalinux:8"
       }
     },
-    "centos:stream9": {
+    "centos:stream": {
       "bootstrap": {
-        "template": "container/centos_stream9.dockerfile",
-        "image": "quay.io/centos/centos:stream9"
+        "template": "container/centos_stream.dockerfile",
+        "image": "quay.io/centos/centos:stream"
       },
       "os_package_manager": "dnf_epel",
-      "build": "spack/centos-stream9",
+      "build": "spack/centos-stream",
       "final": {
-        "image": "quay.io/centos/centos:stream9"
+        "image": "quay.io/centos/centos:stream"
       }
     },
+    "centos:7": {
+      "bootstrap": {
+        "template": "container/centos_7.dockerfile"
+      },
+      "os_package_manager": "yum",
+      "build": "spack/centos7"
+    },
     "opensuse/leap:15": {
       "bootstrap": {
         "template": "container/leap-15.dockerfile"

lib/spack/spack/database.py

@@ -283,12 +283,9 @@ def __reduce__(self):
                 database.  If it is a spec, we'll evaluate
                 ``spec.satisfies(query_spec)``
 
-            known (bool or None): Specs that are "known" are those
-                for which Spack can locate a ``package.py`` file -- i.e.,
-                Spack "knows" how to install them.  Specs that are unknown may
-                represent packages that existed in a previous version of
-                Spack, but have since either changed their name or
-                been removed
+            predicate_fn: optional predicate taking an InstallRecord as argument, and returning
+                whether that record is selected for the query. It can be used to craft criteria
+                that need some data for selection not provided by the Database itself.
 
             installed (bool or InstallStatus or typing.Iterable or None):
                 if ``True``, includes only installed
@@ -588,6 +585,9 @@ def _path(self, spec: "spack.spec.Spec") -> pathlib.Path:
         return self.dir / f"{spec.name}-{spec.dag_hash()}"
 
 
+SelectType = Callable[[InstallRecord], bool]
+
+
 class Database:
     #: Fields written for each install record
     record_fields: Tuple[str, ...] = DEFAULT_INSTALL_RECORD_FIELDS
@@ -1367,7 +1367,7 @@ def _deprecate(self, spec, deprecator):
         self._data[spec_key] = spec_rec
 
     @_autospec
-    def mark(self, spec, key, value):
+    def mark(self, spec: "spack.spec.Spec", key: str, value: Any) -> None:
         """Mark an arbitrary record on a spec."""
         with self.write_transaction():
             return self._mark(spec, key, value)
@@ -1516,7 +1516,7 @@ def get_by_hash(self, dag_hash, default=None, installed=any):
     def _query(
         self,
         query_spec=any,
-        known=any,
+        predicate_fn: Optional[SelectType] = None,
         installed=True,
         explicit=any,
         start_date=None,
@@ -1524,7 +1524,7 @@ def _query(
         hashes=None,
         in_buildcache=any,
         origin=None,
-    ):
+    ) -> List["spack.spec.Spec"]:
         """Run a query on the database."""
 
         # TODO: Specs are a lot like queries.  Should there be a
@@ -1570,7 +1570,7 @@ def _query(
             if explicit is not any and rec.explicit != explicit:
                 continue
 
-            if known is not any and known(rec.spec.name):
+            if predicate_fn is not None and not predicate_fn(rec):
                 continue
 
             if start_date or end_date:
@@ -1655,14 +1655,14 @@ def query(self, *args, **kwargs):
         query.__doc__ = ""
     query.__doc__ += _QUERY_DOCSTRING
 
-    def query_one(self, query_spec, known=any, installed=True):
+    def query_one(self, query_spec, predicate_fn=None, installed=True):
         """Query for exactly one spec that matches the query spec.
 
         Raises an assertion error if more than one spec matches the
         query. Returns None if no installed package matches.
 
         """
-        concrete_specs = self.query(query_spec, known=known, installed=installed)
+        concrete_specs = self.query(query_spec, predicate_fn=predicate_fn, installed=installed)
         assert len(concrete_specs) <= 1
         return concrete_specs[0] if concrete_specs else None
 
@@ -1709,24 +1709,6 @@ def root(key, record):
                 if id(rec.spec) not in needed and rec.installed
             ]
 
-    def update_explicit(self, spec, explicit):
-        """
-        Update the spec's explicit state in the database.
-
-        Args:
-            spec (spack.spec.Spec): the spec whose install record is being updated
-            explicit (bool): ``True`` if the package was requested explicitly
-                by the user, ``False`` if it was pulled in as a dependency of
-                an explicit package.
-        """
-        rec = self.get_record(spec)
-        if explicit != rec.explicit:
-            with self.write_transaction():
-                message = "{s.name}@{s.version} : marking the package {0}"
-                status = "explicit" if explicit else "implicit"
-                tty.debug(message.format(status, s=spec))
-                rec.explicit = explicit
-
 
 class UpstreamDatabaseLockingError(SpackError):
     """Raised when an operation would need to lock an upstream database"""

lib/spack/spack/detection/__init__.py

@@ -2,12 +2,7 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-from .common import (
-    DetectedPackage,
-    executable_prefix,
-    set_virtuals_nonbuildable,
-    update_configuration,
-)
+from .common import DetectedPackage, executable_prefix, update_configuration
 from .path import by_path, executables_in_path
 from .test import detection_tests
 
@@ -17,6 +12,5 @@
     "executables_in_path",
     "executable_prefix",
     "update_configuration",
-    "set_virtuals_nonbuildable",
     "detection_tests",
 ]