check for backtrace instead of debug

Signed-off-by: Todd Gamblin <tgamblin@llnl.gov>

.codecov.yml

@@ -5,7 +5,7 @@ coverage:
   status:
     project:
       default:
-        threshold: 0.2%
+        threshold: 2.0%
 
 ignore:
   - lib/spack/spack/test/.*

.github/workflows/audit.yaml

@@ -28,7 +28,7 @@ jobs:
       run:
         shell: ${{ matrix.system.shell }}
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
     - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: ${{inputs.python_version}}
@@ -66,7 +66,7 @@ jobs:
           ./share/spack/qa/validate_last_exit.ps1
           spack -d audit externals
           ./share/spack/qa/validate_last_exit.ps1
-    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
+    - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
       if: ${{ inputs.with_coverage == 'true' && runner.os != 'Windows' }}
       with:
         name: coverage-audits-${{ matrix.system.os }}

.github/workflows/bootstrap.yml

@@ -37,7 +37,7 @@ jobs:
               make patch unzip which xz python3 python3-devel tree \
               cmake bison
       - name: Checkout
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
         with:
           fetch-depth: 0
       - name: Bootstrap clingo
@@ -53,33 +53,27 @@ jobs:
     runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        runner: ['macos-13', 'macos-14', "ubuntu-latest", "windows-latest"]
+        runner: ['macos-13', 'macos-14', "ubuntu-latest"]
     steps:
       - name: Setup macOS
-        if: ${{ matrix.runner != 'ubuntu-latest' && matrix.runner != 'windows-latest' }}
+        if: ${{ matrix.runner != 'ubuntu-latest' }}
         run: |
           brew install cmake bison tree
       - name: Checkout
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
         with:
           fetch-depth: 0
       - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
         with:
           python-version: "3.12"
       - name: Bootstrap clingo
-        env:
-          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
-          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
-          USER_SCOPE_PARENT_DIR: ${{ matrix.runner == 'windows-latest' && '$env:userprofile' || '$HOME' }}
-          VALIDATE_LAST_EXIT: ${{ matrix.runner == 'windows-latest' && './share/spack/qa/validate_last_exit.ps1' || '' }}
         run: |
-          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
+          source share/spack/setup-env.sh
           spack bootstrap disable github-actions-v0.5
           spack bootstrap disable github-actions-v0.4
           spack external find --not-buildable cmake bison
           spack -d solve zlib
-          ${{ env.VALIDATE_LAST_EXIT }}
-          tree ${{ env.USER_SCOPE_PARENT_DIR }}/.spack/bootstrap/store/
+          tree $HOME/.spack/bootstrap/store/
 
   gnupg-sources:
     runs-on: ${{ matrix.runner }}
@@ -96,7 +90,7 @@ jobs:
         if: ${{ matrix.runner == 'ubuntu-latest' }}
         run: sudo rm -rf $(command -v gpg gpg2 patchelf)
       - name: Checkout
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
         with:
           fetch-depth: 0
       - name: Bootstrap GnuPG
@@ -112,10 +106,10 @@ jobs:
     runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        runner: ['macos-13', 'macos-14', "ubuntu-latest", "windows-latest"]
+        runner: ['macos-13', 'macos-14', "ubuntu-latest"]
     steps:
       - name: Setup macOS
-        if: ${{ matrix.runner != 'ubuntu-latest' && matrix.runner != 'windows-latest'}}
+        if: ${{ matrix.runner != 'ubuntu-latest' }}
         run: |
           brew install tree
           # Remove GnuPG since we want to bootstrap it
@@ -124,13 +118,8 @@ jobs:
         if: ${{ matrix.runner == 'ubuntu-latest' }}
         run: |
           sudo rm -rf $(which gpg) $(which gpg2) $(which patchelf)
-      - name: Setup Windows
-        if: ${{ matrix.runner == 'windows-latest' }}
-        run: |
-          Remove-Item -Path (Get-Command gpg).Path
-          Remove-Item -Path (Get-Command file).Path
       - name: Checkout
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
         with:
           fetch-depth: 0
       - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
@@ -142,20 +131,11 @@ jobs:
             3.11
             3.12
       - name: Set bootstrap sources
-        env:
-          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
-          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
-        run: |
-          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
-          spack bootstrap disable github-actions-v0.4
-      - name: Disable from source bootstrap
-        if: ${{ matrix.runner != 'windows-latest' }}
         run: |
           source share/spack/setup-env.sh
+          spack bootstrap disable github-actions-v0.4
           spack bootstrap disable spack-install
       - name: Bootstrap clingo
-        # No binary clingo on Windows yet
-        if: ${{ matrix.runner != 'windows-latest' }}
         run: |
           set -e
           for ver in '3.8' '3.9' '3.10' '3.11' '3.12' ; do
@@ -178,24 +158,48 @@ jobs:
             fi
           done
       - name: Bootstrap GnuPG
-        env:
-          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
-          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
-          USER_SCOPE_PARENT_DIR: ${{ matrix.runner == 'windows-latest' && '$env:userprofile' || '$HOME' }}
-          VALIDATE_LAST_EXIT: ${{ matrix.runner == 'windows-latest' && './share/spack/qa/validate_last_exit.ps1' || '' }}
         run: |
-          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
+          source share/spack/setup-env.sh
           spack -d gpg list
-          ${{ env.VALIDATE_LAST_EXIT }}
-          tree ${{ env.USER_SCOPE_PARENT_DIR }}/.spack/bootstrap/store/
+          tree $HOME/.spack/bootstrap/store/
       - name: Bootstrap File
-        env:
-          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
-          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
-          USER_SCOPE_PARENT_DIR: ${{ matrix.runner == 'windows-latest' && '$env:userprofile' || '$HOME' }}
-          VALIDATE_LAST_EXIT: ${{ matrix.runner == 'windows-latest' && './share/spack/qa/validate_last_exit.ps1' || '' }}
         run: |
-          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
+          source share/spack/setup-env.sh
           spack -d python share/spack/qa/bootstrap-file.py
-          ${{ env.VALIDATE_LAST_EXIT }}
-          tree ${{ env.USER_SCOPE_PARENT_DIR }}/.spack/bootstrap/store/
+          tree $HOME/.spack/bootstrap/store/
+
+  windows:
+    runs-on: "windows-latest"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
+        with:
+          python-version: "3.12"
+      - name: Setup Windows
+        run: |
+          Remove-Item -Path (Get-Command gpg).Path
+          Remove-Item -Path (Get-Command file).Path
+      - name: Bootstrap clingo
+        run: |
+          ./share/spack/setup-env.ps1
+          spack bootstrap disable github-actions-v0.5
+          spack bootstrap disable github-actions-v0.4
+          spack external find --not-buildable cmake bison
+          spack -d solve zlib
+          ./share/spack/qa/validate_last_exit.ps1
+          tree $env:userprofile/.spack/bootstrap/store/
+      - name: Bootstrap GnuPG
+        run: |
+          ./share/spack/setup-env.ps1
+          spack -d gpg list
+          ./share/spack/qa/validate_last_exit.ps1
+          tree $env:userprofile/.spack/bootstrap/store/
+      - name: Bootstrap File
+        run: |
+          ./share/spack/setup-env.ps1
+          spack -d python share/spack/qa/bootstrap-file.py
+          ./share/spack/qa/validate_last_exit.ps1
+          tree $env:userprofile/.spack/bootstrap/store/

.github/workflows/build-containers.yml

@@ -55,7 +55,7 @@ jobs:
     if: github.repository == 'spack/spack'
     steps:
       - name: Checkout
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
 
       - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
         id: docker_meta
@@ -87,7 +87,7 @@ jobs:
           fi
 
       - name: Upload Dockerfile
-        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
         with:
           name: dockerfiles_${{ matrix.dockerfile[0] }}
           path: dockerfiles
@@ -96,7 +96,7 @@ jobs:
         uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db
+        uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349
 
       - name: Log in to GitHub Container Registry
         uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
@@ -113,7 +113,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build & Deploy ${{ matrix.dockerfile[0] }}
-        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85
+        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75
         with:
           context: dockerfiles/${{ matrix.dockerfile[0] }}
           platforms: ${{ matrix.dockerfile[1] }}
@@ -126,7 +126,7 @@ jobs:
     needs: deploy-images
     steps:
       - name: Merge Artifacts
-        uses: actions/upload-artifact/merge@50769540e7f4bd5e21e526ee35c689e35e0d6874
+        uses: actions/upload-artifact/merge@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
         with:
           name: dockerfiles
           pattern: dockerfiles_*

.github/workflows/ci.yaml

@@ -15,18 +15,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  prechecks:
-    needs: [ changes ]
-    uses: ./.github/workflows/valid-style.yml
-    secrets: inherit
-    with:
-      with_coverage: ${{ needs.changes.outputs.core }}
-  all-prechecks:
-    needs: [ prechecks ]
-    runs-on: ubuntu-latest
-    steps:
-    - name: Success
-      run: "true"
   # Check which files have been updated by the PR
   changes:
     runs-on: ubuntu-latest
@@ -36,7 +24,7 @@ jobs:
       core: ${{ steps.filter.outputs.core }}
       packages: ${{ steps.filter.outputs.packages }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
         if: ${{ github.event_name == 'push' }}
         with:
           fetch-depth: 0
@@ -79,35 +67,34 @@ jobs:
     needs: [ prechecks, changes ]
     uses: ./.github/workflows/bootstrap.yml
     secrets: inherit
+
   unit-tests:
     if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.core == 'true' }}
     needs: [ prechecks, changes ]
     uses: ./.github/workflows/unit_tests.yaml
     secrets: inherit
-  upload-coverage:
-    needs: [ unit-tests, prechecks ]
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
-      with:
-        fetch-depth: 0
-    - name: Download coverage files
-      uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
-      with:
-        pattern: coverage-*
-        path: coverage
-        merge-multiple: true
-    - run: pip install --upgrade coverage
-    - run: ls -la coverage
-    - run: coverage combine -a coverage/.coverage*
-    - run: coverage xml
-    - name: "Upload coverage"
-      uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        verbose: true
-  all:
-    needs: [ upload-coverage, bootstrap ]
+
+  prechecks:
+    needs: [ changes ]
+    uses: ./.github/workflows/valid-style.yml
+    secrets: inherit
+    with:
+      with_coverage: ${{ needs.changes.outputs.core }}
+
+  all-prechecks:
+    needs: [ prechecks ]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Success
+      run: "true"
+
+  coverage:
+    needs: [ unit-tests, prechecks ]
+    uses: ./.github/workflows/coverage.yml
+    secrets: inherit
+
+  all:
+    needs: [ coverage, bootstrap ]
     runs-on: ubuntu-latest
     steps:
     - name: Success

.github/workflows/coverage.yml

@@ -0,0 +1,34 @@
+name: coverage
+
+on:
+  workflow_call:
+
+jobs:
+  # Upload coverage reports to codecov once as a single bundle
+  upload:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+
+    - name: Install python dependencies
+      run: pip install -r .github/workflows/requirements/coverage/requirements.txt
+
+    - name: Download coverage artifact files
+      uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
+      with:
+        pattern: coverage-*
+        path: coverage
+        merge-multiple: true
+
+    - run: ls -la coverage
+    - run: coverage combine -a coverage/.coverage*
+    - run: coverage xml
+
+    - name: "Upload coverage report to CodeCov"
+      uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238
+      with:
+        verbose: true

.github/workflows/nightly-win-builds.yml

@@ -14,7 +14,7 @@ jobs:
   build-paraview-deps:
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       with:
         fetch-depth: 0
     - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3

.github/workflows/requirements/coverage/requirements.txt

@@ -0,0 +1 @@
+coverage==7.6.1

.github/workflows/requirements/style/requirements.txt

@@ -1,4 +1,4 @@
-black==24.8.0
+black==24.10.0
 clingo==5.7.1
 flake8==7.1.1
 isort==5.13.2

.github/workflows/unit_tests.yaml

@@ -40,7 +40,7 @@ jobs:
           on_develop: false
 
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       with:
         fetch-depth: 0
     - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
@@ -80,7 +80,7 @@ jobs:
           UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }}
       run: |
           share/spack/qa/run-unit-tests
-    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
+    - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
       with:
         name: coverage-${{ matrix.os }}-python${{ matrix.python-version }}
         path: coverage
@@ -89,7 +89,7 @@ jobs:
   shell:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       with:
         fetch-depth: 0
     - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
@@ -113,7 +113,7 @@ jobs:
           COVERAGE: true
       run: |
           share/spack/qa/run-shell-tests
-    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
+    - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
       with:
         name: coverage-shell
         path: coverage
@@ -130,7 +130,7 @@ jobs:
           dnf install -y \
               bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
               make patch tcl unzip which xz
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
     - name: Setup repo and non-root user
       run: |
           git --version
@@ -149,32 +149,33 @@ jobs:
   clingo-cffi:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       with:
         fetch-depth: 0
     - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
-        python-version: '3.11'
+        python-version: '3.13'
     - name: Install System packages
       run: |
           sudo apt-get -y update
-          sudo apt-get -y install coreutils cvs gfortran graphviz gnupg2 mercurial ninja-build kcov
+          sudo apt-get -y install coreutils gfortran graphviz gnupg2
     - name: Install Python packages
       run: |
-          pip install --upgrade pip setuptools pytest coverage[toml] pytest-cov clingo pytest-xdist
+          pip install --upgrade pip setuptools pytest coverage[toml] pytest-cov clingo
           pip install --upgrade flake8 "isort>=4.3.5" "mypy>=0.900" "click" "black"
-    - name: Setup git configuration
-      run: |
-          # Need this for the git tests to succeed.
-          git --version
-          . .github/workflows/bin/setup_git.sh
     - name: Run unit tests (full suite with coverage)
       env:
           COVERAGE: true
           COVERAGE_FILE: coverage/.coverage-clingo-cffi
       run: |
-          share/spack/qa/run-unit-tests
-    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
+        . share/spack/setup-env.sh
+        spack bootstrap disable spack-install
+        spack bootstrap disable github-actions-v0.4
+        spack bootstrap disable github-actions-v0.5
+        spack bootstrap status
+        spack solve zlib
+        spack unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml lib/spack/spack/test/concretize.py
+    - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
       with:
         name: coverage-clingo-cffi
         path: coverage
@@ -187,7 +188,7 @@ jobs:
         os: [macos-13, macos-14]
         python-version: ["3.11"]
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       with:
         fetch-depth: 0
     - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
@@ -199,7 +200,7 @@ jobs:
           pip install --upgrade pytest coverage[toml] pytest-xdist pytest-cov
     - name: Setup Homebrew packages
       run: |
-        brew install dash fish gcc gnupg2 kcov
+        brew install dash fish gcc gnupg kcov
     - name: Run unit tests
       env:
         SPACK_TEST_PARALLEL: 4
@@ -212,7 +213,7 @@ jobs:
         $(which spack) solve zlib
         common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
         $(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
-    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
+    - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
       with:
         name: coverage-${{ matrix.os }}-python${{ matrix.python-version }}
         path: coverage
@@ -225,7 +226,7 @@ jobs:
           powershell Invoke-Expression -Command "./share/spack/qa/windows_test_setup.ps1"; {0}
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       with:
         fetch-depth: 0
     - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
@@ -243,7 +244,7 @@ jobs:
       run: |
         spack unit-test -x --verbose --cov --cov-config=pyproject.toml
         ./share/spack/qa/validate_last_exit.ps1
-    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
+    - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
       with:
         name: coverage-windows
         path: coverage

.github/workflows/valid-style.yml

@@ -18,7 +18,7 @@ jobs:
   validate:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
     - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: '3.11'
@@ -35,7 +35,7 @@ jobs:
   style:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       with:
         fetch-depth: 0
     - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
@@ -70,7 +70,7 @@ jobs:
           dnf install -y \
               bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
               make patch tcl unzip which xz
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       - name: Setup repo and non-root user
         run: |
           git --version
@@ -85,7 +85,7 @@ jobs:
           source share/spack/setup-env.sh
           spack debug report
           spack -d bootstrap now --dev
-          spack style -t black
+          spack -d style -t black
           spack unit-test -V
   import-check:
     runs-on: ubuntu-latest
@@ -98,14 +98,14 @@ jobs:
     # PR: use the base of the PR as the old commit
     - name: Checkout PR base commit
       if: github.event_name == 'pull_request'
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       with:
         ref: ${{ github.event.pull_request.base.sha }}
         path: old
     # not a PR: use the previous commit as the old commit
     - name: Checkout previous commit
       if: github.event_name != 'pull_request'
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       with:
         fetch-depth: 2
         path: old
@@ -114,11 +114,11 @@ jobs:
       run: git -C old reset --hard HEAD^
 
     - name: Checkout new commit
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       with:
         path: new
     - name: Install circular import checker
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
       with:
         repository: haampie/circular-import-fighter
         ref: 555519c6fd5564fd2eb844e7b87e84f4d12602e2

.readthedocs.yml

@@ -14,3 +14,26 @@ sphinx:
 python:
   install:
     - requirements: lib/spack/docs/requirements.txt
+
+search:
+  ranking:
+    spack.html: -10
+    spack.*.html: -10
+    llnl.html: -10
+    llnl.*.html: -10
+    _modules/*: -10
+    command_index.html: -9
+    basic_usage.html: 5
+    configuration.html: 5
+    config_yaml.html: 5
+    packages_yaml.html: 5
+    build_settings.html: 5
+    environments.html: 5
+    containers.html: 5
+    mirrors.html: 5
+    module_file_support.html: 5
+    repositories.html: 5
+    binary_caches.html: 5
+    chain.html: 5
+    pipelines.html: 5
+    packaging_guide.html: 5

CHANGELOG.md

@@ -1,3 +1,64 @@
+# v0.22.2 (2024-09-21)
+
+## Bugfixes
+- Forward compatibility with Spack 0.23 packages with language dependencies (#45205, #45191)
+- Forward compatibility with `urllib` from Python 3.12.6+ (#46453, #46483)
+- Bump vendored `archspec` for better aarch64 support (#45721, #46445)
+- Support macOS Sequoia (#45018, #45127)
+- Fix regression in `{variants.X}` and `{variants.X.value}` format strings (#46206)
+- Ensure shell escaping of environment variable values in load and activate commands (#42780)
+- Fix an issue where `spec[pkg]` considers specs outside the current DAG (#45090)
+- Do not halt concretization on unknown variants in externals (#45326)
+- Improve validation of `develop` config section (#46485)
+- Explicitly disable `ccache` if turned off in config, to avoid cache pollution (#45275)
+- Improve backwards compatibility in `include_concrete` (#45766)
+- Fix issue where package tags were sometimes repeated (#45160)
+- Make `setup-env.sh` "sourced only" by dropping execution bits (#45641)
+- Make certain source/binary fetch errors recoverable instead of a hard error (#45683)
+- Remove debug statements in package hash computation (#45235)
+- Remove redundant clingo warnings (#45269)
+- Remove hard-coded layout version (#45645)
+- Do not initialize previous store state in `use_store` (#45268)
+- Docs improvements (#46475)
+
+## Package updates
+- `chapel` major update (#42197, #44931, #45304)
+
+# v0.22.1 (2024-07-04)
+
+## Bugfixes
+- Fix reuse of externals on Linux (#44316)
+- Ensure parent gcc-runtime version >= child (#44834, #44870)
+- Ensure the latest gcc-runtime is rpath'ed when multiple exist among link deps (#44219)
+- Improve version detection of glibc (#44154)
+- Improve heuristics for solver (#44893, #44976, #45023)
+- Make strong preferences override reuse (#44373)
+- Reduce verbosity when C compiler is missing (#44182)
+- Make missing ccache executable an error when required (#44740)
+- Make every environment view containing `python` a `venv` (#44382)
+- Fix external detection for compilers with os but no target (#44156)
+- Fix version optimization for roots (#44272)
+- Handle common implementations of pagination of tags in OCI build caches (#43136)
+- Apply fetched patches to develop specs (#44950)
+- Avoid Windows wrappers for filesystem utilities on non-Windows (#44126)
+- Fix issue with long filenames in build caches on Windows (#43851)
+- Fix formatting issue in `spack audit` (#45045)
+- CI fixes (#44582, #43965, #43967, #44279, #44213)
+
+## Package updates
+- protobuf: fix 3.4:3.21 patch checksum (#44443)
+- protobuf: update hash for patch needed when="@3.4:3.21" (#44210)
+- git: bump v2.39 to 2.45; deprecate unsafe versions (#44248)
+- gcc: use -rpath {rpath_dir} not -rpath={rpath dir} (#44315)
+- Remove mesa18 and libosmesa (#44264)
+- Enforce consistency of `gl` providers (#44307)
+- Require libiconv for iconv (#44335, #45026).
+  Notice that glibc/musl also provide iconv, but are not guaranteed to be
+  complete. Set `packages:iconv:require:[glibc]` to restore the old behavior.
+- py-matplotlib: qualify when to do a post install (#44191)
+- rust: fix v1.78.0 instructions (#44127)
+- suite-sparse: improve setting of the `libs` property (#44214)
+- netlib-lapack: provide blas and lapack together (#44981)
 
 # v0.22.0 (2024-05-12)
 
@@ -319,6 +380,16 @@
     * 344 committers to packages
     * 45 committers to core
 
+# v0.21.3 (2024-10-02)
+
+## Bugfixes
+- Forward compatibility with Spack 0.23 packages with language dependencies (#45205, #45191)
+- Forward compatibility with `urllib` from Python 3.12.6+ (#46453, #46483)
+- Bump `archspec` to 0.2.5-dev for better aarch64 and Windows support (#42854, #44005,
+  #45721, #46445)
+- Support macOS Sequoia (#45018, #45127, #43862)
+- CI and test maintenance (#42909, #42728, #46711, #41943, #43363)
+
 # v0.21.2 (2024-03-01)
 
 ## Bugfixes

README.md

@@ -46,13 +46,18 @@ See the
 [Feature Overview](https://spack.readthedocs.io/en/latest/features.html)
 for examples and highlights.
 
-To install spack and your first package, make sure you have Python.
+To install spack and your first package, make sure you have Python & Git.
 Then:
 
-    $ git clone -c feature.manyFiles=true https://github.com/spack/spack.git
+    $ git clone -c feature.manyFiles=true --depth=2 https://github.com/spack/spack.git
     $ cd spack/bin
     $ ./spack install zlib
 
+> [!TIP]
+> `-c feature.manyFiles=true` improves git's performance on repositories with 1,000+ files.
+>
+> `--depth=2` prunes the git history to reduce the size of the Spack installation.
+
 Documentation
 ----------------
 

lib/spack/docs/basic_usage.rst

@@ -1175,6 +1175,17 @@ unspecified version, but packages can depend on other packages with
 could depend on ``mpich@1.2:`` if it can only build with version
 ``1.2`` or higher of ``mpich``.
 
+.. note:: Windows Spec Syntax Caveats
+   Windows has a few idiosyncrasies when it comes to the Spack spec syntax and the use of certain shells
+   Spack's spec dependency syntax uses the carat (``^``) character, however this is an escape string in CMD
+   so it must be escaped with an additional carat (i.e. ``^^``).
+   CMD also will attempt to interpret strings with ``=`` characters in them. Any spec including this symbol
+   must double quote the string.
+
+   Note: All of these issues are unique to CMD, they can be avoided by using Powershell.
+
+   For more context on these caveats see the related issues: `carat <https://github.com/spack/spack/issues/42833>`_ and `equals <https://github.com/spack/spack/issues/43348>`_
+
 Below are more details about the specifiers that you can add to specs.
 
 .. _version-specifier:

lib/spack/docs/build_settings.rst

@@ -166,3 +166,74 @@ while `py-numpy` still needs an older version:
 
 Up to Spack v0.20 ``duplicates:strategy:none`` was the default (and only) behavior. From Spack v0.21 the
 default behavior is ``duplicates:strategy:minimal``.
+
+--------
+Splicing
+--------
+
+The ``splice`` key covers config attributes for splicing specs in the solver.
+
+"Splicing" is a method for replacing a dependency with another spec
+that provides the same package or virtual. There are two types of
+splices, referring to different behaviors for shared dependencies
+between the root spec and the new spec replacing a dependency:
+"transitive" and "intransitive". A "transitive" splice is one that
+resolves all conflicts by taking the dependency from the new node. An
+"intransitive" splice is one that resolves all conflicts by taking the
+dependency from the original root. From a theory perspective, hybrid
+splices are possible but are not modeled by Spack.
+
+All spliced specs retain a ``build_spec`` attribute that points to the
+original Spec before any splice occurred. The ``build_spec`` for a
+non-spliced spec is itself.
+
+The figure below shows examples of transitive and intransitive splices:
+
+.. figure:: images/splices.png
+   :align: center
+
+The concretizer can be configured to explicitly splice particular
+replacements for a target spec. Splicing will allow the user to make
+use of generically built public binary caches, while swapping in
+highly optimized local builds for performance critical components
+and/or components that interact closely with the specific hardware
+details of the system. The most prominent candidate for splicing is
+MPI providers. MPI packages have relatively well-understood ABI
+characteristics, and most High Performance Computing facilities deploy
+highly optimized MPI packages tailored to their particular
+hardware. The following config block configures Spack to replace
+whatever MPI provider each spec was concretized to use with the
+particular package of ``mpich`` with the hash that begins ``abcdef``.
+
+.. code-block:: yaml
+
+   concretizer:
+     splice:
+       explicit:
+       - target: mpi
+         replacement: mpich/abcdef
+         transitive: false
+
+.. warning::
+
+   When configuring an explicit splice, you as the user take on the
+   responsibility for ensuring ABI compatibility between the specs
+   matched by the target and the replacement you provide. If they are
+   not compatible, Spack will not warn you and your application will
+   fail to run.
+
+The ``target`` field of an explicit splice can be any abstract
+spec. The ``replacement`` field must be a spec that includes the hash
+of a concrete spec, and the replacement must either be the same
+package as the target, provide the virtual that is the target, or
+provide a virtual that the target provides. The ``transitive`` field
+is optional -- by default, splices will be transitive.
+
+.. note::
+
+   With explicit splices configured, it is possible for Spack to
+   concretize to a spec that does not satisfy the input. For example,
+   with the config above ``hdf5 ^mvapich2`` will concretize to user
+   ``mpich/abcdef`` instead of ``mvapich2`` as the MPI provider. Spack
+   will warn the user in this case, but will not fail the
+   concretization.

lib/spack/docs/build_systems/custompackage.rst

@@ -130,14 +130,19 @@ before or after a particular phase. For example, in ``perl``, we see:
 
    @run_after("install")
    def install_cpanm(self):
-       spec = self.spec
-
-       if spec.satisfies("+cpanm"):
-           with working_dir(join_path("cpanm", "cpanm")):
-               perl = spec["perl"].command
-               perl("Makefile.PL")
-               make()
-               make("install")
+        spec = self.spec
+        maker = make
+        cpan_dir = join_path("cpanm", "cpanm")
+        if sys.platform == "win32":
+            maker = nmake
+            cpan_dir = join_path(self.stage.source_path, cpan_dir)
+            cpan_dir = windows_sfn(cpan_dir)
+        if "+cpanm" in spec:
+            with working_dir(cpan_dir):
+                perl = spec["perl"].command
+                perl("Makefile.PL")
+                maker()
+                maker("install")
 
 This extra step automatically installs ``cpanm`` in addition to the
 base Perl installation.
@@ -176,8 +181,14 @@ In the ``perl`` package, we can see:
 
    @run_after("build")
    @on_package_attributes(run_tests=True)
-   def test(self):
-       make("test")
+   def build_test(self):
+        if sys.platform == "win32":
+            win32_dir = os.path.join(self.stage.source_path, "win32")
+            win32_dir = windows_sfn(win32_dir)
+            with working_dir(win32_dir):
+                nmake("test", ignore_quotes=True)
+        else:
+            make("test")
 
 As you can guess, this runs ``make test`` *after* building the package,
 if and only if testing is requested. Again, this is not specific to

lib/spack/docs/build_systems/sconspackage.rst

@@ -49,14 +49,14 @@ following phases:
 #. ``install`` - install the package
 
 Package developers often add unit tests that can be invoked with
-``scons test`` or ``scons check``. Spack provides a ``test`` method
+``scons test`` or ``scons check``. Spack provides a ``build_test`` method
 to handle this. Since we don't know which one the package developer
-chose, the ``test`` method does nothing by default, but can be easily
+chose, the ``build_test`` method does nothing by default, but can be easily
 overridden like so:
 
 .. code-block:: python
 
-   def test(self):
+   def build_test(self):
        scons("check")
 
 

lib/spack/docs/conf.py

@@ -219,6 +219,9 @@ def setup(sphinx):
     ("py:class", "spack.install_test.Pb"),
     ("py:class", "spack.filesystem_view.SimpleFilesystemView"),
     ("py:class", "spack.traverse.EdgeAndDepth"),
+    ("py:class", "archspec.cpu.microarchitecture.Microarchitecture"),
+    # TypeVar that is not handled correctly
+    ("py:class", "llnl.util.lang.T"),
 ]
 
 # The reST default role (used for this markup: `text`) to use for all documents.

lib/spack/docs/configuration.rst

@@ -281,7 +281,7 @@ When spack queries for configuration parameters, it searches in
 higher-precedence scopes first. So, settings in a higher-precedence file
 can override those with the same key in a lower-precedence one. For
 list-valued settings, Spack *prepends* higher-precedence settings to
-lower-precedence settings. Completely ignoring higher-level configuration
+lower-precedence settings. Completely ignoring lower-precedence configuration
 options is supported with the ``::`` notation for keys (see
 :ref:`config-overrides` below).
 

lib/spack/docs/contribution_guide.rst

@@ -316,6 +316,215 @@ documentation tests to make sure there are no errors. Documentation changes can
 in some obfuscated warning messages. If you don't understand what they mean, feel free
 to ask when you submit your PR.
 
+.. _spack-builders-and-pipelines:
+
+^^^^^^^^^
+GitLab CI
+^^^^^^^^^
+
+""""""""""""""""""
+Build Cache Stacks
+""""""""""""""""""
+
+Spack welcomes the contribution of software stacks of interest to the community. These
+stacks are used to test package recipes and generate publicly available build caches.
+Spack uses GitLab CI for managing the orchestration of build jobs.
+
+GitLab Entry Point
+~~~~~~~~~~~~~~~~~~
+
+Add stack entrypoint to the ``share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml``. There
+are two stages required for each new stack, the generation stage and the build stage.
+
+The generate stage is defined using the job template ``.generate`` configured with
+environment variables defining the name of the stack in ``SPACK_CI_STACK_NAME`` and the
+platform (``SPACK_TARGET_PLATFORM``) and architecture (``SPACK_TARGET_ARCH``) configuration,
+and the tags associated with the class of runners to build on.
+
+.. note::
+
+    The ``SPACK_CI_STACK_NAME`` must match the name of the directory containing the
+    stacks ``spack.yaml``.
+
+
+.. note::
+
+    The platform and architecture variables are specified in order to select the
+    correct configurations from the generic configurations used in Spack CI. The
+    configurations currently available are:
+
+    * ``.cray_rhel_zen4``
+    * ``.cray_sles_zen4``
+    * ``.darwin_aarch64``
+    * ``.darwin_x86_64``
+    * ``.linux_aarch64``
+    * ``.linux_icelake``
+    * ``.linux_neoverse_n1``
+    * ``.linux_neoverse_v1``
+    * ``.linux_neoverse_v2``
+    * ``.linux_power``
+    * ``.linux_skylake``
+    * ``.linux_x86_64``
+    * ``.linux_x86_64_v4``
+
+    New configurations can be added to accommodate new platforms and architectures.
+
+
+The build stage is defined as a trigger job that consumes the GitLab CI pipeline generated in
+the generate stage for this stack. Build stage jobs use the ``.build`` job template which
+handles the basic configuration.
+
+An example entry point for a new stack called ``my-super-cool-stack``
+
+.. code-block:: yaml
+
+    .my-super-cool-stack:
+      extends: [ ".linux_x86_64_v3" ]
+      variables:
+        SPACK_CI_STACK_NAME: my-super-cool-stack
+        tags: [ "all", "tags", "your", "job", "needs"]
+
+    my-super-cool-stack-generate:
+      extends: [ ".generate", ".my-super-cool-stack" ]
+      image: my-super-cool-stack-image:0.0.1
+
+    my-super-cool-stack-build:
+      extends: [ ".build", ".my-super-cool-stack" ]
+      trigger:
+        include:
+          - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+            job: my-super-cool-stack-generate
+        strategy: depend
+      needs:
+        - artifacts: True
+          job: my-super-cool-stack-generate
+
+
+Stack Configuration
+~~~~~~~~~~~~~~~~~~~
+
+The stack configuration is a spack environment file with two additional sections added.
+Stack configurations should be located in ``share/spack/gitlab/cloud_pipelines/stacks/<stack_name>/spack.yaml``.
+
+The ``ci`` section is generally used to define stack specific mappings such as image or tags.
+For more information on what can go into the ``ci`` section refer to the docs on pipelines.
+
+The ``cdash`` section is used for defining where to upload the results of builds. Spack configures
+most of the details for posting pipeline results to
+`cdash.spack.io <https://cdash.spack.io/index.php?project=Spack+Testing>`_. The only
+requirement in the stack configuration is to define a ``build-group`` that is unique,
+this is usually the long name of the stack.
+
+An example stack that builds ``zlib``.
+
+.. code-block:: yaml
+
+    spack:
+      view: false
+      packages:
+        all:
+          require: ["%gcc", "target=x86_64_v3"]
+      specs:
+      - zlib
+
+      ci:
+        pipeline-gen
+        - build-job:
+            image: my-super-cool-stack-image:0.0.1
+
+      cdash:
+        build-group: My Super Cool Stack
+
+.. note::
+
+    The ``image`` used in the ``*-generate`` job must match exactly the ``image`` used in the ``build-job``.
+    When the images do not match the build job may fail.
+
+
+"""""""""""""""""""
+Registering Runners
+"""""""""""""""""""
+
+Contributing computational resources to Spack's CI build farm is one way to help expand the
+capabilities and offerings of the public Spack build caches. Currently, Spack utilizes linux runners
+from AWS, Google, and the University of Oregon (UO).
+
+Runners require three key peices:
+* Runner Registration Token
+* Accurate tags
+* OIDC Authentication script
+* GPG keys
+
+
+Minimum GitLab Runner Version: ``16.1.0``
+`Intallation instructions <https://docs.gitlab.com/runner/install/>`_
+
+Registration Token
+~~~~~~~~~~~~~~~~~~
+
+The first step to contribute new runners is to open an issue in the `spack infrastructure <https://github.com/spack/spack-infrastructure/issues/new?assignees=&labels=runner-registration&projects=&template=runner_registration.yml>`_
+project. This will be reported to the spack infrastructure team who will guide users through the process
+of registering new runners for Spack CI.
+
+The information needed to register a runner is the motivation for the new resources, a semi-detailed description of
+the runner, and finallly the point of contact for maintaining the software on the runner.
+
+The point of contact will then work with the infrastruture team to obtain runner registration token(s) for interacting with
+with Spack's GitLab instance. Once the runner is active, this point of contact will also be responsible for updating the
+GitLab runner software to keep pace with Spack's Gitlab.
+
+Tagging
+~~~~~~~
+
+In the initial stages of runner registration it is important to **exclude** the special tag ``spack``. This will prevent
+the new runner(s) from being picked up for production CI jobs while it is configured and evaluated. Once it is determined
+that the runner is ready for production use the ``spack`` tag will be added.
+
+Because gitlab has no concept of tag exclustion, runners that provide specialized resource also require specialized tags.
+For example, a basic CPU only x86_64 runner may have a tag ``x86_64`` associated with it. However, a runner containing an
+CUDA capable GPU may have the tag ``x86_64-cuda`` to denote that it should only be used for packages that will benefit from
+a CUDA capable resource.
+
+OIDC
+~~~~
+
+Spack runners use OIDC authentication for connecting to the appropriate AWS bucket
+which is used for coordinating the communication of binaries between build jobs. In
+order to configure OIDC authentication, Spack CI runners use a python script with minimal
+dependencies. This script can be configured for runners as seen here using the ``pre_build_script``.
+
+.. code-block:: toml
+
+    [[runners]]
+      pre_build_script = """
+      echo 'Executing Spack pre-build setup script'
+
+      for cmd in "${PY3:-}" python3 python; do
+        if command -v > /dev/null "$cmd"; then
+          export PY3="$(command -v "$cmd")"
+          break
+        fi
+      done
+
+      if [ -z "${PY3:-}" ]; then
+        echo "Unable to find python3 executable"
+        exit 1
+      fi
+
+      $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')"
+      $PY3 pre_build.py > envvars
+
+      . ./envvars
+      rm -f envvars
+      unset GITLAB_OIDC_TOKEN
+      """
+
+GPG Keys
+~~~~~~~~
+
+Runners that may be utilized for ``protected`` CI require the registration of an intermediate signing key that
+can be used to sign packages. For more information on package signing read :ref:`key_architecture`.
+
 --------
 Coverage
 --------

lib/spack/docs/developer_guide.rst

@@ -712,27 +712,27 @@ Release branches
 ^^^^^^^^^^^^^^^^
 
 There are currently two types of Spack releases: :ref:`major releases
-<major-releases>` (``0.17.0``, ``0.18.0``, etc.) and :ref:`point releases
-<point-releases>` (``0.17.1``, ``0.17.2``, ``0.17.3``, etc.). Here is a
+<major-releases>` (``0.21.0``, ``0.22.0``, etc.) and :ref:`patch releases
+<patch-releases>` (``0.22.1``, ``0.22.2``, ``0.22.3``, etc.). Here is a
 diagram of how Spack release branches work::
 
-    o    branch: develop  (latest version, v0.19.0.dev0)
+    o    branch: develop  (latest version, v0.23.0.dev0)
     |
     o
-    | o  branch: releases/v0.18, tag: v0.18.1
+    | o  branch: releases/v0.22, tag: v0.22.1
     o |
-    | o  tag: v0.18.0
+    | o  tag: v0.22.0
     o |
     | o
     |/
     o
     |
     o
-    | o  branch: releases/v0.17, tag: v0.17.2
+    | o  branch: releases/v0.21, tag: v0.21.2
     o |
-    | o  tag: v0.17.1
+    | o  tag: v0.21.1
     o |
-    | o  tag: v0.17.0
+    | o  tag: v0.21.0
     o |
     | o
     |/
@@ -743,8 +743,8 @@ requests target ``develop``. The ``develop`` branch will report that its
 version is that of the next **major** release with a ``.dev0`` suffix.
 
 Each Spack release series also has a corresponding branch, e.g.
-``releases/v0.18`` has ``0.18.x`` versions of Spack, and
-``releases/v0.17`` has ``0.17.x`` versions. A major release is the first
+``releases/v0.22`` has ``v0.22.x`` versions of Spack, and
+``releases/v0.21`` has ``v0.21.x`` versions. A major release is the first
 tagged version on a release branch. Minor releases are back-ported from
 develop onto release branches. This is typically done by cherry-picking
 bugfix commits off of ``develop``.
@@ -774,27 +774,40 @@ for more details.
 Scheduling work for releases
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-We schedule work for releases by creating `GitHub projects
-<https://github.com/spack/spack/projects>`_. At any time, there may be
-several open release projects. For example, below are two releases (from
-some past version of the page linked above):
+We schedule work for **major releases** through `milestones
+<https://github.com/spack/spack/milestones>`_ and `GitHub Projects
+<https://github.com/spack/spack/projects>`_, while **patch releases** use `labels
+<https://github.com/spack/spack/labels>`_.
 
-.. image:: images/projects.png
+There is only one milestone open at a time. Its name corresponds to the next major version, for
+example ``v0.23``. Important issues and pull requests should be assigned to this milestone by
+core developers, so that they are not forgotten at the time of release. The milestone is closed
+when the release is made, and a new milestone is created for the next major release.
 
-This image shows one release in progress for ``0.15.1`` and another for
-``0.16.0``. Each of these releases has a project board containing issues
-and pull requests. GitHub shows a status bar with completed work in
-green, work in progress in purple, and work not started yet in gray, so
-it's fairly easy to see progress.
+Bug reports in GitHub issues are automatically labelled ``bug`` and ``triage``. Spack developers
+assign one of the labels ``impact-low``, ``impact-medium`` or ``impact-high``. This will make the
+issue appear in the `Triaged bugs <https://github.com/orgs/spack/projects/6>`_ project board.
+Important issues should be assigned to the next milestone as well, so they appear at the top of
+the project board.
 
-Spack's project boards are not firm commitments so we move work between
-releases frequently. If we need to make a release and some tasks are not
-yet done, we will simply move them to the next minor or major release, rather
-than delaying the release to complete them.
+Spack's milestones are not firm commitments so we move work between releases frequently. If we
+need to make a release and some tasks are not yet done, we will simply move them to the next major
+release milestone, rather than delaying the release to complete them.
 
-For more on using GitHub project boards, see `GitHub's documentation
-<https://docs.github.com/en/github/managing-your-work-on-github/about-project-boards>`_.
+^^^^^^^^^^^^^^^^^^^^^
+Backporting bug fixes
+^^^^^^^^^^^^^^^^^^^^^
 
+When a bug is fixed in the ``develop`` branch, it is often necessary to backport the fix to one
+(or more) of the ``release/vX.Y`` branches. Only the release manager is responsible for doing
+backports, but Spack maintainers are responsible for labelling pull requests (and issues if no bug
+fix is available yet) with ``vX.Y.Z`` labels. The label should correspond to the next patch version
+that the bug fix should be backported to.
+
+Backports are done publicly by the release manager using a pull request named ``Backports vX.Y.Z``.
+This pull request is opened from the ``backports/vX.Y.Z`` branch, targets the ``releases/vX.Y``
+branch and contains a (growing) list of cherry-picked commits from the ``develop`` branch.
+Typically there are one or two backport pull requests open at any given time.
 
 .. _major-releases:
 
@@ -802,25 +815,21 @@ For more on using GitHub project boards, see `GitHub's documentation
 Making major releases
 ^^^^^^^^^^^^^^^^^^^^^
 
-Assuming a project board has already been created and all required work
-completed, the steps to make the major release are:
+Assuming all required work from the milestone is completed, the steps to make the major release
+are:
 
-#. Create two new project boards:
+#. `Create a new milestone <https://github.com/spack/spack/milestones>`_ for the next major
+   release.
 
-   * One for the next major release
-   * One for the next point release
+#. `Create a new label <https://github.com/spack/spack/labels>`_ for the next patch release.
 
-#. Move any optional tasks that are not done to one of the new project boards.
-
-   In general, small bugfixes should go to the next point release. Major
-   features, refactors, and changes that could affect concretization should
-   go in the next major release.
+#. Move any optional tasks that are not done to the next milestone.
 
 #. Create a branch for the release, based on ``develop``:
 
    .. code-block:: console
 
-      $ git checkout -b releases/v0.15 develop
+      $ git checkout -b releases/v0.23 develop
 
    For a version ``vX.Y.Z``, the branch's name should be
    ``releases/vX.Y``. That is, you should create a ``releases/vX.Y``
@@ -856,8 +865,8 @@ completed, the steps to make the major release are:
 
    Create a pull request targeting the ``develop`` branch, bumping the major
    version in ``lib/spack/spack/__init__.py`` with a ``dev0`` release segment.
-   For instance when you have just released ``v0.15.0``, set the version
-   to ``(0, 16, 0, 'dev0')`` on ``develop``.
+   For instance when you have just released ``v0.23.0``, set the version
+   to ``(0, 24, 0, 'dev0')`` on ``develop``.
 
 #. Follow the steps in :ref:`publishing-releases`.
 
@@ -866,82 +875,52 @@ completed, the steps to make the major release are:
 #. Follow the steps in :ref:`announcing-releases`.
 
 
-.. _point-releases:
+.. _patch-releases:
 
 ^^^^^^^^^^^^^^^^^^^^^
-Making point releases
+Making patch releases
 ^^^^^^^^^^^^^^^^^^^^^
 
-Assuming a project board has already been created and all required work
-completed, the steps to make the point release are:
+To make the patch release process both efficient and transparent, we use a *backports pull request*
+which contains cherry-picked commits from the ``develop`` branch. The majority of the work is to
+cherry-pick the bug fixes, which ideally should be done as soon as they land on ``develop``:
+this ensures cherry-picking happens in order, and makes conflicts easier to resolve since the
+changes are fresh in the mind of the developer.
 
-#. Create a new project board for the next point release.
+The backports pull request is always titled ``Backports vX.Y.Z`` and is labelled ``backports``. It
+is opened from a branch named ``backports/vX.Y.Z`` and targets the ``releases/vX.Y`` branch.
 
-#. Move any optional tasks that are not done to the next project board.
+Whenever a pull request labelled ``vX.Y.Z`` is merged, cherry-pick the associated squashed commit
+on ``develop`` to the ``backports/vX.Y.Z`` branch. For pull requests that were rebased (or not
+squashed), cherry-pick each associated commit individually. Never force push to the
+``backports/vX.Y.Z`` branch.
 
-#. Check out the release branch (it should already exist).
+.. warning::
 
-    For the ``X.Y.Z`` release, the release branch is called ``releases/vX.Y``.
-    For ``v0.15.1``, you would check out ``releases/v0.15``:
+   Sometimes you may **still** get merge conflicts even if you have
+   cherry-picked all the commits in order. This generally means there
+   is some other intervening pull request that the one you're trying
+   to pick depends on. In these cases, you'll need to make a judgment
+   call regarding those pull requests.  Consider the number of affected
+   files and/or the resulting differences.
 
-   .. code-block:: console
+   1. If the changes are small, you might just cherry-pick it.
 
-      $ git checkout releases/v0.15
+   2. If the changes are large, then you may decide that this fix is not
+      worth including in a patch release, in which case you should remove
+      the label from the pull request. Remember that large, manual backports
+      are seldom the right choice for a patch release.
 
-#. If a pull request to the release branch named ``Backports vX.Y.Z`` is not already
-   in the project, create it. This pull request ought to be created as early as
-   possible when working on a release project, so that we can build the release
-   commits incrementally, and identify potential conflicts at an early stage.
+When all commits are cherry-picked in the ``backports/vX.Y.Z`` branch, make the patch
+release as follows:
 
-#. Cherry-pick each pull request in the ``Done`` column of the release
-   project board onto the ``Backports vX.Y.Z`` pull request.
+#. `Create a new label <https://github.com/spack/spack/labels>`_ ``vX.Y.{Z+1}`` for the next patch
+   release.
 
-   This is **usually** fairly simple since we squash the commits from the
-   vast majority of pull requests. That means there is only one commit
-   per pull request to cherry-pick. For example, `this pull request
-   <https://github.com/spack/spack/pull/15777>`_ has three commits, but
-   they were squashed into a single commit on merge. You can see the
-   commit that was created here:
+#. Replace the label ``vX.Y.Z`` with ``vX.Y.{Z+1}`` for all PRs and issues that are not done.
 
-   .. image:: images/pr-commit.png
-
-   You can easily cherry pick it like this (assuming you already have the
-   release branch checked out):
-
-   .. code-block:: console
-
-      $ git cherry-pick 7e46da7
-
-   For pull requests that were rebased (or not squashed), you'll need to
-   cherry-pick each associated commit individually.
-
-   .. warning::
-
-      It is important to cherry-pick commits in the order they happened,
-      otherwise you can get conflicts while cherry-picking. When
-      cherry-picking look at the merge date,
-      **not** the number of the pull request or the date it was opened.
-
-      Sometimes you may **still** get merge conflicts even if you have
-      cherry-picked all the commits in order. This generally means there
-      is some other intervening pull request that the one you're trying
-      to pick depends on. In these cases, you'll need to make a judgment
-      call regarding those pull requests.  Consider the number of affected
-      files and or the resulting differences.
-
-      1. If the dependency changes are small, you might just cherry-pick it,
-         too. If you do this, add the task to the release board.
-
-      2. If the changes are large, then you may decide that this fix is not
-         worth including in a point release, in which case you should remove
-         the task from the release project.
-
-      3. You can always decide to manually back-port the fix to the release
-         branch if neither of the above options makes sense, but this can
-         require a lot of work. It's seldom the right choice.
-
-#. When all the commits from the project board are cherry-picked into
-   the ``Backports vX.Y.Z`` pull request, you can push a commit to:
+#. Manually push a single commit with commit message ``Set version to vX.Y.Z`` to the
+   ``backports/vX.Y.Z`` branch, that both bumps the Spack version number and updates the changelog:
 
    1. Bump the version in ``lib/spack/spack/__init__.py``.
    2. Update ``CHANGELOG.md`` with a list of the changes.
@@ -950,20 +929,22 @@ completed, the steps to make the point release are:
    release branch. See `the changelog from 0.14.1
    <https://github.com/spack/spack/commit/ff0abb9838121522321df2a054d18e54b566b44a>`_.
 
-#. Merge the ``Backports vX.Y.Z`` PR with the **Rebase and merge** strategy. This
-   is needed to keep track in the release branch of all the commits that were
-   cherry-picked.
-
-#. Make sure CI passes on the release branch, including:
+#. Make sure CI passes on the **backports pull request**, including:
 
    * Regular unit tests
    * Build tests
    * The E4S pipeline at `gitlab.spack.io <https://gitlab.spack.io>`_
 
-   If CI does not pass, you'll need to figure out why, and make changes
-   to the release branch until it does. You can make more commits, modify
-   or remove cherry-picked commits, or cherry-pick **more** from
-   ``develop`` to make this happen.
+#. Merge the ``Backports vX.Y.Z`` PR with the **Rebase and merge** strategy. This
+   is needed to keep track in the release branch of all the commits that were
+   cherry-picked.
+
+#. Make sure CI passes on the last commit of the **release branch**.
+
+#. In the rare case you need to include additional commits in the patch release after the backports
+   PR is merged, it is best to delete the last commit ``Set version to vX.Y.Z`` from the release
+   branch with a single force push, open a new backports PR named ``Backports vX.Y.Z (2)``, and
+   repeat the process. Avoid repeated force pushes to the release branch.
 
 #. Follow the steps in :ref:`publishing-releases`.
 
@@ -1038,25 +1019,31 @@ Updating `releases/latest`
 
 If the new release is the **highest** Spack release yet, you should
 also tag it as ``releases/latest``. For example, suppose the highest
-release is currently ``0.15.3``:
+release is currently ``0.22.3``:
 
-* If you are releasing ``0.15.4`` or ``0.16.0``, then you should tag
-  it with ``releases/latest``, as these are higher than ``0.15.3``.
+* If you are releasing ``0.22.4`` or ``0.23.0``, then you should tag
+  it with ``releases/latest``, as these are higher than ``0.22.3``.
 
 * If you are making a new release of an **older** major version of
-  Spack, e.g. ``0.14.4``, then you should not tag it as
+  Spack, e.g. ``0.21.4``, then you should not tag it as
   ``releases/latest`` (as there are newer major versions).
 
-To tag ``releases/latest``, do this:
+To do so, first fetch the latest tag created on GitHub, since you may not have it locally:
 
 .. code-block:: console
 
-   $ git checkout releases/vX.Y     # vX.Y is the new release's branch
-   $ git tag --force releases/latest
-   $ git push --force --tags
+   $ git fetch --force git@github.com:spack/spack vX.Y.Z
 
-The ``--force`` argument to ``git tag`` makes ``git`` overwrite the existing
-``releases/latest`` tag with the new one.
+Then tag ``vX.Y.Z`` as ``releases/latest`` and push the individual tag to GitHub.
+
+.. code-block:: console
+
+   $ git tag --force releases/latest vX.Y.Z
+   $ git push --force git@github.com:spack/spack releases/latest
+
+The ``--force`` argument to ``git tag`` makes ``git`` overwrite the existing ``releases/latest``
+tag with the new one. Do **not** use the ``--tags`` flag when pushing, since this will push *all*
+local tags.
 
 
 .. _announcing-releases:

lib/spack/docs/environments.rst

@@ -5,49 +5,56 @@
 
 .. _environments:
 
-=========================
-Environments (spack.yaml)
-=========================
+=====================================
+Environments (spack.yaml, spack.lock)
+=====================================
 
-An environment is used to group together a set of specs for the
-purpose of building, rebuilding and deploying in a coherent fashion.
-Environments provide a number of advantages over the *à la carte*
-approach of building and loading individual Spack modules:
+An environment is used to group a set of specs intended for some purpose
+to be built, rebuilt, and deployed in a coherent fashion. Environments
+define aspects of the installation of the software, such as:
 
-#. Environments separate the steps of (a) choosing what to
-   install, (b) concretizing, and (c) installing.  This allows
-   Environments to remain stable and repeatable, even if Spack packages
-   are upgraded: specs are only re-concretized when the user
-   explicitly asks for it.  It is even possible to reliably
-   transport environments between different computers running
-   different versions of Spack!
-#. Environments allow several specs to be built at once; a more robust
-   solution than ad-hoc scripts making multiple calls to ``spack
-   install``.
-#. An Environment that is built as a whole can be loaded as a whole
-   into the user environment. An Environment can be built to maintain
-   a filesystem view of its packages, and the environment can load
-   that view into the user environment at activation time. Spack can
-   also generate a script to load all modules related to an
-   environment.
+#. *which* specs to install;
+#. *how* those specs are configured; and
+#. *where* the concretized software will be installed.
+
+Aggregating this information into an environment for processing has advantages
+over the *à la carte* approach of building and loading individual Spack modules.
+
+With environments, you concretize, install, or load (activate) all of the
+specs with a single command. Concretization fully configures the specs
+and dependencies of the environment in preparation for installing the
+software. This is a more robust solution than ad-hoc installation scripts.
+And you can share an environment or even re-use it on a different computer.
+
+Environment definitions, especially *how* specs are configured, allow the
+software to remain stable and repeatable even when Spack packages are upgraded. Changes are only picked up when the environment is explicitly re-concretized.
+
+Defining *where* specs are installed supports a filesystem view of the
+environment. Yet Spack maintains a single installation of the software that
+can be re-used across multiple environments.
+
+Activating an environment determines *when* all of the associated (and
+installed) specs are loaded so limits the software loaded to those specs
+actually needed by the environment. Spack can even generate a script to
+load all modules related to an environment.
 
 Other packaging systems also provide environments that are similar in
 some ways to Spack environments; for example, `Conda environments
 <https://conda.io/docs/user-guide/tasks/manage-environments.html>`_ or
 `Python Virtual Environments
 <https://docs.python.org/3/tutorial/venv.html>`_.  Spack environments
-provide some distinctive features:
+provide some distinctive features though:
 
 #. A spec installed "in" an environment is no different from the same
-   spec installed anywhere else in Spack.  Environments are assembled
-   simply by collecting together a set of specs.
-#. Spack Environments may contain more than one spec of the same
+   spec installed anywhere else in Spack.
+#. Spack environments may contain more than one spec of the same
    package.
 
 Spack uses a "manifest and lock" model similar to `Bundler gemfiles
-<https://bundler.io/man/gemfile.5.html>`_ and other package
-managers. The user input file is named ``spack.yaml`` and the lock
-file is named ``spack.lock``
+<https://bundler.io/man/gemfile.5.html>`_ and other package managers.
+The environment's user input file (or manifest), is named ``spack.yaml``.
+The lock file, which contains the fully configured and concretized specs,
+is named ``spack.lock``.
 
 .. _environments-using:
 
@@ -68,55 +75,60 @@ An environment is created by:
 
    $ spack env create myenv
 
-Spack then creates the directory ``var/spack/environments/myenv``.
+The directory ``$SPACK_ROOT/var/spack/environments/myenv`` is created
+to manage the environment.
 
 .. note::
 
-   All managed environments by default are stored in the ``var/spack/environments`` folder.
-   This location can be changed by setting the ``environments_root`` variable in ``config.yaml``.
+   All managed environments by default are stored in the
+   ``$SPACK_ROOT/var/spack/environments`` folder. This location can be changed
+   by setting the ``environments_root`` variable in ``config.yaml``.
 
-In the ``var/spack/environments/myenv`` directory, Spack creates the
-file ``spack.yaml`` and the hidden directory ``.spack-env``.
-
-Spack stores metadata in the ``.spack-env`` directory. User
-interaction will occur through the ``spack.yaml`` file and the Spack
-commands that affect it. When the environment is concretized, Spack
-will create a file ``spack.lock`` with the concrete information for
+Spack creates the file ``spack.yaml``, hidden directory ``.spack-env``, and
+``spack.lock`` file under ``$SPACK_ROOT/var/spack/environments/myenv``. User
+interaction occurs through the ``spack.yaml`` file and the Spack commands
+that affect it. Metadata and, by default, the view are stored in the
+``.spack-env`` directory. When the environment is concretized, Spack creates
+the ``spack.lock`` file with the fully configured specs and dependencies for
 the environment.
 
-In addition to being the default location for the view associated with
-an Environment, the ``.spack-env`` directory also contains:
+The ``.spack-env`` subdirectory also contains:
 
-  * ``repo/``: A repo consisting of the Spack packages used in this
-    environment.  This allows the environment to build the same, in
-    theory, even on different versions of Spack with different
+  * ``repo/``: A subdirectory acting as the repo consisting of the Spack
+    packages used in the environment. It allows the environment to build
+    the same, in theory, even on different versions of Spack with different
     packages!
-  * ``logs/``: A directory containing the build logs for the packages
-    in this Environment.
+  * ``logs/``: A subdirectory containing the build logs for the packages
+    in this environment.
 
-Spack Environments can also be created from either a manifest file
-(usually but not necessarily named, ``spack.yaml``) or a lockfile.
-To create an Environment from a manifest:
+Spack Environments can also be created from either the user input, or
+manifest, file or the lockfile. Create an environment from a manifest using:
 
 .. code-block:: console
 
    $ spack env create myenv spack.yaml
 
-To create an Environment from a ``spack.lock`` lockfile:
+The resulting environment is guaranteed to have the same root specs as
+the original but may concretize differently in the presence of different
+explicit or default configuration settings (e.g., a different version of
+Spack or for a different user account).
+
+Create an environment from a ``spack.lock`` file using:
 
 .. code-block:: console
 
    $ spack env create myenv spack.lock
 
-Either of these commands can also take a full path to the
-initialization file.
+The resulting environment, when on the same or a compatible machine, is
+guaranteed to initially have the same concrete specs as the original.
 
-A Spack Environment created from a ``spack.yaml`` manifest is
-guaranteed to have the same root specs as the original Environment,
-but may concretize differently. A Spack Environment created from a
-``spack.lock`` lockfile is guaranteed to have the same concrete specs
-as the original Environment. Either may obviously then differ as the
-user modifies it.
+.. note::
+
+   Environment creation also accepts a full path to the file.
+
+   If the path is not under the ``$SPACK_ROOT/var/spack/environments``
+   directory then the source is referred to as an
+   :ref:`independent environment <independent_environments>`.
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 Activating an Environment
@@ -129,7 +141,7 @@ To activate an environment, use the following command:
    $ spack env activate myenv
 
 By default, the ``spack env activate`` will load the view associated
-with the Environment into the user environment. The ``-v,
+with the environment into the user environment. The ``-v,
 --with-view`` argument ensures this behavior, and the ``-V,
 --without-view`` argument activates the environment without changing
 the user environment variables.
@@ -142,8 +154,11 @@ user's prompt to begin with the environment name in brackets.
    $ spack env activate -p myenv
    [myenv] $ ...
 
-The ``activate`` command can also be used to create a new environment if it does not already
-exist.
+The ``activate`` command can also be used to create a new environment, if it is
+not already defined, by adding the ``--create`` flag. Managed and independent
+environments can both be created using the same flags that `spack env create`
+accepts.  If an environment already exists then spack will simply activate it
+and ignore the create-specific flags.
 
 .. code-block:: console
    
@@ -168,49 +183,50 @@ or the shortcut alias
 If the environment was activated with its view, deactivating the
 environment will remove the view from the user environment.
 
-^^^^^^^^^^^^^^^^^^^^^^
-Anonymous Environments
-^^^^^^^^^^^^^^^^^^^^^^
+.. _independent_environments:
 
-Apart from managed environments, Spack also supports anonymous environments.
+^^^^^^^^^^^^^^^^^^^^^^^^
+Independent Environments
+^^^^^^^^^^^^^^^^^^^^^^^^
 
-Anonymous environments can be placed in any directory of choice.
+Independent environments can be located in any directory outside of Spack.
 
 .. note::
 
    When uninstalling packages, Spack asks the user to confirm the removal of packages
-   that are still used in a managed environment. This is not the case for anonymous
+   that are still used in a managed environment. This is not the case for independent
    environments.
 
-To create an anonymous environment, use one of the following commands:
+To create an independent environment, use one of the following commands:
 
 .. code-block:: console
 
    $ spack env create --dir my_env
    $ spack env create ./my_env
 
-As a shorthand, you can also create an anonymous environment upon activation if it does not
+As a shorthand, you can also create an independent environment upon activation if it does not
 already exist:
 
 .. code-block:: console
 
    $ spack env activate --create ./my_env
 
-For convenience, Spack can also place an anonymous environment in a temporary directory for you:
+For convenience, Spack can also place an independent environment in a temporary directory for you:
 
 .. code-block:: console
 
    $ spack env activate --temp
 
 
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Environment Sensitive Commands
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+Environment-Aware Commands
+^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Spack commands are environment sensitive. For example, the ``find``
-command shows only the specs in the active Environment if an
-Environment has been activated. Similarly, the ``install`` and
-``uninstall`` commands act on the active environment.
+Spack commands are environment-aware. For example, the ``find``
+command shows only the specs in the active environment if an
+environment has been activated. Otherwise it shows all specs in
+the Spack instance. The same rule applies to the ``install`` and
+``uninstall`` commands.
 
 .. code-block:: console
 
@@ -255,32 +271,33 @@ Environment has been activated. Similarly, the ``install`` and
 
 
 Note that when we installed the abstract spec ``zlib@1.2.8``, it was
-presented as a root of the Environment. All explicitly installed
-packages will be listed as roots of the Environment.
+presented as a root of the environment. All explicitly installed
+packages will be listed as roots of the environment.
 
 All of the Spack commands that act on the list of installed specs are
-Environment-sensitive in this way, including ``install``,
-``uninstall``, ``find``, ``extensions``, and more. In the
+environment-aware in this way, including ``install``,
+``uninstall``, ``find``, ``extensions``, etcetera. In the
 :ref:`environment-configuration` section we will discuss
-Environment-sensitive commands further.
+environment-aware commands further.
 
 ^^^^^^^^^^^^^^^^^^^^^
 Adding Abstract Specs
 ^^^^^^^^^^^^^^^^^^^^^
 
-An abstract spec is the user-specified spec before Spack has applied
-any defaults or dependency information.
+An abstract spec is the user-specified spec before Spack applies
+defaults or dependency information.
 
-Users can add abstract specs to an Environment using the ``spack add``
-command. The most important component of an Environment is a list of
+Users can add abstract specs to an environment using the ``spack add``
+command. The most important component of an environment is a list of
 abstract specs.
 
-Adding a spec adds to the manifest (the ``spack.yaml`` file), which is
-used to define the roots of the Environment, but does not affect the
-concrete specs in the lockfile, nor does it install the spec.
+Adding a spec adds it as a root spec of the environment in the user
+input file (``spack.yaml``). It does not affect the concrete specs
+in the lock file (``spack.lock``) and it does not install the spec.
 
-The ``spack add`` command is environment aware. It adds to the
-currently active environment. All environment aware commands can also
+The ``spack add`` command is environment-aware. It adds the spec to the
+currently active environment. An error is generated if there isn't an
+active environment. All environment-aware commands can also
 be called using the ``spack -e`` flag to specify the environment.
 
 .. code-block:: console
@@ -300,11 +317,11 @@ or
 Concretizing
 ^^^^^^^^^^^^
 
-Once some user specs have been added to an environment, they can be concretized.
-There are at the moment three different modes of operation to concretize an environment,
-which are explained in details in :ref:`environments_concretization_config`.
-Regardless of which mode of operation has been chosen, the following
-command will ensure all the root specs are concretized according to the
+Once user specs have been added to an environment, they can be concretized.
+There are three different modes of operation to concretize an environment,
+explained in detail in :ref:`environments_concretization_config`.
+Regardless of which mode of operation is chosen, the following
+command will ensure all of the root specs are concretized according to the
 constraints that are prescribed in the configuration:
 
 .. code-block:: console
@@ -313,16 +330,15 @@ constraints that are prescribed in the configuration:
 
 In the case of specs that are not concretized together, the command
 above will concretize only the specs that were added and not yet
-concretized. Forcing a re-concretization of all the specs can be done
-instead with this command:
+concretized. Forcing a re-concretization of all of the specs can be done
+by adding the ``-f`` option:
 
 .. code-block:: console
 
    [myenv]$ spack concretize -f
 
-When the ``-f`` flag is not used to reconcretize all specs, Spack
-guarantees that already concretized specs are unchanged in the
-environment.
+Without the option, Spack guarantees that already concretized specs are
+unchanged in the environment.
 
 The ``concretize`` command does not install any packages. For packages
 that have already been installed outside of the environment, the
@@ -355,16 +371,16 @@ installed specs using the ``-c`` (``--concretized``) flag.
 Installing an Environment
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
-In addition to installing individual specs into an Environment, one
-can install the entire Environment at once using the command
+In addition to adding individual specs to an environment, one
+can install the entire environment at once using the command
 
 .. code-block:: console
 
    [myenv]$ spack install
 
-If the Environment has been concretized, Spack will install the
-concretized specs. Otherwise, ``spack install`` will first concretize
-the Environment and then install the concretized specs.
+If the environment has been concretized, Spack will install the
+concretized specs. Otherwise, ``spack install`` will concretize
+the environment before installing the concretized specs.
 
 .. note::
 
@@ -385,17 +401,17 @@ the Environment and then install the concretized specs.
 
 
 As it installs, ``spack install`` creates symbolic links in the
-``logs/`` directory in the Environment, allowing for easy inspection
+``logs/`` directory in the environment, allowing for easy inspection
 of build logs related to that environment. The ``spack install``
 command also stores a Spack repo containing the ``package.py`` file
 used at install time for each package in the ``repos/`` directory in
-the Environment.
+the environment.
 
 The ``--no-add`` option can be used in a concrete environment to tell
 spack to install specs already present in the environment but not to
 add any new root specs to the environment.  For root specs provided
 to ``spack install`` on the command line, ``--no-add`` is the default,
-while for dependency specs on the other hand, it is optional.  In other
+while for dependency specs, it is optional.  In other
 words, if there is an unambiguous match in the active concrete environment
 for a root spec provided to ``spack install`` on the command line, spack
 does not require you to specify the ``--no-add`` option to prevent the spec
@@ -409,12 +425,22 @@ Developing Packages in a Spack Environment
 
 The ``spack develop`` command allows one to develop Spack packages in
 an environment. It requires a spec containing a concrete version, and
-will configure Spack to install the package from local source. By
-default, it will also clone the package to a subdirectory in the
-environment. This package will have a special variant ``dev_path``
+will configure Spack to install the package from local source. 
+If a version is not provided from the command line interface then spack 
+will automatically pick the highest version the package has defined.
+This means any infinity versions (``develop``, ``main``, ``stable``) will be
+preferred in this selection process.
+By default, ``spack develop`` will also clone the package to a subdirectory in the
+environment for the local source. This package will have a special variant ``dev_path``
 set, and Spack will ensure the package and its dependents are rebuilt
 any time the environment is installed if the package's local source
-code has been modified. Spack ensures that all instances of a
+code has been modified. Spack's native implementation to check for modifications
+is to check if ``mtime`` is newer than the installation.
+A custom check can be created by overriding the ``detect_dev_src_change`` method 
+in your package class. This is particularly useful for projects using custom spack repo's 
+to drive development and want to optimize performance. 
+
+Spack ensures that all instances of a
 developed package in the environment are concretized to match the
 version (and other constraints) passed as the spec argument to the
 ``spack develop`` command.
@@ -424,7 +450,7 @@ also be used as valid concrete versions (see :ref:`version-specifier`).
 This means that for a package ``foo``, ``spack develop foo@git.main`` will clone
 the ``main`` branch of the package, and ``spack install`` will install from
 that git clone if ``foo`` is in the environment.
-Further development on ``foo`` can be tested by reinstalling the environment,
+Further development on ``foo`` can be tested by re-installing the environment,
 and eventually committed and pushed to the upstream git repo.
 
 If the package being developed supports out-of-source builds then users can use the
@@ -609,7 +635,7 @@ manipulate configuration inline in the ``spack.yaml`` file.
 Inline configurations
 ^^^^^^^^^^^^^^^^^^^^^
 
-Inline Environment-scope configuration is done using the same yaml
+Inline environment-scope configuration is done using the same yaml
 format as standard Spack configuration scopes, covered in the
 :ref:`configuration` section. Each section is contained under a
 top-level yaml object with it's name. For example, a ``spack.yaml``
@@ -634,7 +660,7 @@ Included configurations
 
 Spack environments allow an ``include`` heading in their yaml
 schema. This heading pulls in external configuration files and applies
-them to the Environment.
+them to the environment.
 
 .. code-block:: yaml
 
@@ -647,6 +673,9 @@ them to the Environment.
 Environments can include files or URLs. File paths can be relative or
 absolute. URLs include the path to the text for individual files or
 can be the path to a directory containing configuration files.
+Spack supports ``file``, ``http``, ``https`` and ``ftp`` protocols (or
+schemes). Spack-specific, environment and user path variables may be
+used in these paths. See :ref:`config-file-variables` for more information.
 
 ^^^^^^^^^^^^^^^^^^^^^^^^
 Configuration precedence
@@ -661,7 +690,7 @@ have higher precedence, as the included configs are applied in reverse order.
 Manually Editing the Specs List
 -------------------------------
 
-The list of abstract/root specs in the Environment is maintained in
+The list of abstract/root specs in the environment is maintained in
 the ``spack.yaml`` manifest under the heading ``specs``.
 
 .. code-block:: yaml
@@ -769,7 +798,7 @@ evaluates to the cross-product of those specs. Spec matrices also
 contain an ``excludes`` directive, which eliminates certain
 combinations from the evaluated result.
 
-The following two Environment manifests are identical:
+The following two environment manifests are identical:
 
 .. code-block:: yaml
 
@@ -844,7 +873,7 @@ files are identical.
 In short files like the example, it may be easier to simply list the
 included specs. However for more complicated examples involving many
 packages across many toolchains, separately factored lists make
-Environments substantially more manageable.
+environments substantially more manageable.
 
 Additionally, the ``-l`` option to the ``spack add`` command allows
 one to add to named lists in the definitions section of the manifest
@@ -1060,7 +1089,7 @@ true``). The argument ``--without-view`` can be used to create an
 environment without any view configured.
 
 The ``spack env view`` command can be used to change the manage views
-of an Environment. The subcommand ``spack env view enable`` will add a
+of an environment. The subcommand ``spack env view enable`` will add a
 view named ``default`` to an environment. It takes an optional
 argument to specify the path for the new default view. The subcommand
 ``spack env view disable`` will remove the view named ``default`` from
@@ -1228,7 +1257,7 @@ gets installed and is available for use in the ``env`` target.
    	$(SPACK) -e . env depfile -o $@ --make-prefix spack
 
    env: spack/env
-   	$(info Environment installed!)
+   	$(info environment installed!)
 
    clean:
    	rm -rf spack.lock env.mk spack/

lib/spack/docs/getting_started.rst

@@ -61,10 +61,15 @@ Getting Spack is easy.  You can clone it from the `github repository
 
 .. code-block:: console
 
-   $ git clone -c feature.manyFiles=true https://github.com/spack/spack.git
+   $ git clone -c feature.manyFiles=true --depth=2 https://github.com/spack/spack.git
 
 This will create a directory called ``spack``.
 
+.. note::
+   ``-c feature.manyFiles=true`` improves git's performance on repositories with 1,000+ files.
+
+   ``--depth=2`` prunes the git history to reduce the size of the Spack installation.
+
 .. _shell-support:
 
 ^^^^^^^^^^^^^
@@ -1475,16 +1480,14 @@ in a Windows CMD prompt.
 Step 3: Run and configure Spack
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-To use Spack, run ``bin\spack_cmd.bat`` (you may need to Run as Administrator) from the top-level spack
-directory. This will provide a Windows command prompt with an environment properly set up with Spack
-and its prerequisites. If you receive a warning message that Python is not in your ``PATH``
+On Windows, Spack supports both primary native shells, Powershell and the traditional command prompt.
+To use Spack, pick your favorite shell, and run ``bin\spack_cmd.bat`` or ``share/spack/setup-env.ps1``
+(you may need to Run as Administrator) from the top-level spack
+directory. This will provide a Spack enabled shell. If you receive a warning message that Python is not in your ``PATH``
 (which may happen if you installed Python from the website and not the Windows Store) add the location
 of the Python executable to your ``PATH`` now. You can permanently add Python to your ``PATH`` variable
 by using the ``Edit the system environment variables`` utility in Windows Control Panel.
 
-.. note::
-   Alternatively, Powershell can be used in place of CMD
-
 To configure Spack, first run the following command inside the Spack console:
 
 .. code-block:: console
@@ -1549,7 +1552,7 @@ and not tabs, so ensure that this is the case when editing one directly.
 
 .. note:: Cygwin
    The use of Cygwin is not officially supported by Spack and is not tested.
-   However Spack will not throw an error, so use if choosing to use Spack
+   However Spack will not prevent this, so use if choosing to use Spack
    with Cygwin, know that no functionality is garunteed.
 
 ^^^^^^^^^^^^^^^^^
@@ -1563,21 +1566,12 @@ Spack console via:
 
    spack install cpuinfo
 
-If in the previous step, you did not have CMake or Ninja installed, running the command above should bootstrap both packages
+If in the previous step, you did not have CMake or Ninja installed, running the command above should install both packages
 
-"""""""""""""""""""""""""""
-Windows Compatible Packages
-"""""""""""""""""""""""""""
+.. note:: Spec Syntax Caveats
+   Windows has a few idiosyncrasies when it comes to the Spack spec syntax and the use of certain shells
+   See the Spack spec syntax doc for more information
 
-Not all spack packages currently have Windows support. Some are inherently incompatible with the
-platform, and others simply have yet to be ported. To view the current set of packages with Windows
-support, the list command should be used via `spack list -t windows`. If there's a package you'd like
-to install on Windows but is not in that list, feel free to reach out to request the port or contribute
-the port yourself.
-
-.. note::
-   This is by no means a comprehensive list, some packages may have ports that were not tagged
-   while others may just work out of the box on Windows and have not been tagged as such.
 
 ^^^^^^^^^^^^^^
 For developers
@@ -1587,6 +1581,3 @@ The intent is to provide a Windows installer that will automatically set up
 Python, Git, and Spack, instead of requiring the user to do so manually.
 Instructions for creating the installer are at
 https://github.com/spack/spack/blob/develop/lib/spack/spack/cmd/installer/README.md
-
-Alternatively a pre-built copy of the Windows installer is available as an artifact of Spack's Windows CI
-available at each run of the CI on develop or any PR.

lib/spack/docs/index.rst

@@ -39,10 +39,15 @@ package:
 
 .. code-block:: console
 
-   $ git clone -c feature.manyFiles=true https://github.com/spack/spack.git
+   $ git clone -c feature.manyFiles=true --depth=2 https://github.com/spack/spack.git
    $ cd spack/bin
    $ ./spack install libelf
 
+.. note::
+   ``-c feature.manyFiles=true`` improves git's performance on repositories with 1,000+ files.
+
+   ``--depth=2`` prunes the git history to reduce the size of the Spack installation.
+
 If you're new to spack and want to start using it, see :doc:`getting_started`,
 or refer to the full manual below.
 

lib/spack/docs/module_file_support.rst

@@ -457,11 +457,11 @@ For instance, the following config options,
        tcl:
          all:
            suffixes:
-             ^python@3.12: 'python-3.12'
+             ^python@3: 'python{^python.version}'
              ^openblas: 'openblas'
 
-will add a ``python-3.12`` version string to any packages compiled with
-Python matching the spec, ``python@3.12``. This is useful to know which
+will add a ``python-3.12.1`` version string to any packages compiled with
+Python matching the spec, ``python@3``. This is useful to know which
 version of Python a set of Python extensions is associated with. Likewise, the
 ``openblas`` string is attached to any program that has openblas in the spec,
 most likely via the ``+blas`` variant specification.

lib/spack/docs/pipelines.rst

@@ -592,6 +592,77 @@ the attributes will be merged starting from the bottom match going up to the top
 
 In the case that no match is found in a submapping section, no additional attributes will be applied.
 
+
+^^^^^^^^^^^^^^^^^^^^^^^^
+Dynamic Mapping Sections
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+For large scale CI where cost optimization is required, dynamic mapping allows for the use of real-time
+mapping schemes served by a web service. This type of mapping does not support the ``-remove`` type
+behavior, but it does follow the rest of the merge rules for configurations.
+
+The dynamic mapping service needs to implement a single REST API interface for getting
+requests ``GET <URL>[:PORT][/PATH]?spec=<pkg_name@pkg_version +variant1+variant2%compiler@compiler_version>``.
+
+example request.
+
+.. code-block::
+
+  https://my-dyn-mapping.spack.io/allocation?spec=zlib-ng@2.1.6 +compat+opt+shared+pic+new_strategies arch=linux-ubuntu20.04-x86_64_v3%gcc@12.0.0
+
+
+With an example response the updates kubernetes request variables, overrides the max retries for gitlab,
+and prepends a note about the modifications made by the my-dyn-mapping.spack.io service.
+
+.. code-block::
+
+  200 OK
+
+  {
+    "variables":
+    {
+      "KUBERNETES_CPU_REQUEST": "500m",
+      "KUBERNETES_MEMORY_REQUEST": "2G",
+    },
+    "retry": { "max:": "1"}
+    "script+:":
+    [
+      "echo \"Job modified by my-dyn-mapping.spack.io\""
+    ]
+  }
+
+
+The ci.yaml configuration section takes the URL endpoint as well as a number of options to configure how responses are handled.
+
+It is possible to specify a list of allowed and ignored configuration attributes under ``allow`` and ``ignore``
+respectively. It is also possible to configure required attributes under ``required`` section.
+
+Options to configure the client timeout and SSL verification using the ``timeout`` and ``verify_ssl`` options.
+By default, the ``timeout`` is set to the option in ``config:timeout`` and ``veryify_ssl`` is set the the option in ``config::verify_ssl``.
+
+Passing header parameters to the request can be achieved through the ``header`` section. The values of the variables passed to the
+header may be environment variables that are expanded at runtime, such as a private token configured on the runner.
+
+Here is an example configuration pointing to ``my-dyn-mapping.spack.io/allocation``.
+
+
+.. code-block:: yaml
+
+  ci:
+  - dynamic-mapping:
+      endpoint: my-dyn-mapping.spack.io/allocation
+      timeout: 10
+      verify_ssl: True
+      header:
+        PRIVATE_TOKEN: ${MY_PRIVATE_TOKEN}
+        MY_CONFIG: "fuzz_allocation:false"
+      allow:
+      - variables
+      ignore:
+      - script
+      require: []
+
+
 ^^^^^^^^^^^^^
 Bootstrapping
 ^^^^^^^^^^^^^

lib/spack/docs/requirements.txt

@@ -1,13 +1,13 @@
 sphinx==7.4.7
 sphinxcontrib-programoutput==0.17
 sphinx_design==0.6.1
-sphinx-rtd-theme==2.0.0
-python-levenshtein==0.25.1
+sphinx-rtd-theme==3.0.1
+python-levenshtein==0.26.0
 docutils==0.20.1
 pygments==2.18.0
 urllib3==2.2.3
 pytest==8.3.3
 isort==5.13.2
-black==24.8.0
+black==24.10.0
 flake8==7.1.1
 mypy==1.11.1

lib/spack/external/__init__.py

@@ -18,7 +18,7 @@
 
 * Homepage: https://pypi.python.org/pypi/archspec
 * Usage: Labeling, comparison and detection of microarchitectures
-* Version: 0.2.5-dev (commit cbb1fd5eb397a70d466e5160b393b87b0dbcc78f)
+* Version: 0.2.5 (commit 38ce485258ffc4fc6dd6688f8dc90cb269478c47)
 
 astunparse
 ----------------

lib/spack/external/archspec/cpu/microarchitecture.py

@@ -81,8 +81,13 @@ def __init__(self, name, parents, vendor, features, compilers, generation=0, cpu
         self.generation = generation
         # Only relevant for AArch64
         self.cpu_part = cpu_part
-        # Cache the ancestor computation
+
+        # Cache the "ancestor" computation
         self._ancestors = None
+        # Cache the "generic" computation
+        self._generic = None
+        # Cache the "family" computation
+        self._family = None
 
     @property
     def ancestors(self):
@@ -115,6 +120,9 @@ def __eq__(self, other):
             and self.cpu_part == other.cpu_part
         )
 
+    def __hash__(self):
+        return hash(self.name)
+
     @coerce_target_names
     def __ne__(self, other):
         return not self == other
@@ -171,18 +179,22 @@ def __contains__(self, feature):
     @property
     def family(self):
         """Returns the architecture family a given target belongs to"""
-        roots = [x for x in [self] + self.ancestors if not x.ancestors]
-        msg = "a target is expected to belong to just one architecture family"
-        msg += f"[found {', '.join(str(x) for x in roots)}]"
-        assert len(roots) == 1, msg
+        if self._family is None:
+            roots = [x for x in [self] + self.ancestors if not x.ancestors]
+            msg = "a target is expected to belong to just one architecture family"
+            msg += f"[found {', '.join(str(x) for x in roots)}]"
+            assert len(roots) == 1, msg
+            self._family = roots.pop()
 
-        return roots.pop()
+        return self._family
 
     @property
     def generic(self):
         """Returns the best generic architecture that is compatible with self"""
-        generics = [x for x in [self] + self.ancestors if x.vendor == "generic"]
-        return max(generics, key=lambda x: len(x.ancestors))
+        if self._generic is None:
+            generics = [x for x in [self] + self.ancestors if x.vendor == "generic"]
+            self._generic = max(generics, key=lambda x: len(x.ancestors))
+        return self._generic
 
     def to_dict(self):
         """Returns a dictionary representation of this object."""

lib/spack/external/archspec/json/cpu/microarchitectures.json

@@ -1482,7 +1482,6 @@
         "cldemote",
         "movdir64b",
         "movdiri",
-        "pdcm",
         "serialize",
         "waitpkg"
       ],
@@ -2237,6 +2236,84 @@
 	]
       }
     },
+    "zen5": {
+      "from": ["zen4"],
+      "vendor": "AuthenticAMD",
+      "features": [
+        "abm",
+        "aes",
+        "avx",
+        "avx2",
+        "avx512_bf16",
+        "avx512_bitalg",
+        "avx512bw",
+        "avx512cd",
+        "avx512dq",
+        "avx512f",
+        "avx512ifma",
+        "avx512vbmi",
+        "avx512_vbmi2",
+        "avx512vl",
+        "avx512_vnni",
+        "avx512_vp2intersect",
+        "avx512_vpopcntdq",
+        "avx_vnni",
+        "bmi1",
+        "bmi2",
+        "clflushopt",
+        "clwb",
+        "clzero",
+        "cppc",
+        "cx16",
+        "f16c",
+        "flush_l1d",
+        "fma",
+        "fsgsbase",
+        "gfni",
+        "ibrs_enhanced",
+        "mmx",
+        "movbe",
+        "movdir64b",
+        "movdiri",
+        "pclmulqdq",
+        "popcnt",
+        "rdseed",
+        "sse",
+        "sse2",
+        "sse4_1",
+        "sse4_2",
+        "sse4a",
+        "ssse3",
+        "tsc_adjust",
+        "vaes",
+        "vpclmulqdq",
+        "xsavec",
+        "xsaveopt"
+      ],
+      "compilers": {
+        "gcc": [
+          {
+            "versions": "14.1:",
+            "name": "znver5",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
+        "aocc": [
+          {
+            "versions": "5.0:",
+            "name": "znver5",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
+        "clang": [
+          {
+            "versions": "19.1:",
+            "name": "znver5",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ]
+      }
+    },
     "ppc64": {
       "from": [],
       "vendor": "generic",

lib/spack/llnl/string.py

@@ -41,6 +41,20 @@ def comma_and(sequence: List[str]) -> str:
     return comma_list(sequence, "and")
 
 
+def ordinal(number: int) -> str:
+    """Return the ordinal representation (1st, 2nd, 3rd, etc.) for the provided number.
+
+    Args:
+        number: int to convert to ordinal number
+
+    Returns: number's corresponding ordinal
+    """
+    idx = (number % 10) << 1
+    tens = number % 100 // 10
+    suffix = "th" if tens == 1 or idx > 6 else "thstndrd"[idx : idx + 2]
+    return f"{number}{suffix}"
+
+
 def quote(sequence: List[str], q: str = "'") -> List[str]:
     """Quotes each item in the input list with the quote character passed as second argument."""
     return [f"{q}{e}{q}" for e in sequence]

lib/spack/llnl/util/filesystem.py

@@ -47,6 +47,7 @@
     "copy_mode",
     "filter_file",
     "find",
+    "find_first",
     "find_headers",
     "find_all_headers",
     "find_libraries",

lib/spack/llnl/util/lang.py

@@ -12,7 +12,7 @@
 import sys
 import traceback
 from datetime import datetime, timedelta
-from typing import Any, Callable, Iterable, List, Tuple
+from typing import Callable, Iterable, List, Tuple, TypeVar
 
 # Ignore emacs backups when listing modules
 ignore_modules = r"^\.#|~$"
@@ -879,9 +879,12 @@ def enum(**kwargs):
     return type("Enum", (object,), kwargs)
 
 
+T = TypeVar("T")
+
+
 def stable_partition(
-    input_iterable: Iterable, predicate_fn: Callable[[Any], bool]
-) -> Tuple[List[Any], List[Any]]:
+    input_iterable: Iterable[T], predicate_fn: Callable[[T], bool]
+) -> Tuple[List[T], List[T]]:
     """Partition the input iterable according to a custom predicate.
 
     Args:
@@ -893,12 +896,13 @@ def stable_partition(
         Tuple of the list of elements evaluating to True, and
         list of elements evaluating to False.
     """
-    true_items, false_items = [], []
+    true_items: List[T] = []
+    false_items: List[T] = []
     for item in input_iterable:
         if predicate_fn(item):
             true_items.append(item)
-            continue
-        false_items.append(item)
+        else:
+            false_items.append(item)
     return true_items, false_items
 
 

lib/spack/llnl/util/tty/log.py

@@ -348,7 +348,19 @@ def close(self):
 class MultiProcessFd:
     """Return an object which stores a file descriptor and can be passed as an
     argument to a function run with ``multiprocessing.Process``, such that
-    the file descriptor is available in the subprocess."""
+    the file descriptor is available in the subprocess. It provides access via
+    the `fd` property.
+
+    This object takes control over the associated FD: files opened from this
+    using `fdopen` need to use `closefd=False`.
+    """
+
+    # As for why you have to fdopen(..., closefd=False): when a
+    # multiprocessing.connection.Connection object stores an fd, it assumes
+    # control over it, and will attempt to close it when gc'ed during __del__;
+    # if you fdopen(multiprocessfd.fd, closefd=True) then the resulting file
+    # will also assume control, and you can see warnings when there is an
+    # attempted double close.
 
     def __init__(self, fd):
         self._connection = None
@@ -361,33 +373,20 @@ def __init__(self, fd):
     @property
     def fd(self):
         if self._connection:
-            return self._connection._handle
+            return self._connection.fileno()
         else:
             return self._fd
 
     def close(self):
+        """Rather than `.close()`ing any file opened from the associated
+        `.fd`, the `MultiProcessFd` should be closed with this.
+        """
         if self._connection:
             self._connection.close()
         else:
             os.close(self._fd)
 
 
-def close_connection_and_file(multiprocess_fd, file):
-    # MultiprocessFd is intended to transmit a FD
-    # to a child process, this FD is then opened to a Python File object
-    # (using fdopen). In >= 3.8, MultiprocessFd encapsulates a
-    # multiprocessing.connection.Connection; Connection closes the FD
-    # when it is deleted, and prints a warning about duplicate closure if
-    # it is not explicitly closed. In < 3.8, MultiprocessFd encapsulates a
-    # simple FD; closing the FD here appears to conflict with
-    # closure of the File object (in < 3.8 that is). Therefore this needs
-    # to choose whether to close the File or the Connection.
-    if sys.version_info >= (3, 8):
-        multiprocess_fd.close()
-    else:
-        file.close()
-
-
 @contextmanager
 def replace_environment(env):
     """Replace the current environment (`os.environ`) with `env`.
@@ -932,10 +931,10 @@ def _writer_daemon(
     # 1. Use line buffering (3rd param = 1) since Python 3 has a bug
     # that prevents unbuffered text I/O.
     # 2. Python 3.x before 3.7 does not open with UTF-8 encoding by default
-    in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1, encoding="utf-8")
+    in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1, encoding="utf-8", closefd=False)
 
     if stdin_multiprocess_fd:
-        stdin = os.fdopen(stdin_multiprocess_fd.fd)
+        stdin = os.fdopen(stdin_multiprocess_fd.fd, closefd=False)
     else:
         stdin = None
 
@@ -1025,9 +1024,9 @@ def _writer_daemon(
         if isinstance(log_file, io.StringIO):
             control_pipe.send(log_file.getvalue())
         log_file_wrapper.close()
-        close_connection_and_file(read_multiprocess_fd, in_pipe)
+        read_multiprocess_fd.close()
         if stdin_multiprocess_fd:
-            close_connection_and_file(stdin_multiprocess_fd, stdin)
+            stdin_multiprocess_fd.close()
 
         # send echo value back to the parent so it can be preserved.
         control_pipe.send(echo)

lib/spack/spack/audit.py

@@ -39,6 +39,7 @@ def _search_duplicate_compilers(error_cls):
 import collections
 import collections.abc
 import glob
+import inspect
 import io
 import itertools
 import os
@@ -50,6 +51,7 @@ def _search_duplicate_compilers(error_cls):
 from urllib.request import urlopen
 
 import llnl.util.lang
+from llnl.string import plural
 
 import spack.builder
 import spack.config
@@ -386,6 +388,14 @@ def _make_config_error(config_data, summary, error_cls):
 )
 
 
+package_deprecated_attributes = AuditClass(
+    group="packages",
+    tag="PKG-DEPRECATED-ATTRIBUTES",
+    description="Sanity checks to preclude use of deprecated package attributes",
+    kwargs=("pkgs",),
+)
+
+
 package_properties = AuditClass(
     group="packages",
     tag="PKG-PROPERTIES",
@@ -404,22 +414,23 @@ def _make_config_error(config_data, summary, error_cls):
 )
 
 
-@package_directives
+@package_properties
 def _check_build_test_callbacks(pkgs, error_cls):
-    """Ensure stand-alone test method is not included in build-time callbacks"""
+    """Ensure stand-alone test methods are not included in build-time callbacks.
+
+    Test methods are for checking the installed software as stand-alone tests.
+    They could also be called during the post-install phase of a build.
+    """
     errors = []
     for pkg_name in pkgs:
         pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
         test_callbacks = getattr(pkg_cls, "build_time_test_callbacks", None)
 
-        # TODO (post-34236): "test*"->"test_*" once remove deprecated methods
-        # TODO (post-34236): "test"->"test_" once remove deprecated methods
-        has_test_method = test_callbacks and any([m.startswith("test") for m in test_callbacks])
+        has_test_method = test_callbacks and any([m.startswith("test_") for m in test_callbacks])
         if has_test_method:
-            msg = '{0} package contains "test*" method(s) in ' "build_time_test_callbacks"
-            instr = 'Remove all methods whose names start with "test" from: [{0}]'.format(
-                ", ".join(test_callbacks)
-            )
+            msg = f"Package {pkg_name} includes stand-alone test methods in build-time checks."
+            callbacks = ", ".join(test_callbacks)
+            instr = f"Remove the following from 'build_time_test_callbacks': {callbacks}"
             errors.append(error_cls(msg.format(pkg_name), [instr]))
 
     return errors
@@ -517,6 +528,46 @@ def _search_for_reserved_attributes_names_in_packages(pkgs, error_cls):
     return errors
 
 
+@package_deprecated_attributes
+def _search_for_deprecated_package_methods(pkgs, error_cls):
+    """Ensure the package doesn't define or use deprecated methods"""
+    DEPRECATED_METHOD = (("test", "a name starting with 'test_'"),)
+    DEPRECATED_USE = (
+        ("self.cache_extra_test_sources(", "cache_extra_test_sources(self, ..)"),
+        ("self.install_test_root(", "install_test_root(self, ..)"),
+        ("self.run_test(", "test_part(self, ..)"),
+    )
+    errors = []
+    for pkg_name in pkgs:
+        pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
+        methods = inspect.getmembers(pkg_cls, predicate=lambda x: inspect.isfunction(x))
+        method_errors = collections.defaultdict(list)
+        for name, function in methods:
+            for deprecated_name, alternate in DEPRECATED_METHOD:
+                if name == deprecated_name:
+                    msg = f"Rename '{deprecated_name}' method to {alternate} instead."
+                    method_errors[name].append(msg)
+
+            source = inspect.getsource(function)
+            for deprecated_name, alternate in DEPRECATED_USE:
+                if deprecated_name in source:
+                    msg = f"Change '{deprecated_name}' to '{alternate}' in '{name}' method."
+                    method_errors[name].append(msg)
+
+        num_methods = len(method_errors)
+        if num_methods > 0:
+            methods = plural(num_methods, "method", show_n=False)
+            error_msg = (
+                f"Package '{pkg_name}' implements or uses unsupported deprecated {methods}."
+            )
+            instr = [f"Make changes to '{pkg_cls.__module__}':"]
+            for name in sorted(method_errors):
+                instr.extend([f"    {msg}" for msg in method_errors[name]])
+            errors.append(error_cls(error_msg, instr))
+
+    return errors
+
+
 @package_properties
 def _ensure_all_package_names_are_lowercase(pkgs, error_cls):
     """Ensure package names are lowercase and consistent"""
@@ -771,6 +822,89 @@ def _uses_deprecated_globals(pkgs, error_cls):
     return errors
 
 
+@package_properties
+def _ensure_test_docstring(pkgs, error_cls):
+    """Ensure stand-alone test methods have a docstring.
+
+    The docstring of a test method is implicitly used as the description of
+    the corresponding test part during test results reporting.
+    """
+    doc_regex = r'\s+("""[^"]+""")'
+
+    errors = []
+    for pkg_name in pkgs:
+        pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
+        methods = inspect.getmembers(pkg_cls, predicate=lambda x: inspect.isfunction(x))
+        method_names = []
+        for name, test_fn in methods:
+            if not name.startswith("test_"):
+                continue
+
+            # Ensure the test method has a docstring
+            source = inspect.getsource(test_fn)
+            match = re.search(doc_regex, source)
+            if match is None or len(match.group(0).replace('"', "").strip()) == 0:
+                method_names.append(name)
+
+        num_methods = len(method_names)
+        if num_methods > 0:
+            methods = plural(num_methods, "method", show_n=False)
+            docstrings = plural(num_methods, "docstring", show_n=False)
+            msg = f"Package {pkg_name} has test {methods} with empty or missing {docstrings}."
+            names = ", ".join(method_names)
+            instr = [
+                "Docstrings are used as descriptions in test outputs.",
+                f"Add a concise summary to the following {methods} in '{pkg_cls.__module__}':",
+                f"{names}",
+            ]
+            errors.append(error_cls(msg, instr))
+
+    return errors
+
+
+@package_properties
+def _ensure_test_implemented(pkgs, error_cls):
+    """Ensure stand-alone test methods are implemented.
+
+    The test method is also required to be non-empty.
+    """
+
+    def skip(line):
+        ln = line.strip()
+        return ln.startswith("#") or "pass" in ln
+
+    doc_regex = r'\s+("""[^"]+""")'
+
+    errors = []
+    for pkg_name in pkgs:
+        pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
+        methods = inspect.getmembers(pkg_cls, predicate=lambda x: inspect.isfunction(x))
+        method_names = []
+        for name, test_fn in methods:
+            if not name.startswith("test_"):
+                continue
+
+            source = inspect.getsource(test_fn)
+
+            # Attempt to ensure the test method is implemented.
+            impl = re.sub(doc_regex, r"", source).splitlines()[1:]
+            lines = [ln.strip() for ln in impl if not skip(ln)]
+            if not lines:
+                method_names.append(name)
+
+        num_methods = len(method_names)
+        if num_methods > 0:
+            methods = plural(num_methods, "method", show_n=False)
+            msg = f"Package {pkg_name} has empty or missing test {methods}."
+            names = ", ".join(method_names)
+            instr = [
+                f"Implement or remove the following {methods} from '{pkg_cls.__module__}': {names}"
+            ]
+            errors.append(error_cls(msg, instr))
+
+    return errors
+
+
 @package_https_directives
 def _linting_package_file(pkgs, error_cls):
     """Check for correctness of links"""

lib/spack/spack/binary_distribution.py

@@ -35,6 +35,7 @@
 import spack.caches
 import spack.config as config
 import spack.database as spack_db
+import spack.deptypes as dt
 import spack.error
 import spack.hash_types as ht
 import spack.hooks
@@ -712,15 +713,32 @@ def get_buildfile_manifest(spec):
     return data
 
 
-def hashes_to_prefixes(spec):
-    """Return a dictionary of hashes to prefixes for a spec and its deps, excluding externals"""
-    return {
-        s.dag_hash(): str(s.prefix)
+def deps_to_relocate(spec):
+    """Return the transitive link and direct run dependencies of the spec.
+
+    This is a special traversal for dependencies we need to consider when relocating a package.
+
+    Package binaries, scripts, and other files may refer to the prefixes of  dependencies, so
+    we need to rewrite those locations when dependencies are in a different place at install time
+    than they were at build time.
+
+    This traversal covers transitive link dependencies and direct run dependencies because:
+
+    1. Spack adds RPATHs for transitive link dependencies so that packages can find needed
+       dependency libraries.
+    2. Packages may call any of their *direct* run dependencies (and may bake their paths into
+       binaries or scripts), so we also need to search for run dependency prefixes when relocating.
+
+    This returns a deduplicated list of transitive link dependencies and direct run dependencies.
+    """
+    deps = [
+        s
         for s in itertools.chain(
             spec.traverse(root=True, deptype="link"), spec.dependencies(deptype="run")
         )
         if not s.external
-    }
+    ]
+    return llnl.util.lang.dedupe(deps, key=lambda s: s.dag_hash())
 
 
 def get_buildinfo_dict(spec):
@@ -736,7 +754,7 @@ def get_buildinfo_dict(spec):
         "relocate_binaries": manifest["binary_to_relocate"],
         "relocate_links": manifest["link_to_relocate"],
         "hardlinks_deduped": manifest["hardlinks_deduped"],
-        "hash_to_prefix": hashes_to_prefixes(spec),
+        "hash_to_prefix": {d.dag_hash(): str(d.prefix) for d in deps_to_relocate(spec)},
     }
 
 
@@ -1631,7 +1649,6 @@ def _oci_push(
     Dict[str, spack.oci.oci.Blob],
     List[Tuple[Spec, BaseException]],
 ]:
-
     # Spec dag hash -> blob
     checksums: Dict[str, spack.oci.oci.Blob] = {}
 
@@ -2201,11 +2218,36 @@ def relocate_package(spec):
     # First match specific prefix paths. Possibly the *local* install prefix
     # of some dependency is in an upstream, so we cannot assume the original
     # spack store root can be mapped uniformly to the new spack store root.
-    for dag_hash, new_dep_prefix in hashes_to_prefixes(spec).items():
-        if dag_hash in hash_to_old_prefix:
-            old_dep_prefix = hash_to_old_prefix[dag_hash]
-            prefix_to_prefix_bin[old_dep_prefix] = new_dep_prefix
-            prefix_to_prefix_text[old_dep_prefix] = new_dep_prefix
+    #
+    # If the spec is spliced, we need to handle the simultaneous mapping
+    # from the old install_tree to the new install_tree and from the build_spec
+    # to the spliced spec.
+    # Because foo.build_spec is foo for any non-spliced spec, we can simplify
+    # by checking for spliced-in nodes by checking for nodes not in the build_spec
+    # without any explicit check for whether the spec is spliced.
+    # An analog in this algorithm is any spec that shares a name or provides the same virtuals
+    # in the context of the relevant root spec. This ensures that the analog for a spec s
+    # is the spec that s replaced when we spliced.
+    relocation_specs = deps_to_relocate(spec)
+    build_spec_ids = set(id(s) for s in spec.build_spec.traverse(deptype=dt.ALL & ~dt.BUILD))
+    for s in relocation_specs:
+        analog = s
+        if id(s) not in build_spec_ids:
+            analogs = [
+                d
+                for d in spec.build_spec.traverse(deptype=dt.ALL & ~dt.BUILD)
+                if s._splice_match(d, self_root=spec, other_root=spec.build_spec)
+            ]
+            if analogs:
+                # Prefer same-name analogs and prefer higher versions
+                # This matches the preferences in Spec.splice, so we will find same node
+                analog = max(analogs, key=lambda a: (a.name == s.name, a.version))
+
+        lookup_dag_hash = analog.dag_hash()
+        if lookup_dag_hash in hash_to_old_prefix:
+            old_dep_prefix = hash_to_old_prefix[lookup_dag_hash]
+            prefix_to_prefix_bin[old_dep_prefix] = str(s.prefix)
+            prefix_to_prefix_text[old_dep_prefix] = str(s.prefix)
 
     # Only then add the generic fallback of install prefix -> install prefix.
     prefix_to_prefix_text[old_prefix] = new_prefix
@@ -2520,7 +2562,13 @@ def _ensure_common_prefix(tar: tarfile.TarFile) -> str:
     return pkg_prefix
 
 
-def install_root_node(spec, unsigned=False, force=False, sha256=None):
+def install_root_node(
+    spec: spack.spec.Spec,
+    unsigned=False,
+    force: bool = False,
+    sha256: Optional[str] = None,
+    allow_missing: bool = False,
+) -> None:
     """Install the root node of a concrete spec from a buildcache.
 
     Checking the sha256 sum of a node before installation is usually needed only
@@ -2529,11 +2577,10 @@ def install_root_node(spec, unsigned=False, force=False, sha256=None):
 
     Args:
         spec: spec to be installed (note that only the root node will be installed)
-        unsigned (bool): if True allows installing unsigned binaries
-        force (bool): force installation if the spec is already present in the
-            local store
-        sha256 (str): optional sha256 of the binary package, to be checked
-            before installation
+        unsigned: if True allows installing unsigned binaries
+        force: force installation if the spec is already present in the local store
+        sha256: optional sha256 of the binary package, to be checked before installation
+        allow_missing: when true, allows installing a node with missing dependencies
     """
     # Early termination
     if spec.external or spec.virtual:
@@ -2543,10 +2590,10 @@ def install_root_node(spec, unsigned=False, force=False, sha256=None):
         warnings.warn("Package for spec {0} already installed.".format(spec.format()))
         return
 
-    download_result = download_tarball(spec, unsigned)
+    download_result = download_tarball(spec.build_spec, unsigned)
     if not download_result:
         msg = 'download of binary cache file for spec "{0}" failed'
-        raise RuntimeError(msg.format(spec.format()))
+        raise RuntimeError(msg.format(spec.build_spec.format()))
 
     if sha256:
         checker = spack.util.crypto.Checker(sha256)
@@ -2565,8 +2612,13 @@ def install_root_node(spec, unsigned=False, force=False, sha256=None):
     with spack.util.path.filter_padding():
         tty.msg('Installing "{0}" from a buildcache'.format(spec.format()))
         extract_tarball(spec, download_result, force)
+        spec.package.windows_establish_runtime_linkage()
+        if spec.spliced:  # overwrite old metadata with new
+            spack.store.STORE.layout.write_spec(
+                spec, spack.store.STORE.layout.spec_file_path(spec)
+            )
         spack.hooks.post_install(spec, False)
-        spack.store.STORE.db.add(spec)
+        spack.store.STORE.db.add(spec, allow_missing=allow_missing)
 
 
 def install_single_spec(spec, unsigned=False, force=False):

lib/spack/spack/bootstrap/_common.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Common basic functions used through the spack.bootstrap package"""
 import fnmatch
+import glob
 import importlib
 import os.path
 import re
@@ -60,10 +61,19 @@ def _try_import_from_store(
             python, *_ = candidate_spec.dependencies("python-venv")
         else:
             python, *_ = candidate_spec.dependencies("python")
-        module_paths = [
-            os.path.join(candidate_spec.prefix, python.package.purelib),
-            os.path.join(candidate_spec.prefix, python.package.platlib),
-        ]
+
+        # if python is installed, ask it for the layout
+        if python.installed:
+            module_paths = [
+                os.path.join(candidate_spec.prefix, python.package.purelib),
+                os.path.join(candidate_spec.prefix, python.package.platlib),
+            ]
+        # otherwise search for the site-packages directory
+        # (clingo from binaries with truncated python-venv runtime)
+        else:
+            module_paths = glob.glob(
+                os.path.join(candidate_spec.prefix, "lib", "python*", "site-packages")
+            )
         path_before = list(sys.path)
 
         # NOTE: try module_paths first and last, last allows an existing version in path

lib/spack/spack/bootstrap/core.py

@@ -46,6 +46,7 @@
 import spack.util.spack_yaml
 import spack.util.url
 import spack.version
+from spack.installer import PackageInstaller
 
 from ._common import _executables_in_store, _python_import, _root_spec, _try_import_from_store
 from .clingo import ClingoBootstrapConcretizer
@@ -174,7 +175,15 @@ def _install_by_hash(
             query = spack.binary_distribution.BinaryCacheQuery(all_architectures=True)
             for match in spack.store.find([f"/{pkg_hash}"], multiple=False, query_fn=query):
                 spack.binary_distribution.install_root_node(
-                    match, unsigned=True, force=True, sha256=pkg_sha256
+                    # allow_missing is true since when bootstrapping clingo we truncate runtime
+                    # deps such as gcc-runtime, since we link libstdc++ statically, and the other
+                    # further runtime deps are loaded by the Python interpreter. This just silences
+                    # warnings about missing dependencies.
+                    match,
+                    unsigned=True,
+                    force=True,
+                    sha256=pkg_sha256,
+                    allow_missing=True,
                 )
 
     def _install_and_test(
@@ -277,7 +286,7 @@ def try_import(self, module: str, abstract_spec_str: str) -> bool:
 
         # Install the spec that should make the module importable
         with spack.config.override(self.mirror_scope):
-            concrete_spec.package.do_install(fail_fast=True)
+            PackageInstaller([concrete_spec.package], fail_fast=True).install()
 
         if _try_import_from_store(module, query_spec=concrete_spec, query_info=info):
             self.last_search = info
@@ -300,7 +309,7 @@ def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bo
         msg = "[BOOTSTRAP] Try installing '{0}' from sources"
         tty.debug(msg.format(abstract_spec_str))
         with spack.config.override(self.mirror_scope):
-            concrete_spec.package.do_install()
+            PackageInstaller([concrete_spec.package], fail_fast=True).install()
         if _executables_in_store(executables, concrete_spec, query_info=info):
             self.last_search = info
             return True

lib/spack/spack/build_environment.py

@@ -37,13 +37,16 @@
 import multiprocessing
 import os
 import re
+import stat
 import sys
 import traceback
 import types
 from collections import defaultdict
 from enum import Flag, auto
 from itertools import chain
-from typing import Dict, List, Set, Tuple
+from typing import Callable, Dict, List, Optional, Set, Tuple
+
+import archspec.cpu
 
 import llnl.util.tty as tty
 from llnl.string import plural
@@ -72,6 +75,7 @@
 import spack.store
 import spack.subprocess_context
 import spack.util.executable
+import spack.util.libc
 from spack import traverse
 from spack.context import Context
 from spack.error import InstallError, NoHeadersError, NoLibrariesError
@@ -87,7 +91,7 @@
 )
 from spack.util.executable import Executable
 from spack.util.log_parse import make_log_context, parse_log_events
-from spack.util.module_cmd import load_module, path_from_modules
+from spack.util.module_cmd import load_module
 
 #
 # This can be set by the user to globally disable parallel builds.
@@ -358,7 +362,7 @@ def set_compiler_environment_variables(pkg, env):
     _add_werror_handling(keep_werror, env)
 
     # Set the target parameters that the compiler will add
-    isa_arg = spec.architecture.target.optimization_flags(compiler)
+    isa_arg = optimization_flags(compiler, spec.target)
     env.set("SPACK_TARGET_ARGS", isa_arg)
 
     # Trap spack-tracked compiler flags as appropriate.
@@ -403,6 +407,65 @@ def set_compiler_environment_variables(pkg, env):
     return env
 
 
+def optimization_flags(compiler, target):
+    if spack.compilers.is_mixed_toolchain(compiler):
+        msg = (
+            "microarchitecture specific optimizations are not "
+            "supported yet on mixed compiler toolchains [check"
+            f" {compiler.name}@{compiler.version} for further details]"
+        )
+        tty.debug(msg)
+        return ""
+
+    # Try to check if the current compiler comes with a version number or
+    # has an unexpected suffix. If so, treat it as a compiler with a
+    # custom spec.
+    compiler_version = compiler.version
+    version_number, suffix = archspec.cpu.version_components(compiler.version)
+    if not version_number or suffix:
+        try:
+            compiler_version = compiler.real_version
+        except spack.util.executable.ProcessError as e:
+            # log this and just return compiler.version instead
+            tty.debug(str(e))
+
+    try:
+        result = target.optimization_flags(compiler.name, compiler_version.dotted_numeric_string)
+    except (ValueError, archspec.cpu.UnsupportedMicroarchitecture):
+        result = ""
+
+    return result
+
+
+class FilterDefaultDynamicLinkerSearchPaths:
+    """Remove rpaths to directories that are default search paths of the dynamic linker."""
+
+    def __init__(self, dynamic_linker: Optional[str]) -> None:
+        # Identify directories by (inode, device) tuple, which handles symlinks too.
+        self.default_path_identifiers: Set[Tuple[int, int]] = set()
+        if not dynamic_linker:
+            return
+        for path in spack.util.libc.default_search_paths_from_dynamic_linker(dynamic_linker):
+            try:
+                s = os.stat(path)
+                if stat.S_ISDIR(s.st_mode):
+                    self.default_path_identifiers.add((s.st_ino, s.st_dev))
+            except OSError:
+                continue
+
+    def is_dynamic_loader_default_path(self, p: str) -> bool:
+        try:
+            s = os.stat(p)
+            return (s.st_ino, s.st_dev) in self.default_path_identifiers
+        except OSError:
+            return False
+
+    def __call__(self, dirs: List[str]) -> List[str]:
+        if not self.default_path_identifiers:
+            return dirs
+        return [p for p in dirs if not self.is_dynamic_loader_default_path(p)]
+
+
 def set_wrapper_variables(pkg, env):
     """Set environment variables used by the Spack compiler wrapper (which have the prefix
     `SPACK_`) and also add the compiler wrappers to PATH.
@@ -460,69 +523,71 @@ def set_wrapper_variables(pkg, env):
         env.set("CCACHE_DISABLE", "1")
 
     # Gather information about various types of dependencies
-    link_deps = set(pkg.spec.traverse(root=False, deptype=("link")))
-    rpath_deps = get_rpath_deps(pkg)
+    rpath_hashes = set(s.dag_hash() for s in get_rpath_deps(pkg))
+    link_deps = pkg.spec.traverse(root=False, order="topo", deptype=dt.LINK)
+    external_link_deps, nonexternal_link_deps = stable_partition(link_deps, lambda d: d.external)
 
     link_dirs = []
     include_dirs = []
     rpath_dirs = []
 
-    def _prepend_all(list_to_modify, items_to_add):
-        # Update the original list (creating a new list would be faster but
-        # may not be convenient)
-        for item in reversed(list(items_to_add)):
-            list_to_modify.insert(0, item)
+    for dep in chain(external_link_deps, nonexternal_link_deps):
+        # TODO: is_system_path is wrong, but even if we knew default -L, -I flags from the compiler
+        # and default search dirs from the dynamic linker, it's not obvious how to avoid a possibly
+        # expensive search in `query.libs.directories` and `query.headers.directories`, which is
+        # what this branch is trying to avoid.
+        if is_system_path(dep.prefix):
+            continue
+        # TODO: as of Spack 0.22, multiple instances of the same package may occur among the link
+        # deps, so keying by name is wrong. In practice it is not problematic: we obtain the same
+        # gcc-runtime / glibc here, and repeatedly add the same dirs that are later deduped.
+        query = pkg.spec[dep.name]
+        dep_link_dirs = []
+        try:
+            # Locating libraries can be time consuming, so log start and finish.
+            tty.debug(f"Collecting libraries for {dep.name}")
+            dep_link_dirs.extend(query.libs.directories)
+            tty.debug(f"Libraries for {dep.name} have been collected.")
+        except NoLibrariesError:
+            tty.debug(f"No libraries found for {dep.name}")
 
-    def update_compiler_args_for_dep(dep):
-        if dep in link_deps and (not is_system_path(dep.prefix)):
-            query = pkg.spec[dep.name]
-            dep_link_dirs = list()
-            try:
-                # In some circumstances (particularly for externals) finding
-                # libraries packages can be time consuming, so indicate that
-                # we are performing this operation (and also report when it
-                # finishes).
-                tty.debug("Collecting libraries for {0}".format(dep.name))
-                dep_link_dirs.extend(query.libs.directories)
-                tty.debug("Libraries for {0} have been collected.".format(dep.name))
-            except NoLibrariesError:
-                tty.debug("No libraries found for {0}".format(dep.name))
+        for default_lib_dir in ("lib", "lib64"):
+            default_lib_prefix = os.path.join(dep.prefix, default_lib_dir)
+            if os.path.isdir(default_lib_prefix):
+                dep_link_dirs.append(default_lib_prefix)
 
-            for default_lib_dir in ["lib", "lib64"]:
-                default_lib_prefix = os.path.join(dep.prefix, default_lib_dir)
-                if os.path.isdir(default_lib_prefix):
-                    dep_link_dirs.append(default_lib_prefix)
+        link_dirs[:0] = dep_link_dirs
+        if dep.dag_hash() in rpath_hashes:
+            rpath_dirs[:0] = dep_link_dirs
 
-            _prepend_all(link_dirs, dep_link_dirs)
-            if dep in rpath_deps:
-                _prepend_all(rpath_dirs, dep_link_dirs)
+        try:
+            tty.debug(f"Collecting headers for {dep.name}")
+            include_dirs[:0] = query.headers.directories
+            tty.debug(f"Headers for {dep.name} have been collected.")
+        except NoHeadersError:
+            tty.debug(f"No headers found for {dep.name}")
 
-            try:
-                _prepend_all(include_dirs, query.headers.directories)
-            except NoHeadersError:
-                tty.debug("No headers found for {0}".format(dep.name))
-
-    for dspec in pkg.spec.traverse(root=False, order="post"):
-        if dspec.external:
-            update_compiler_args_for_dep(dspec)
-
-    # Just above, we prepended entries for -L/-rpath for externals. We
-    # now do this for non-external packages so that Spack-built packages
-    # are searched first for libraries etc.
-    for dspec in pkg.spec.traverse(root=False, order="post"):
-        if not dspec.external:
-            update_compiler_args_for_dep(dspec)
-
-    # The top-level package is always RPATHed. It hasn't been installed yet
-    # so the RPATHs are added unconditionally (e.g. even though lib64/ may
-    # not be created for the install).
-    for libdir in ["lib64", "lib"]:
+    # The top-level package is heuristically rpath'ed.
+    for libdir in ("lib64", "lib"):
         lib_path = os.path.join(pkg.prefix, libdir)
         rpath_dirs.insert(0, lib_path)
 
+    filter_default_dynamic_linker_search_paths = FilterDefaultDynamicLinkerSearchPaths(
+        pkg.compiler.default_dynamic_linker
+    )
+
+    # TODO: filter_system_paths is again wrong (and probably unnecessary due to the is_system_path
+    # branch above). link_dirs should be filtered with entries from _parse_link_paths.
     link_dirs = list(dedupe(filter_system_paths(link_dirs)))
     include_dirs = list(dedupe(filter_system_paths(include_dirs)))
     rpath_dirs = list(dedupe(filter_system_paths(rpath_dirs)))
+    rpath_dirs = filter_default_dynamic_linker_search_paths(rpath_dirs)
+
+    # TODO: implicit_rpaths is prefiltered by is_system_path, that should be removed in favor of
+    # just this filter.
+    implicit_rpaths = filter_default_dynamic_linker_search_paths(pkg.compiler.implicit_rpaths())
+    if implicit_rpaths:
+        env.set("SPACK_COMPILER_IMPLICIT_RPATHS", ":".join(implicit_rpaths))
 
     # Spack managed directories include the stage, store and upstream stores. We extend this with
     # their real paths to make it more robust (e.g. /tmp vs /private/tmp on macOS).
@@ -552,13 +617,11 @@ def set_package_py_globals(pkg, context: Context = Context.BUILD):
     """
     module = ModuleChangePropagator(pkg)
 
-    if context == Context.BUILD:
-        module.std_cmake_args = spack.build_systems.cmake.CMakeBuilder.std_args(pkg)
-        module.std_meson_args = spack.build_systems.meson.MesonBuilder.std_args(pkg)
-        module.std_pip_args = spack.build_systems.python.PythonPipBuilder.std_args(pkg)
-
     jobs = spack.config.determine_number_of_jobs(parallel=pkg.parallel)
     module.make_jobs = jobs
+    if context == Context.BUILD:
+        module.std_meson_args = spack.build_systems.meson.MesonBuilder.std_args(pkg)
+        module.std_pip_args = spack.build_systems.python.PythonPipBuilder.std_args(pkg)
 
     # TODO: make these build deps that can be installed if not found.
     module.make = MakeExecutable("make", jobs)
@@ -727,21 +790,6 @@ def get_rpath_deps(pkg: spack.package_base.PackageBase) -> List[spack.spec.Spec]
     return _get_rpath_deps_from_spec(pkg.spec, pkg.transitive_rpaths)
 
 
-def get_rpaths(pkg):
-    """Get a list of all the rpaths for a package."""
-    rpaths = [pkg.prefix.lib, pkg.prefix.lib64]
-    deps = get_rpath_deps(pkg)
-    rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib))
-    rpaths.extend(d.prefix.lib64 for d in deps if os.path.isdir(d.prefix.lib64))
-    # Second module is our compiler mod name. We use that to get rpaths from
-    # module show output.
-    if pkg.compiler.modules and len(pkg.compiler.modules) > 1:
-        mod_rpath = path_from_modules([pkg.compiler.modules[1]])
-        if mod_rpath:
-            rpaths.append(mod_rpath)
-    return list(dedupe(filter_system_paths(rpaths)))
-
-
 def load_external_modules(pkg):
     """Traverse a package's spec DAG and load any external modules.
 
@@ -783,7 +831,6 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD):
     # Platform specific setup goes before package specific setup. This is for setting
     # defaults like MACOSX_DEPLOYMENT_TARGET on macOS.
     platform = spack.platforms.by_name(pkg.spec.architecture.platform)
-    target = platform.target(pkg.spec.architecture.target)
     platform.setup_platform_environment(pkg, env_mods)
 
     tty.debug("setup_package: grabbing modifications from dependencies")
@@ -808,15 +855,8 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD):
         for mod in pkg.compiler.modules:
             load_module(mod)
 
-    if target and target.module_name:
-        load_module(target.module_name)
-
     load_external_modules(pkg)
 
-    implicit_rpaths = pkg.compiler.implicit_rpaths()
-    if implicit_rpaths:
-        env_mods.set("SPACK_COMPILER_IMPLICIT_RPATHS", ":".join(implicit_rpaths))
-
     # Make sure nothing's strange about the Spack environment.
     validate(env_mods, tty.warn)
     env_mods.apply_modifications()
@@ -1006,6 +1046,12 @@ def set_all_package_py_globals(self):
                     # This includes runtime dependencies, also runtime deps of direct build deps.
                     set_package_py_globals(pkg, context=Context.RUN)
 
+        # Looping over the set of packages a second time
+        # ensures all globals are loaded into the module space prior to
+        # any package setup. This guarantees package setup methods have
+        # access to expected module level definitions such as "spack_cc"
+        for dspec, flag in chain(self.external, self.nonexternal):
+            pkg = dspec.package
             for spec in dspec.dependents():
                 # Note: some specs have dependents that are unreachable from the root, so avoid
                 # setting globals for those.
@@ -1015,6 +1061,15 @@ def set_all_package_py_globals(self):
                 pkg.setup_dependent_package(dependent_module, spec)
                 dependent_module.propagate_changes_to_mro()
 
+        pkg = self.specs[0].package
+        if self.context == Context.BUILD:
+            module = ModuleChangePropagator(pkg)
+            # std_cmake_args is not sufficiently static to be defined
+            # in set_package_py_globals and is deprecated so its handled
+            # here as a special case
+            module.std_cmake_args = spack.build_systems.cmake.CMakeBuilder.std_args(pkg)
+            module.propagate_changes_to_mro()
+
     def get_env_modifications(self) -> EnvironmentModifications:
         """Returns the environment variable modifications for the given input specs and context.
         Environment modifications include:
@@ -1084,35 +1139,52 @@ def _make_runnable(self, dep: spack.spec.Spec, env: EnvironmentModifications):
                 env.prepend_path("PATH", bin_dir)
 
 
-def get_cmake_prefix_path(pkg):
-    # Note that unlike modifications_from_dependencies, this does not include
-    # any edits to CMAKE_PREFIX_PATH defined in custom
-    # setup_dependent_build_environment implementations of dependency packages
-    build_deps = set(pkg.spec.dependencies(deptype=("build", "test")))
-    link_deps = set(pkg.spec.traverse(root=False, deptype=("link")))
-    build_link_deps = build_deps | link_deps
-    spack_built = []
-    externals = []
-    # modifications_from_dependencies updates CMAKE_PREFIX_PATH by first
-    # prepending all externals and then all non-externals
-    for dspec in pkg.spec.traverse(root=False, order="post"):
-        if dspec in build_link_deps:
-            if dspec.external:
-                externals.insert(0, dspec)
-            else:
-                spack_built.insert(0, dspec)
-
-    ordered_build_link_deps = spack_built + externals
-    cmake_prefix_path_entries = []
-    for spec in ordered_build_link_deps:
-        cmake_prefix_path_entries.extend(spec.package.cmake_prefix_paths)
-
-    return filter_system_paths(cmake_prefix_path_entries)
-
-
 def _setup_pkg_and_run(
-    serialized_pkg, function, kwargs, write_pipe, input_multiprocess_fd, jsfd1, jsfd2
+    serialized_pkg: "spack.subprocess_context.PackageInstallContext",
+    function: Callable,
+    kwargs: Dict,
+    write_pipe: multiprocessing.connection.Connection,
+    input_multiprocess_fd: Optional[MultiProcessFd],
+    jsfd1: Optional[MultiProcessFd],
+    jsfd2: Optional[MultiProcessFd],
 ):
+    """Main entry point in the child process for Spack builds.
+
+    ``_setup_pkg_and_run`` is called by the child process created in
+    ``start_build_process()``, and its main job is to run ``function()`` on behalf of
+    some Spack installation (see :ref:`spack.installer.PackageInstaller._install_task`).
+
+    The child process is passed a ``write_pipe``, on which it's expected to send one of
+    the following:
+
+    * ``StopPhase``: error raised by a build process indicating it's stopping at a
+      particular build phase.
+
+    * ``BaseException``: any exception raised by a child build process, which will be
+      wrapped in ``ChildError`` (which adds a bunch of debug info and log context) and
+      raised in the parent.
+
+    * The return value of ``function()``, which can be anything (except an exception).
+      This is returned to the caller.
+
+    Note: ``jsfd1`` and ``jsfd2`` are passed solely to ensure that the child process
+    does not close these file descriptors. Some ``multiprocessing`` backends will close
+    them automatically in the child if they are not passed at process creation time.
+
+    Arguments:
+        serialized_pkg: Spack package install context object (serialized form of the
+            package that we'll build in the child process).
+        function: function to call in the child process; serialized_pkg is passed to
+            this as the first argument.
+        kwargs: additional keyword arguments to pass to ``function()``.
+        write_pipe: multiprocessing ``Connection`` to the parent process, to which the
+            child *must* send a result (or an error) back to parent on.
+        input_multiprocess_fd: stdin from the parent (not passed currently on Windows)
+        jsfd1: gmake Jobserver file descriptor 1.
+        jsfd2: gmake Jobserver file descriptor 2.
+
+    """
+
     context: str = kwargs.get("context", "build")
 
     try:
@@ -1122,7 +1194,7 @@ def _setup_pkg_and_run(
         # that the parent process is not going to read from it till we
         # are done with the child, so we undo Python's precaution.
         if input_multiprocess_fd is not None:
-            sys.stdin = os.fdopen(input_multiprocess_fd.fd)
+            sys.stdin = os.fdopen(input_multiprocess_fd.fd, closefd=False)
 
         pkg = serialized_pkg.restore()
 
@@ -1138,13 +1210,14 @@ def _setup_pkg_and_run(
         # Do not create a full ChildError from this, it's not an error
         # it's a control statement.
         write_pipe.send(e)
-    except BaseException:
+    except BaseException as e:
         # catch ANYTHING that goes wrong in the child process
-        exc_type, exc, tb = sys.exc_info()
 
         # Need to unwind the traceback in the child because traceback
         # objects can't be sent to the parent.
-        tb_string = traceback.format_exc()
+        exc_type = type(e)
+        tb = e.__traceback__
+        tb_string = "".join(traceback.format_exception(exc_type, e, tb))
 
         # build up some context from the offending package so we can
         # show that, too.
@@ -1161,8 +1234,8 @@ def _setup_pkg_and_run(
         elif context == "test":
             logfile = os.path.join(pkg.test_suite.stage, pkg.test_suite.test_log_name(pkg.spec))
 
-        error_msg = str(exc)
-        if isinstance(exc, (spack.multimethod.NoSuchMethodError, AttributeError)):
+        error_msg = str(e)
+        if isinstance(e, (spack.multimethod.NoSuchMethodError, AttributeError)):
             process = "test the installation" if context == "test" else "build from sources"
             error_msg = (
                 "The '{}' package cannot find an attribute while trying to {}. "
@@ -1172,7 +1245,7 @@ def _setup_pkg_and_run(
                 "More information at https://spack.readthedocs.io/en/latest/packaging_guide.html#installation-procedure"
             ).format(pkg.name, process, context)
             error_msg = colorize("@*R{{{}}}".format(error_msg))
-            error_msg = "{}\n\n{}".format(str(exc), error_msg)
+            error_msg = "{}\n\n{}".format(str(e), error_msg)
 
         # make a pickleable exception to send to parent.
         msg = "%s: %s" % (exc_type.__name__, error_msg)

lib/spack/spack/build_systems/aspell_dict.py

@@ -2,10 +2,11 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os
+
 import llnl.util.filesystem as fs
 
 import spack.directives
-import spack.package_base
 import spack.util.executable
 
 from .autotools import AutotoolsBuilder, AutotoolsPackage
@@ -46,18 +47,12 @@ class AspellDictPackage(AutotoolsPackage):
     #: Override the default autotools builder
     AutotoolsBuilder = AspellBuilder
 
-    def view_destination(self, view):
-        aspell_spec = self.spec["aspell"]
-        if view.get_projection_for_spec(aspell_spec) != aspell_spec.prefix:
-            raise spack.package_base.ExtensionError(
-                "aspell does not support non-global extensions"
-            )
-        aspell = aspell_spec.command
-        return aspell("dump", "config", "dict-dir", output=str).strip()
-
-    def view_source(self):
-        return self.prefix.lib
-
     def patch(self):
-        fs.filter_file(r"^dictdir=.*$", "dictdir=/lib", "configure")
-        fs.filter_file(r"^datadir=.*$", "datadir=/lib", "configure")
+        aspell_spec = self.spec["aspell"]
+        aspell = aspell_spec.command
+        dictdir = aspell("dump", "config", "dict-dir", output=str).strip()
+        datadir = aspell("dump", "config", "data-dir", output=str).strip()
+        dictdir = os.path.relpath(dictdir, aspell_spec.prefix)
+        datadir = os.path.relpath(datadir, aspell_spec.prefix)
+        fs.filter_file(r"^dictdir=.*$", f"dictdir=/{dictdir}", "configure")
+        fs.filter_file(r"^datadir=.*$", f"datadir=/{datadir}", "configure")

lib/spack/spack/build_systems/cached_cmake.py

@@ -10,7 +10,6 @@
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 
-import spack.build_environment
 import spack.builder
 
 from .cmake import CMakeBuilder, CMakePackage
@@ -297,18 +296,6 @@ def initconfig_hardware_entries(self):
     def std_initconfig_entries(self):
         cmake_prefix_path_env = os.environ["CMAKE_PREFIX_PATH"]
         cmake_prefix_path = cmake_prefix_path_env.replace(os.pathsep, ";")
-        cmake_rpaths_env = spack.build_environment.get_rpaths(self.pkg)
-        cmake_rpaths_path = ";".join(cmake_rpaths_env)
-        complete_rpath_list = cmake_rpaths_path
-        if "SPACK_COMPILER_EXTRA_RPATHS" in os.environ:
-            spack_extra_rpaths_env = os.environ["SPACK_COMPILER_EXTRA_RPATHS"]
-            spack_extra_rpaths_path = spack_extra_rpaths_env.replace(os.pathsep, ";")
-            complete_rpath_list = "{0};{1}".format(complete_rpath_list, spack_extra_rpaths_path)
-
-        if "SPACK_COMPILER_IMPLICIT_RPATHS" in os.environ:
-            spack_implicit_rpaths_env = os.environ["SPACK_COMPILER_IMPLICIT_RPATHS"]
-            spack_implicit_rpaths_path = spack_implicit_rpaths_env.replace(os.pathsep, ";")
-            complete_rpath_list = "{0};{1}".format(complete_rpath_list, spack_implicit_rpaths_path)
 
         return [
             "#------------------{0}".format("-" * 60),
@@ -318,8 +305,6 @@ def std_initconfig_entries(self):
             "#------------------{0}\n".format("-" * 60),
             cmake_cache_string("CMAKE_PREFIX_PATH", cmake_prefix_path),
             cmake_cache_string("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON"),
-            cmake_cache_string("CMAKE_BUILD_RPATH", complete_rpath_list),
-            cmake_cache_string("CMAKE_INSTALL_RPATH", complete_rpath_list),
             self.define_cmake_cache_from_variant("CMAKE_BUILD_TYPE", "build_type"),
         ]
 

lib/spack/spack/build_systems/cmake.py

@@ -8,17 +8,19 @@
 import platform
 import re
 import sys
-from typing import List, Optional, Tuple
+from itertools import chain
+from typing import List, Optional, Set, Tuple
 
 import llnl.util.filesystem as fs
+from llnl.util.lang import stable_partition
 
-import spack.build_environment
 import spack.builder
 import spack.deptypes as dt
 import spack.error
 import spack.package_base
 from spack.directives import build_system, conflicts, depends_on, variant
 from spack.multimethod import when
+from spack.util.environment import filter_system_paths
 
 from ._checks import BaseBuilder, execute_build_time_tests
 
@@ -152,6 +154,24 @@ def _values(x):
         conflicts(f"generator={x}")
 
 
+def get_cmake_prefix_path(pkg: spack.package_base.PackageBase) -> List[str]:
+    """Obtain the CMAKE_PREFIX_PATH entries for a package, based on the cmake_prefix_path package
+    attribute of direct build/test and transitive link dependencies."""
+    # Add direct build/test deps
+    selected: Set[str] = {s.dag_hash() for s in pkg.spec.dependencies(deptype=dt.BUILD | dt.TEST)}
+    # Add transitive link deps
+    selected.update(s.dag_hash() for s in pkg.spec.traverse(root=False, deptype=dt.LINK))
+    # Separate out externals so they do not shadow Spack prefixes
+    externals, spack_built = stable_partition(
+        (s for s in pkg.spec.traverse(root=False, order="topo") if s.dag_hash() in selected),
+        lambda x: x.external,
+    )
+
+    return filter_system_paths(
+        path for spec in chain(spack_built, externals) for path in spec.package.cmake_prefix_paths
+    )
+
+
 class CMakePackage(spack.package_base.PackageBase):
     """Specialized class for packages built using CMake
 
@@ -358,6 +378,16 @@ def std_args(pkg, generator=None):
             "-G",
             generator,
             define("CMAKE_INSTALL_PREFIX", pathlib.Path(pkg.prefix).as_posix()),
+            define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", True),
+            # only include the install prefix lib dirs; rpaths for deps are added by USE_LINK_PATH
+            define(
+                "CMAKE_INSTALL_RPATH",
+                [
+                    pathlib.Path(pkg.prefix, "lib").as_posix(),
+                    pathlib.Path(pkg.prefix, "lib64").as_posix(),
+                ],
+            ),
+            define("CMAKE_PREFIX_PATH", get_cmake_prefix_path(pkg)),
             define("CMAKE_BUILD_TYPE", build_type),
         ]
 
@@ -372,15 +402,6 @@ def std_args(pkg, generator=None):
         _conditional_cmake_defaults(pkg, args)
         _maybe_set_python_hints(pkg, args)
 
-        # Set up CMake rpath
-        args.extend(
-            [
-                define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", True),
-                define("CMAKE_INSTALL_RPATH", spack.build_environment.get_rpaths(pkg)),
-                define("CMAKE_PREFIX_PATH", spack.build_environment.get_cmake_prefix_path(pkg)),
-            ]
-        )
-
         return args
 
     @staticmethod
@@ -541,6 +562,13 @@ def cmake_args(self):
 
     def cmake(self, pkg, spec, prefix):
         """Runs ``cmake`` in the build directory"""
+
+        # skip cmake phase if it is an incremental develop build
+        if spec.is_develop and os.path.isfile(
+            os.path.join(self.build_directory, "CMakeCache.txt")
+        ):
+            return
+
         options = self.std_cmake_args
         options += self.cmake_args()
         options.append(os.path.abspath(self.root_cmakelists_dir))

lib/spack/spack/build_systems/cuda.py

@@ -110,8 +110,8 @@ def compute_capabilities(arch_list: Iterable[str]) -> List[str]:
 
     depends_on("cuda@5.0:10.2", when="cuda_arch=30")
     depends_on("cuda@5.0:10.2", when="cuda_arch=32")
-    depends_on("cuda@5.0:", when="cuda_arch=35")
-    depends_on("cuda@6.5:", when="cuda_arch=37")
+    depends_on("cuda@5.0:11.8", when="cuda_arch=35")
+    depends_on("cuda@6.5:11.8", when="cuda_arch=37")
 
     depends_on("cuda@6.0:", when="cuda_arch=50")
     depends_on("cuda@6.5:", when="cuda_arch=52")
@@ -131,6 +131,7 @@ def compute_capabilities(arch_list: Iterable[str]) -> List[str]:
     depends_on("cuda@11.8:", when="cuda_arch=89")
 
     depends_on("cuda@12.0:", when="cuda_arch=90")
+    depends_on("cuda@12.0:", when="cuda_arch=90a")
 
     # From the NVIDIA install guide we know of conflicts for particular
     # platforms (linux, darwin), architectures (x86, powerpc) and compilers
@@ -149,7 +150,6 @@ def compute_capabilities(arch_list: Iterable[str]) -> List[str]:
         # minimum supported versions
         conflicts("%gcc@:4", when="+cuda ^cuda@11.0:")
         conflicts("%gcc@:5", when="+cuda ^cuda@11.4:")
-        conflicts("%gcc@:7.2", when="+cuda ^cuda@12.4:")
         conflicts("%clang@:6", when="+cuda ^cuda@12.2:")
 
         # maximum supported version
@@ -241,6 +241,11 @@ def compute_capabilities(arch_list: Iterable[str]) -> List[str]:
         conflicts("%intel@19.2:", when="+cuda ^cuda@:11.1.0")
         conflicts("%intel@2021:", when="+cuda ^cuda@:11.4.0")
 
+        # ARM
+        # https://github.com/spack/spack/pull/39666#issuecomment-2377609263
+        # Might need to be expanded to other gcc versions
+        conflicts("%gcc@13.2.0", when="+cuda ^cuda@:12.4 target=aarch64:")
+
         # XL is mostly relevant for ppc64le Linux
         conflicts("%xl@:12,14:", when="+cuda ^cuda@:9.1")
         conflicts("%xl@:12,14:15,17:", when="+cuda ^cuda@9.2")

lib/spack/spack/build_systems/go.py

@@ -44,16 +44,27 @@ class GoBuilder(BaseBuilder):
         +-----------------------------------------------+--------------------+
         | **Method**                                    | **Purpose**        |
         +===============================================+====================+
-        | :py:meth:`~.GoBuilder.build_args`             | Specify arguments  |
+        | :py:attr:`~.GoBuilder.build_args`             | Specify arguments  |
         |                                               | to ``go build``    |
         +-----------------------------------------------+--------------------+
-        | :py:meth:`~.GoBuilder.check_args`             | Specify arguments  |
+        | :py:attr:`~.GoBuilder.check_args`             | Specify arguments  |
         |                                               | to ``go test``     |
         +-----------------------------------------------+--------------------+
     """
 
     phases = ("build", "install")
 
+    #: Names associated with package methods in the old build-system format
+    legacy_methods = ("check", "installcheck")
+
+    #: Names associated with package attributes in the old build-system format
+    legacy_attributes = (
+        "build_args",
+        "check_args",
+        "build_directory",
+        "install_time_test_callbacks",
+    )
+
     #: Callback names for install-time test
     install_time_test_callbacks = ["check"]
 

lib/spack/spack/build_systems/python.py

@@ -339,7 +339,7 @@ class PythonPackage(PythonExtension):
     legacy_buildsystem = "python_pip"
 
     #: Callback names for install-time test
-    install_time_test_callbacks = ["test"]
+    install_time_test_callbacks = ["test_imports"]
 
     build_system("python_pip")
 
@@ -429,7 +429,7 @@ class PythonPipBuilder(BaseBuilder):
     phases = ("install",)
 
     #: Names associated with package methods in the old build-system format
-    legacy_methods = ("test",)
+    legacy_methods = ("test_imports",)
 
     #: Same as legacy_methods, but the signature is different
     legacy_long_methods = ("install_options", "global_options", "config_settings")
@@ -438,7 +438,7 @@ class PythonPipBuilder(BaseBuilder):
     legacy_attributes = ("archive_files", "build_directory", "install_time_test_callbacks")
 
     #: Callback names for install-time test
-    install_time_test_callbacks = ["test"]
+    install_time_test_callbacks = ["test_imports"]
 
     @staticmethod
     def std_args(cls) -> List[str]:

lib/spack/spack/builder.py

@@ -521,10 +521,6 @@ def stage(self):
     def prefix(self):
         return self.pkg.prefix
 
-    def test(self):
-        # Defer tests to virtual and concrete packages
-        pass
-
     def setup_build_environment(self, env):
         """Sets up the build environment for a package.
 

lib/spack/spack/ci.py

@@ -10,6 +10,7 @@
 import os
 import re
 import shutil
+import ssl
 import stat
 import subprocess
 import sys
@@ -19,14 +20,14 @@
 from collections import defaultdict, namedtuple
 from typing import Dict, List, Optional, Set, Tuple
 from urllib.error import HTTPError, URLError
-from urllib.parse import urlencode
-from urllib.request import HTTPHandler, Request, build_opener
+from urllib.parse import quote, urlencode, urlparse
+from urllib.request import HTTPHandler, HTTPSHandler, Request, build_opener
 
 import ruamel.yaml
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
-from llnl.util.lang import memoized
+from llnl.util.lang import Singleton, memoized
 from llnl.util.tty.color import cescape, colorize
 
 import spack
@@ -50,6 +51,31 @@
 from spack.reporters.cdash import SPACK_CDASH_TIMEOUT
 from spack.reporters.cdash import build_stamp as cdash_build_stamp
 
+
+def _urlopen():
+    error_handler = web_util.SpackHTTPDefaultErrorHandler()
+
+    # One opener with HTTPS ssl enabled
+    with_ssl = build_opener(
+        HTTPHandler(), HTTPSHandler(context=web_util.ssl_create_default_context()), error_handler
+    )
+
+    # One opener with HTTPS ssl disabled
+    without_ssl = build_opener(
+        HTTPHandler(), HTTPSHandler(context=ssl._create_unverified_context()), error_handler
+    )
+
+    # And dynamically dispatch based on the config:verify_ssl.
+    def dispatch_open(fullurl, data=None, timeout=None, verify_ssl=True):
+        opener = with_ssl if verify_ssl else without_ssl
+        timeout = timeout or spack.config.get("config:connect_timeout", 1)
+        return opener.open(fullurl, data, timeout)
+
+    return dispatch_open
+
+
+_dyn_mapping_urlopener = Singleton(_urlopen)
+
 # See https://docs.gitlab.com/ee/ci/yaml/#retry for descriptions of conditions
 JOB_RETRY_CONDITIONS = [
     # "always",
@@ -405,9 +431,20 @@ def __init__(self, ci_config, spec_labels, stages):
             if name not in ["any", "build"]:
                 jobs[name] = self.__init_job("")
 
-    def __init_job(self, spec):
+    def __init_job(self, release_spec):
         """Initialize job object"""
-        return {"spec": spec, "attributes": {}}
+        job_object = {"spec": release_spec, "attributes": {}}
+        if release_spec:
+            job_vars = job_object["attributes"].setdefault("variables", {})
+            job_vars["SPACK_JOB_SPEC_DAG_HASH"] = release_spec.dag_hash()
+            job_vars["SPACK_JOB_SPEC_PKG_NAME"] = release_spec.name
+            job_vars["SPACK_JOB_SPEC_PKG_VERSION"] = release_spec.format("{version}")
+            job_vars["SPACK_JOB_SPEC_COMPILER_NAME"] = release_spec.format("{compiler.name}")
+            job_vars["SPACK_JOB_SPEC_COMPILER_VERSION"] = release_spec.format("{compiler.version}")
+            job_vars["SPACK_JOB_SPEC_ARCH"] = release_spec.format("{architecture}")
+            job_vars["SPACK_JOB_SPEC_VARIANTS"] = release_spec.format("{variants}")
+
+        return job_object
 
     def __is_named(self, section):
         """Check if a pipeline-gen configuration section is for a named job,
@@ -500,6 +537,7 @@ def generate_ir(self):
         for section in reversed(pipeline_gen):
             name = self.__is_named(section)
             has_submapping = "submapping" in section
+            has_dynmapping = "dynamic-mapping" in section
             section = cfg.InternalConfigScope._process_dict_keyname_overrides(section)
 
             if name:
@@ -542,6 +580,108 @@ def _apply_section(dest, src):
                         job["attributes"] = self.__apply_submapping(
                             job["attributes"], job["spec"], section
                         )
+            elif has_dynmapping:
+                mapping = section["dynamic-mapping"]
+
+                dynmap_name = mapping.get("name")
+
+                # Check if this section should be skipped
+                dynmap_skip = os.environ.get("SPACK_CI_SKIP_DYNAMIC_MAPPING")
+                if dynmap_name and dynmap_skip:
+                    if re.match(dynmap_skip, dynmap_name):
+                        continue
+
+                # Get the endpoint
+                endpoint = mapping["endpoint"]
+                endpoint_url = urlparse(endpoint)
+
+                # Configure the request header
+                header = {"User-Agent": web_util.SPACK_USER_AGENT}
+                header.update(mapping.get("header", {}))
+
+                # Expand header environment variables
+                # ie. if tokens are passed
+                for value in header.values():
+                    value = os.path.expandvars(value)
+
+                verify_ssl = mapping.get("verify_ssl", spack.config.get("config:verify_ssl", True))
+                timeout = mapping.get("timeout", spack.config.get("config:connect_timeout", 1))
+
+                required = mapping.get("require", [])
+                allowed = mapping.get("allow", [])
+                ignored = mapping.get("ignore", [])
+
+                # required keys are implicitly allowed
+                allowed = sorted(set(allowed + required))
+                ignored = sorted(set(ignored))
+                required = sorted(set(required))
+
+                # Make sure required things are not also ignored
+                assert not any([ikey in required for ikey in ignored])
+
+                def job_query(job):
+                    job_vars = job["attributes"]["variables"]
+                    query = (
+                        "{SPACK_JOB_SPEC_PKG_NAME}@{SPACK_JOB_SPEC_PKG_VERSION}"
+                        # The preceding spaces are required (ref. https://github.com/spack/spack-gantry/blob/develop/docs/api.md#allocation)
+                        " {SPACK_JOB_SPEC_VARIANTS}"
+                        " arch={SPACK_JOB_SPEC_ARCH}"
+                        "%{SPACK_JOB_SPEC_COMPILER_NAME}@{SPACK_JOB_SPEC_COMPILER_VERSION}"
+                    ).format_map(job_vars)
+                    return f"spec={quote(query)}"
+
+                for job in jobs.values():
+                    if not job["spec"]:
+                        continue
+
+                    # Create request for this job
+                    query = job_query(job)
+                    request = Request(
+                        endpoint_url._replace(query=query).geturl(), headers=header, method="GET"
+                    )
+                    try:
+                        response = _dyn_mapping_urlopener(
+                            request, verify_ssl=verify_ssl, timeout=timeout
+                        )
+                    except Exception as e:
+                        # For now just ignore any errors from dynamic mapping and continue
+                        # This is still experimental, and failures should not stop CI
+                        # from running normally
+                        tty.warn(f"Failed to fetch dynamic mapping for query:\n\t{query}")
+                        tty.warn(f"{e}")
+                        continue
+
+                    config = json.load(codecs.getreader("utf-8")(response))
+
+                    # Strip ignore keys
+                    if ignored:
+                        for key in ignored:
+                            if key in config:
+                                config.pop(key)
+
+                    # Only keep allowed keys
+                    clean_config = {}
+                    if allowed:
+                        for key in allowed:
+                            if key in config:
+                                clean_config[key] = config[key]
+                    else:
+                        clean_config = config
+
+                    # Verify all of the required keys are present
+                    if required:
+                        missing_keys = []
+                        for key in required:
+                            if key not in clean_config.keys():
+                                missing_keys.append(key)
+
+                        if missing_keys:
+                            tty.warn(f"Response missing required keys: {missing_keys}")
+
+                    if clean_config:
+                        job["attributes"] = spack.config.merge_yaml(
+                            job.get("attributes", {}), clean_config
+                        )
 
         for _, job in jobs.items():
             if job["spec"]:
@@ -952,15 +1092,6 @@ def main_script_replacements(cmd):
 
             job_name = get_job_name(release_spec, build_group)
 
-            job_vars = job_object.setdefault("variables", {})
-            job_vars["SPACK_JOB_SPEC_DAG_HASH"] = release_spec_dag_hash
-            job_vars["SPACK_JOB_SPEC_PKG_NAME"] = release_spec.name
-            job_vars["SPACK_JOB_SPEC_PKG_VERSION"] = release_spec.format("{version}")
-            job_vars["SPACK_JOB_SPEC_COMPILER_NAME"] = release_spec.format("{compiler.name}")
-            job_vars["SPACK_JOB_SPEC_COMPILER_VERSION"] = release_spec.format("{compiler.version}")
-            job_vars["SPACK_JOB_SPEC_ARCH"] = release_spec.format("{architecture}")
-            job_vars["SPACK_JOB_SPEC_VARIANTS"] = release_spec.format("{variants}")
-
             job_object["needs"] = []
             if spec_label in dependencies:
                 if enable_artifacts_buildcache:
@@ -1038,6 +1169,7 @@ def main_script_replacements(cmd):
 
             # Let downstream jobs know whether the spec needed rebuilding, regardless
             # whether DAG pruning was enabled or not.
+            job_vars = job_object["variables"]
             job_vars["SPACK_SPEC_NEEDS_REBUILD"] = str(rebuild_spec)
 
             if cdash_handler:
@@ -1272,7 +1404,9 @@ def main_script_replacements(cmd):
     else:
         # No jobs were generated
         noop_job = spack_ci_ir["jobs"]["noop"]["attributes"]
-        noop_job["retry"] = service_job_retries
+        # If this job fails ignore the status and carry on
+        noop_job["retry"] = 0
+        noop_job["allow_failure"] = True
 
         if copy_only_pipeline and config_deprecated:
             tty.debug("Generating no-op job as copy-only is unsupported here.")

lib/spack/spack/cmd/arch.py

@@ -19,12 +19,23 @@
 
 
 def setup_parser(subparser):
+    # DEPRECATED: equivalent to --generic --target
     subparser.add_argument(
-        "-g", "--generic-target", action="store_true", help="show the best generic target"
+        "-g",
+        "--generic-target",
+        action="store_true",
+        help="show the best generic target (deprecated)",
     )
     subparser.add_argument(
         "--known-targets", action="store_true", help="show a list of all known targets and exit"
     )
+    target_type = subparser.add_mutually_exclusive_group()
+    target_type.add_argument(
+        "--family", action="store_true", help="print generic ISA (x86_64, aarch64, ppc64le, ...)"
+    )
+    target_type.add_argument(
+        "--generic", action="store_true", help="print feature level (x86_64_v3, armv8.4a, ...)"
+    )
     parts = subparser.add_mutually_exclusive_group()
     parts2 = subparser.add_mutually_exclusive_group()
     parts.add_argument(
@@ -80,6 +91,7 @@ def display_target_group(header, target_group):
 
 def arch(parser, args):
     if args.generic_target:
+        # TODO: add deprecation warning in 0.24
         print(archspec.cpu.host().generic)
         return
 
@@ -96,6 +108,10 @@ def arch(parser, args):
     host_platform = spack.platforms.host()
     host_os = host_platform.operating_system(os_args)
     host_target = host_platform.target(target_args)
+    if args.family:
+        host_target = host_target.family
+    elif args.generic:
+        host_target = host_target.generic
     architecture = spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))
 
     if args.platform:

lib/spack/spack/cmd/common/arguments.py

@@ -660,34 +660,32 @@ def mirror_name_or_url(m):
     # accidentally to a dir in the current working directory.
 
     # If there's a \ or / in the name, it's interpreted as a path or url.
-    if "/" in m or "\\" in m:
+    if "/" in m or "\\" in m or m in (".", ".."):
         return spack.mirror.Mirror(m)
 
     # Otherwise, the named mirror is required to exist.
     try:
         return spack.mirror.require_mirror_name(m)
     except ValueError as e:
-        raise argparse.ArgumentTypeError(
-            str(e) + ". Did you mean {}?".format(os.path.join(".", m))
-        )
+        raise argparse.ArgumentTypeError(f"{e}. Did you mean {os.path.join('.', m)}?") from e
 
 
 def mirror_url(url):
     try:
         return spack.mirror.Mirror.from_url(url)
     except ValueError as e:
-        raise argparse.ArgumentTypeError(str(e))
+        raise argparse.ArgumentTypeError(str(e)) from e
 
 
 def mirror_directory(path):
     try:
         return spack.mirror.Mirror.from_local_path(path)
     except ValueError as e:
-        raise argparse.ArgumentTypeError(str(e))
+        raise argparse.ArgumentTypeError(str(e)) from e
 
 
 def mirror_name(name):
     try:
         return spack.mirror.require_mirror_name(name)
     except ValueError as e:
-        raise argparse.ArgumentTypeError(str(e))
+        raise argparse.ArgumentTypeError(str(e)) from e

lib/spack/spack/cmd/deprecate.py

@@ -20,6 +20,7 @@
 
 import spack.cmd
 import spack.environment as ev
+import spack.installer
 import spack.store
 from spack.cmd.common import arguments
 from spack.database import InstallStatuses
@@ -142,4 +143,4 @@ def deprecate(parser, args):
             tty.die("Will not deprecate any packages.")
 
     for dcate, dcator in zip(all_deprecate, all_deprecators):
-        dcate.package.do_deprecate(dcator, symlink)
+        spack.installer.deprecate(dcate, dcator, symlink)

lib/spack/spack/cmd/dev_build.py

@@ -14,6 +14,7 @@
 import spack.config
 import spack.repo
 from spack.cmd.common import arguments
+from spack.installer import PackageInstaller
 
 description = "developer build: build from code in current working directory"
 section = "build"
@@ -131,9 +132,9 @@ def dev_build(self, args):
     elif args.test == "root":
         tests = [spec.name for spec in specs]
 
-    spec.package.do_install(
+    PackageInstaller(
+        [spec.package],
         tests=tests,
-        make_jobs=args.jobs,
         keep_prefix=args.keep_prefix,
         install_deps=not args.ignore_deps,
         verbose=not args.quiet,
@@ -141,7 +142,7 @@ def dev_build(self, args):
         stop_before=args.before,
         skip_patch=args.skip_patch,
         stop_at=args.until,
-    )
+    ).install()
 
     # drop into the build environment of the package?
     if args.shell is not None:

lib/spack/spack/cmd/develop.py

@@ -85,8 +85,14 @@ def _retrieve_develop_source(spec: spack.spec.Spec, abspath: str) -> None:
 
 
 def develop(parser, args):
+    # Note: we could put develop specs in any scope, but I assume
+    # users would only ever want to do this for either (a) an active
+    # env or (b) a specified config file (e.g. that is included by
+    # an environment)
+    # TODO: when https://github.com/spack/spack/pull/35307 is merged,
+    # an active env is not required if a scope is specified
+    env = spack.cmd.require_active_env(cmd_name="develop")
     if not args.spec:
-        env = spack.cmd.require_active_env(cmd_name="develop")
         if args.clone is False:
             raise SpackError("No spec provided to spack develop command")
 
@@ -116,16 +122,18 @@ def develop(parser, args):
         raise SpackError("spack develop requires at most one named spec")
 
     spec = specs[0]
+
     version = spec.versions.concrete_range_as_version
     if not version:
-        raise SpackError("Packages to develop must have a concrete version")
+        # look up the maximum version so infintiy versions are preferred for develop
+        version = max(spec.package_class.versions.keys())
+        tty.msg(f"Defaulting to highest version: {spec.name}@{version}")
     spec.versions = spack.version.VersionList([version])
 
     # If user does not specify --path, we choose to create a directory in the
     # active environment's directory, named after the spec
     path = args.path or spec.name
     if not os.path.isabs(path):
-        env = spack.cmd.require_active_env(cmd_name="develop")
         abspath = spack.util.path.canonicalize_path(path, default_wd=env.path)
     else:
         abspath = path
@@ -149,13 +157,6 @@ def develop(parser, args):
 
         _retrieve_develop_source(spec, abspath)
 
-    # Note: we could put develop specs in any scope, but I assume
-    # users would only ever want to do this for either (a) an active
-    # env or (b) a specified config file (e.g. that is included by
-    # an environment)
-    # TODO: when https://github.com/spack/spack/pull/35307 is merged,
-    # an active env is not required if a scope is specified
-    env = spack.cmd.require_active_env(cmd_name="develop")
     tty.debug("Updating develop config for {0} transactionally".format(env.name))
     with env.write_transaction():
         if args.build_directory is not None:

lib/spack/spack/cmd/env.py

@@ -270,7 +270,7 @@ def env_activate_setup_parser(subparser):
         nargs="?",
         default=None,
         help=(
-            "name of managed environment or directory of the anonymous env"
+            "name of managed environment or directory of the independent env"
             " (when using --dir/-d) to activate"
         ),
     )
@@ -540,7 +540,7 @@ def env_rename_setup_parser(subparser):
 def env_rename(args):
     """Rename an environment.
 
-    This renames a managed environment or moves an anonymous environment.
+    This renames a managed environment or moves an independent environment.
     """
 
     # Directory option has been specified

lib/spack/spack/cmd/find.py

@@ -174,9 +174,9 @@ def query_arguments(args):
     if (args.missing or args.only_missing) and not args.only_deprecated:
         installed.append(InstallStatuses.MISSING)
 
-    known = any
+    predicate_fn = None
     if args.unknown:
-        known = False
+        predicate_fn = lambda x: not spack.repo.PATH.exists(x.spec.name)
 
     explicit = any
     if args.explicit:
@@ -184,7 +184,7 @@ def query_arguments(args):
     if args.implicit:
         explicit = False
 
-    q_args = {"installed": installed, "known": known, "explicit": explicit}
+    q_args = {"installed": installed, "predicate_fn": predicate_fn, "explicit": explicit}
 
     install_tree = args.install_tree
     upstreams = spack.config.get("upstreams", {})

lib/spack/spack/cmd/gc.py

@@ -41,7 +41,7 @@ def setup_parser(subparser):
         help="do not remove installed build-only dependencies of roots\n"
         "(default is to keep only link & run dependencies)",
     )
-    spack.cmd.common.arguments.add_common_arguments(subparser, ["yes_to_all"])
+    spack.cmd.common.arguments.add_common_arguments(subparser, ["yes_to_all", "constraint"])
 
 
 def roots_from_environments(args, active_env):
@@ -97,6 +97,12 @@ def gc(parser, args):
             root_hashes = None
 
         specs = spack.store.STORE.db.unused_specs(root_hashes=root_hashes, deptype=deptype)
+
+        # limit search to constraint specs if provided
+        if args.constraint:
+            hashes = set(spec.dag_hash() for spec in args.specs())
+            specs = [spec for spec in specs if spec.dag_hash() in hashes]
+
         if not specs:
             tty.msg("There are no unused specs. Spack's store is clean.")
             return

lib/spack/spack/cmd/info.py

@@ -17,7 +17,6 @@
 import spack.repo
 import spack.spec
 import spack.variant
-import spack.version
 from spack.cmd.common import arguments
 from spack.package_base import preferred_version
 

lib/spack/spack/cmd/install.py

@@ -474,5 +474,5 @@ def install_without_active_env(args, install_kwargs, reporter_factory):
 
         installs = [s.package for s in concrete_specs]
         install_kwargs["explicit"] = [s.dag_hash() for s in concrete_specs]
-        builder = PackageInstaller(installs, install_kwargs)
+        builder = PackageInstaller(installs, **install_kwargs)
         builder.install()

lib/spack/spack/cmd/modules/__init__.py

@@ -173,7 +173,7 @@ def loads(module_type, specs, args, out=None):
     modules = list(
         (
             spec,
-            spack.modules.common.get_module(
+            spack.modules.get_module(
                 module_type,
                 spec,
                 get_full_path=False,
@@ -222,7 +222,7 @@ def find(module_type, specs, args):
 
     try:
         modules = [
-            spack.modules.common.get_module(
+            spack.modules.get_module(
                 module_type,
                 spec,
                 args.full_path,
@@ -233,7 +233,7 @@ def find(module_type, specs, args):
         ]
 
         modules.append(
-            spack.modules.common.get_module(
+            spack.modules.get_module(
                 module_type,
                 single_spec,
                 args.full_path,
@@ -378,7 +378,10 @@ def refresh(module_type, specs, args):
 def modules_cmd(parser, args, module_type, callbacks=callbacks):
     # Qualifiers to be used when querying the db for specs
     constraint_qualifiers = {
-        "refresh": {"installed": True, "known": lambda x: not spack.repo.PATH.exists(x)}
+        "refresh": {
+            "installed": True,
+            "predicate_fn": lambda x: spack.repo.PATH.exists(x.spec.name),
+        }
     }
     query_args = constraint_qualifiers.get(args.subparser_name, {})
 

lib/spack/spack/cmd/view.py

@@ -33,6 +33,8 @@
 YamlFilesystemView.
 
 """
+import sys
+
 import llnl.util.tty as tty
 from llnl.util.link_tree import MergeConflictError
 
@@ -178,7 +180,12 @@ def setup_parser(sp):
 
 
 def view(parser, args):
-    "Produce a view of a set of packages."
+    """Produce a view of a set of packages."""
+
+    if sys.platform == "win32" and args.action in ("hardlink", "hard"):
+        # Hard-linked views are not yet allowed on Windows.
+        # See https://github.com/spack/spack/pull/46335#discussion_r1757411915
+        tty.die("Hard linking is not supported on Windows. Please use symlinks or copy methods.")
 
     specs = spack.cmd.parse_specs(args.specs)
     path = args.path[0]

lib/spack/spack/compiler.py

@@ -202,18 +202,6 @@ class Compiler:
     support for specific compilers, their possible names, arguments,
     and how to identify the particular type of compiler."""
 
-    # Subclasses use possible names of C compiler
-    cc_names: List[str] = []
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names: List[str] = []
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names: List[str] = []
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names: List[str] = []
-
     # Optional prefix regexes for searching for this type of compiler.
     # Prefixes are sometimes used for toolchains
     prefixes: List[str] = []
@@ -427,14 +415,19 @@ def implicit_rpaths(self) -> List[str]:
         return list(paths_containing_libs(link_dirs, all_required_libs))
 
     @property
-    def default_libc(self) -> Optional["spack.spec.Spec"]:
-        """Determine libc targeted by the compiler from link line"""
+    def default_dynamic_linker(self) -> Optional[str]:
+        """Determine default dynamic linker from compiler link line"""
         output = self.compiler_verbose_output
 
         if not output:
             return None
 
-        dynamic_linker = spack.util.libc.parse_dynamic_linker(output)
+        return spack.util.libc.parse_dynamic_linker(output)
+
+    @property
+    def default_libc(self) -> Optional["spack.spec.Spec"]:
+        """Determine libc targeted by the compiler from link line"""
+        dynamic_linker = self.default_dynamic_linker
 
         if not dynamic_linker:
             return None
@@ -619,18 +612,6 @@ def extract_version_from_output(cls, output):
     def cc_version(cls, cc):
         return cls.default_version(cc)
 
-    @classmethod
-    def cxx_version(cls, cxx):
-        return cls.default_version(cxx)
-
-    @classmethod
-    def f77_version(cls, f77):
-        return cls.default_version(f77)
-
-    @classmethod
-    def fc_version(cls, fc):
-        return cls.default_version(fc)
-
     @classmethod
     def search_regexps(cls, language):
         # Compile all the regular expressions used for files beforehand.

lib/spack/spack/compilers/__init__.py

@@ -8,6 +8,7 @@
 """
 import importlib
 import os
+import re
 import sys
 import warnings
 from typing import Dict, List, Optional
@@ -557,7 +558,7 @@ def get_compilers(config, cspec=None, arch_spec=None):
         except KeyError:
             # TODO: Check if this exception handling makes sense, or if we
             # TODO: need to change / refactor tests
-            family = arch_spec.target
+            family = str(arch_spec.target)
         except AttributeError:
             assert arch_spec is None
 
@@ -631,37 +632,34 @@ def is_mixed_toolchain(compiler):
     Args:
         compiler (spack.compiler.Compiler): a valid compiler object
     """
-    cc = os.path.basename(compiler.cc or "")
-    cxx = os.path.basename(compiler.cxx or "")
-    f77 = os.path.basename(compiler.f77 or "")
-    fc = os.path.basename(compiler.fc or "")
+    import spack.detection.path
+
+    executables = [
+        os.path.basename(compiler.cc or ""),
+        os.path.basename(compiler.cxx or ""),
+        os.path.basename(compiler.f77 or ""),
+        os.path.basename(compiler.fc or ""),
+    ]
 
     toolchains = set()
-    for compiler_cls in all_compiler_types():
-        # Inspect all the compiler toolchain we know. If a compiler is the
-        # only compiler supported there it belongs to that toolchain.
-        def name_matches(name, name_list):
-            # This is such that 'gcc' matches variations
-            # like 'ggc-9' etc that are found in distros
-            name, _, _ = name.partition("-")
-            return len(name_list) == 1 and name and name in name_list
+    finder = spack.detection.path.ExecutablesFinder()
 
-        if any(
-            [
-                name_matches(cc, compiler_cls.cc_names),
-                name_matches(cxx, compiler_cls.cxx_names),
-                name_matches(f77, compiler_cls.f77_names),
-                name_matches(fc, compiler_cls.fc_names),
-            ]
-        ):
-            tty.debug("[TOOLCHAIN] MATCH {0}".format(compiler_cls.__name__))
-            toolchains.add(compiler_cls.__name__)
+    for pkg_name in spack.repo.PATH.packages_with_tags(COMPILER_TAG):
+        pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
+        patterns = finder.search_patterns(pkg=pkg_cls)
+        if not patterns:
+            continue
+        joined_pattern = re.compile(r"|".join(patterns))
+
+        if any(joined_pattern.search(exe) for exe in executables):
+            tty.debug(f"[TOOLCHAIN] MATCH {pkg_name}")
+            toolchains.add(pkg_name)
 
     if len(toolchains) > 1:
         if (
-            toolchains == set(["Clang", "AppleClang", "Aocc"])
+            toolchains == {"llvm", "apple-clang", "aocc"}
             # Msvc toolchain uses Intel ifx
-            or toolchains == set(["Msvc", "Dpcpp", "Oneapi"])
+            or toolchains == {"msvc", "intel-oneapi-compilers"}
         ):
             return False
         tty.debug("[TOOLCHAINS] {0}".format(toolchains))
@@ -805,12 +803,11 @@ def _extract_os_and_target(spec: "spack.spec.Spec"):
         if not spec.architecture:
             host_platform = spack.platforms.host()
             operating_system = host_platform.operating_system("default_os")
-            target = host_platform.target("default_target").microarchitecture
+            target = host_platform.target("default_target")
         else:
             target = spec.architecture.target
             if not target:
                 target = spack.platforms.host().target("default_target")
-            target = target.microarchitecture
 
             operating_system = spec.os
             if not operating_system:

lib/spack/spack/compilers/aocc.py

@@ -13,18 +13,6 @@
 
 
 class Aocc(Compiler):
-    # Subclasses use possible names of C compiler
-    cc_names = ["clang"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["clang++"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["flang"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["flang"]
-
     version_argument = "--version"
 
     @property

lib/spack/spack/compilers/arm.py

@@ -9,18 +9,6 @@
 
 
 class Arm(spack.compiler.Compiler):
-    # Subclasses use possible names of C compiler
-    cc_names = ["armclang"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["armclang++"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["armflang"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["armflang"]
-
     # Named wrapper links within lib/spack/env
     link_paths = {
         "cc": os.path.join("arm", "armclang"),
@@ -90,11 +78,3 @@ def fc_pic_flag(self):
         return "-fPIC"
 
     required_libs = ["libclang", "libflang"]
-
-    @classmethod
-    def fc_version(cls, fc):
-        return cls.default_version(fc)
-
-    @classmethod
-    def f77_version(cls, f77):
-        return cls.fc_version(f77)

lib/spack/spack/compilers/cce.py

@@ -19,18 +19,6 @@ def __init__(self, *args, **kwargs):
         if not self.is_clang_based:
             self.version_argument = "-V"
 
-    # Subclasses use possible names of C compiler
-    cc_names = ["craycc"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["crayCC"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["crayftn"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["crayftn"]
-
     # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
     suffixes = [r"-mp-\d\.\d"]
 

lib/spack/spack/compilers/clang.py

@@ -31,18 +31,6 @@
 
 
 class Clang(Compiler):
-    # Subclasses use possible names of C compiler
-    cc_names = ["clang"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["clang++"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["flang-new", "flang"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["flang-new", "flang"]
-
     version_argument = "--version"
 
     @property

lib/spack/spack/compilers/fj.py

@@ -9,18 +9,6 @@
 
 
 class Fj(spack.compiler.Compiler):
-    # Subclasses use possible names of C compiler
-    cc_names = ["fcc"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["FCC"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["frt"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["frt"]
-
     # Named wrapper links within build_env_path
     link_paths = {
         "cc": os.path.join("fj", "fcc"),

lib/spack/spack/compilers/gcc.py

@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import os
-import re
 
 from llnl.util.filesystem import ancestor
 
@@ -15,18 +14,6 @@
 
 
 class Gcc(spack.compiler.Compiler):
-    # Subclasses use possible names of C compiler
-    cc_names = ["gcc"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["g++"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["gfortran"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["gfortran"]
-
     # MacPorts builds gcc versions with prefixes and -mp-X or -mp-X.Y suffixes.
     # Homebrew and Linuxbrew may build gcc with -X, -X.Y suffixes.
     # Old compatibility versions may contain XY suffixes.
@@ -181,40 +168,6 @@ def default_version(cls, cc):
             version = cls.extract_version_from_output(output)
         return version
 
-    @classmethod
-    def fc_version(cls, fc):
-        """Older versions of gfortran use the ``-dumpversion`` option.
-        Output looks like this::
-
-            GNU Fortran (GCC) 4.4.7 20120313 (Red Hat 4.4.7-18)
-            Copyright (C) 2010 Free Software Foundation, Inc.
-
-        or::
-
-            4.8.5
-
-        In GCC 7, this option was changed to only return the major
-        version of the compiler::
-
-            7
-
-        A new ``-dumpfullversion`` option was added that gives us
-        what we want::
-
-            7.2.0
-        """
-        output = spack.compiler.get_compiler_version_output(fc, "-dumpversion")
-        match = re.search(r"(?:GNU Fortran \(GCC\) )?([\d.]+)", output)
-        version = match.group(match.lastindex) if match else "unknown"
-        if Version(version) >= Version("7"):
-            output = spack.compiler.get_compiler_version_output(fc, "-dumpfullversion")
-            version = cls.extract_version_from_output(output)
-        return version
-
-    @classmethod
-    def f77_version(cls, f77):
-        return cls.fc_version(f77)
-
     @property
     def stdcxx_libs(self):
         return ("-lstdc++",)

lib/spack/spack/compilers/intel.py

@@ -11,18 +11,6 @@
 
 
 class Intel(Compiler):
-    # Subclasses use possible names of C compiler
-    cc_names = ["icc"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["icpc"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["ifort"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["ifort"]
-
     # Named wrapper links within build_env_path
     link_paths = {
         "cc": os.path.join("intel", "icc"),
@@ -104,6 +92,14 @@ def c11_flag(self):
         else:
             return "-std=c1x"
 
+    @property
+    def c18_flag(self):
+        # c18 supported since oneapi 2022, which is classic version 2021.5.0
+        if self.real_version < Version("21.5.0"):
+            raise UnsupportedCompilerFlag(self, "the C18 standard", "c18_flag", "< 21.5.0")
+        else:
+            return "-std=c18"
+
     @property
     def cc_pic_flag(self):
         return "-fPIC"

lib/spack/spack/compilers/msvc.py

@@ -8,7 +8,7 @@
 import subprocess
 import sys
 import tempfile
-from typing import Dict, List
+from typing import Dict
 
 import archspec.cpu
 
@@ -117,18 +117,6 @@ def get_valid_fortran_pth():
 
 
 class Msvc(Compiler):
-    # Subclasses use possible names of C compiler
-    cc_names: List[str] = ["cl"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names: List[str] = ["cl"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names: List[str] = ["ifx"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names: List[str] = ["ifx"]
-
     # Named wrapper links within build_env_path
     # Due to the challenges of supporting compiler wrappers
     # in Windows, we leave these blank, and dynamically compute
@@ -305,6 +293,17 @@ def platform_toolset_ver(self):
         vs22_toolset = Version(toolset_ver) > Version("142")
         return toolset_ver if not vs22_toolset else "143"
 
+    @property
+    def visual_studio_version(self):
+        """The four digit Visual Studio version (i.e. 2019 or 2022)
+
+        Note: This differs from the msvc version or toolset version as
+        those properties track the compiler and build tools version
+        respectively, whereas this tracks the VS release associated
+        with a given MSVC compiler.
+        """
+        return re.search(r"[0-9]{4}", self.cc).group(0)
+
     def _compiler_version(self, compiler):
         """Returns version object for given compiler"""
         # ignore_errors below is true here due to ifx's
@@ -393,7 +392,3 @@ def fc_version(cls, fc):
             )
         clp = spack.util.executable.which_string("cl", path=sps)
         return cls.default_version(clp) if clp else fc_ver
-
-    @classmethod
-    def f77_version(cls, f77):
-        return cls.fc_version(f77)

lib/spack/spack/compilers/nag.py

@@ -5,7 +5,6 @@
 
 import os
 import re
-from typing import List
 
 import llnl.util.lang
 
@@ -13,18 +12,6 @@
 
 
 class Nag(spack.compiler.Compiler):
-    # Subclasses use possible names of C compiler
-    cc_names: List[str] = []
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names: List[str] = []
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["nagfor"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["nagfor"]
-
     # Named wrapper links within build_env_path
     # Use default wrappers for C and C++, in case provided in compilers.yaml
     link_paths = {

lib/spack/spack/compilers/nvhpc.py

@@ -9,18 +9,6 @@
 
 
 class Nvhpc(Compiler):
-    # Subclasses use possible names of C compiler
-    cc_names = ["nvc"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["nvc++"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["nvfortran"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["nvfortran"]
-
     # Named wrapper links within build_env_path
     link_paths = {
         "cc": os.path.join("nvhpc", "nvc"),

lib/spack/spack/compilers/oneapi.py

@@ -7,24 +7,14 @@
 from os.path import dirname, join
 
 from llnl.util import tty
+from llnl.util.filesystem import ancestor
 
+import spack.util.executable
 from spack.compiler import Compiler
 from spack.version import Version
 
 
 class Oneapi(Compiler):
-    # Subclasses use possible names of C compiler
-    cc_names = ["icx"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["icpx"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["ifx"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["ifx"]
-
     # Named wrapper links within build_env_path
     link_paths = {
         "cc": os.path.join("oneapi", "icx"),
@@ -128,6 +118,24 @@ def fc_pic_flag(self):
     def stdcxx_libs(self):
         return ("-cxxlib",)
 
+    @property
+    def prefix(self):
+        # OneAPI reports its install prefix when running ``--version``
+        # on the line ``InstalledDir: <prefix>/bin/compiler``.
+        cc = spack.util.executable.Executable(self.cc)
+        with self.compiler_environment():
+            oneapi_output = cc("--version", output=str, error=str)
+
+            for line in oneapi_output.splitlines():
+                if line.startswith("InstalledDir:"):
+                    oneapi_prefix = line.split(":")[1].strip()
+                    # Go from <prefix>/bin/compiler to <prefix>
+                    return ancestor(oneapi_prefix, 2)
+
+            raise RuntimeError(
+                "could not find install prefix of OneAPI from output:\n\t{}".format(oneapi_output)
+            )
+
     def setup_custom_environment(self, pkg, env):
         # workaround bug in icpx driver where it requires sycl-post-link is on the PATH
         # It is located in the same directory as the driver. Error message:

lib/spack/spack/compilers/pgi.py

@@ -10,18 +10,6 @@
 
 
 class Pgi(Compiler):
-    # Subclasses use possible names of C compiler
-    cc_names = ["pgcc"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["pgc++", "pgCC"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["pgfortran", "pgf77"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["pgfortran", "pgf95", "pgf90"]
-
     # Named wrapper links within build_env_path
     link_paths = {
         "cc": os.path.join("pgi", "pgcc"),

lib/spack/spack/compilers/rocmcc.py

@@ -11,18 +11,6 @@
 
 
 class Rocmcc(spack.compilers.clang.Clang):
-    # Subclasses use possible names of C compiler
-    cc_names = ["amdclang"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["amdclang++"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["amdflang"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["amdflang"]
-
     @property
     def link_paths(self):
         link_paths = {
@@ -61,14 +49,6 @@ def extract_version_from_output(cls, output):
         if match:
             return ".".join(match.groups())
 
-    @classmethod
-    def fc_version(cls, fortran_compiler):
-        return cls.default_version(fortran_compiler)
-
-    @classmethod
-    def f77_version(cls, f77):
-        return cls.fc_version(f77)
-
     @property
     def stdcxx_libs(self):
         return ("-lstdc++",)

lib/spack/spack/compilers/xl.py

@@ -10,18 +10,6 @@
 
 
 class Xl(Compiler):
-    # Subclasses use possible names of C compiler
-    cc_names = ["xlc"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["xlC", "xlc++"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["xlf"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["xlf90", "xlf95", "xlf2003", "xlf2008"]
-
     # Named wrapper links within build_env_path
     link_paths = {
         "cc": os.path.join("xl", "xlc"),
@@ -103,31 +91,3 @@ def fflags(self):
         # For Fortran 90 and beyond, it is set by default and has not impact.
         # Its use has no negative side effects.
         return "-qzerosize"
-
-    @classmethod
-    def fc_version(cls, fc):
-        # The fortran and C/C++ versions of the XL compiler are always
-        # two units apart.  By this we mean that the fortran release that
-        # goes with XL C/C++ 11.1 is 13.1.  Having such a difference in
-        # version number is confusing spack quite a lot.  Most notably
-        # if you keep the versions as is the default xl compiler will
-        # only have fortran and no C/C++.  So we associate the Fortran
-        # compiler with the version associated to the C/C++ compiler.
-        # One last stumble. Version numbers over 10 have at least a .1
-        # those under 10 a .0. There is no xlf 9.x or under currently
-        # available. BG/P and BG/L can such a compiler mix and possibly
-        # older version of AIX and linux on power.
-        fortran_version = cls.default_version(fc)
-        if fortran_version >= 16:
-            # Starting with version 16.1, the XL C and Fortran compilers
-            # have the same version.  So no need to downgrade the Fortran
-            # compiler version to match that of the C compiler version.
-            return str(fortran_version)
-        c_version = float(fortran_version) - 2
-        if c_version < 10:
-            c_version = c_version - 0.1
-        return str(c_version)
-
-    @classmethod
-    def f77_version(cls, f77):
-        return cls.fc_version(f77)

lib/spack/spack/compilers/xl_r.py

@@ -9,18 +9,6 @@
 
 
 class XlR(spack.compilers.xl.Xl):
-    # Subclasses use possible names of C compiler
-    cc_names = ["xlc_r"]
-
-    # Subclasses use possible names of C++ compiler
-    cxx_names = ["xlC_r", "xlc++_r"]
-
-    # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["xlf_r"]
-
-    # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["xlf90_r", "xlf95_r", "xlf2003_r", "xlf2008_r"]
-
     # Named wrapper links within build_env_path
     link_paths = {
         "cc": os.path.join("xl_r", "xlc_r"),

lib/spack/spack/database.py

@@ -50,6 +50,7 @@
     pass
 
 import llnl.util.filesystem as fs
+import llnl.util.lang
 import llnl.util.tty as tty
 
 import spack.deptypes as dt
@@ -121,6 +122,17 @@
 )
 
 
+@llnl.util.lang.memoized
+def _getfqdn():
+    """Memoized version of `getfqdn()`.
+
+    If we call `getfqdn()` too many times, DNS can be very slow. We only need to call it
+    one time per process, so we cache it here.
+
+    """
+    return socket.getfqdn()
+
+
 def reader(version: vn.StandardVersion) -> Type["spack.spec.SpecfileReaderBase"]:
     reader_cls = {
         vn.Version("5"): spack.spec.SpecfileV1,
@@ -287,12 +299,9 @@ def __reduce__(self):
                 database.  If it is a spec, we'll evaluate
                 ``spec.satisfies(query_spec)``
 
-            known (bool or None): Specs that are "known" are those
-                for which Spack can locate a ``package.py`` file -- i.e.,
-                Spack "knows" how to install them.  Specs that are unknown may
-                represent packages that existed in a previous version of
-                Spack, but have since either changed their name or
-                been removed
+            predicate_fn: optional predicate taking an InstallRecord as argument, and returning
+                whether that record is selected for the query. It can be used to craft criteria
+                that need some data for selection not provided by the Database itself.
 
             installed (bool or InstallStatus or typing.Iterable or None):
                 if ``True``, includes only installed
@@ -592,6 +601,9 @@ def _path(self, spec: "spack.spec.Spec") -> pathlib.Path:
         return self.dir / f"{spec.name}-{spec.dag_hash()}"
 
 
+SelectType = Callable[[InstallRecord], bool]
+
+
 class Database:
     #: Fields written for each install record
     record_fields: Tuple[str, ...] = DEFAULT_INSTALL_RECORD_FIELDS
@@ -1084,7 +1096,7 @@ def _write(self, type, value, traceback):
             self._state_is_inconsistent = True
             return
 
-        temp_file = self._index_path + (".%s.%s.temp" % (socket.getfqdn(), os.getpid()))
+        temp_file = self._index_path + (".%s.%s.temp" % (_getfqdn(), os.getpid()))
 
         # Write a temporary database file them move it into place
         try:
@@ -1233,7 +1245,7 @@ def _add(
         self._data[key].explicit = explicit
 
     @_autospec
-    def add(self, spec: "spack.spec.Spec", *, explicit: bool = False) -> None:
+    def add(self, spec: "spack.spec.Spec", *, explicit: bool = False, allow_missing=False) -> None:
         """Add spec at path to database, locking and reading DB to sync.
 
         ``add()`` will lock and read from the DB on disk.
@@ -1242,7 +1254,7 @@ def add(self, spec: "spack.spec.Spec", *, explicit: bool = False) -> None:
         # TODO: ensure that spec is concrete?
         # Entire add is transactional.
         with self.write_transaction():
-            self._add(spec, explicit=explicit)
+            self._add(spec, explicit=explicit, allow_missing=allow_missing)
 
     def _get_matching_spec_key(self, spec: "spack.spec.Spec", **kwargs) -> str:
         """Get the exact spec OR get a single spec that matches."""
@@ -1514,7 +1526,7 @@ def get_by_hash(self, dag_hash, default=None, installed=any):
     def _query(
         self,
         query_spec=any,
-        known=any,
+        predicate_fn: Optional[SelectType] = None,
         installed=True,
         explicit=any,
         start_date=None,
@@ -1522,7 +1534,7 @@ def _query(
         hashes=None,
         in_buildcache=any,
         origin=None,
-    ):
+    ) -> List["spack.spec.Spec"]:
         """Run a query on the database."""
 
         # TODO: Specs are a lot like queries.  Should there be a
@@ -1568,7 +1580,7 @@ def _query(
             if explicit is not any and rec.explicit != explicit:
                 continue
 
-            if known is not any and known(rec.spec.name):
+            if predicate_fn is not None and not predicate_fn(rec):
                 continue
 
             if start_date or end_date:
@@ -1653,14 +1665,14 @@ def query(self, *args, **kwargs):
         query.__doc__ = ""
     query.__doc__ += _QUERY_DOCSTRING
 
-    def query_one(self, query_spec, known=any, installed=True):
+    def query_one(self, query_spec, predicate_fn=None, installed=True):
         """Query for exactly one spec that matches the query spec.
 
         Raises an assertion error if more than one spec matches the
         query. Returns None if no installed package matches.
 
         """
-        concrete_specs = self.query(query_spec, known=known, installed=installed)
+        concrete_specs = self.query(query_spec, predicate_fn=predicate_fn, installed=installed)
         assert len(concrete_specs) <= 1
         return concrete_specs[0] if concrete_specs else None
 

lib/spack/spack/deptypes.py

@@ -43,6 +43,31 @@
 ALL_FLAGS: Tuple[DepFlag, DepFlag, DepFlag, DepFlag] = (BUILD, LINK, RUN, TEST)
 
 
+def compatible(flag1: DepFlag, flag2: DepFlag) -> bool:
+    """Returns True if two depflags can be dependencies from a Spec to deps of the same name.
+
+    The only allowable separated dependencies are a build-only dependency, combined with a
+    non-build dependency. This separates our two process spaces, build time and run time.
+
+    These dependency combinations are allowed:
+        single dep on name: [b], [l], [r], [bl], [br], [blr]
+        two deps on name: [b, l], [b, r], [b, lr]
+
+    but none of these make any sense:
+        two build deps: [b, b], [b, br], [b, bl], [b, blr]
+        any two deps that both have an l or an r, i.e. [l, l], [r, r], [l, r], [bl, l], [bl, r]"""
+    # Cannot have overlapping build types to two different dependencies
+    if flag1 & flag2:
+        return False
+
+    # Cannot have two different link/run dependencies for the same name
+    link_run = LINK | RUN
+    if flag1 & link_run and flag2 & link_run:
+        return False
+
+    return True
+
+
 def flag_from_string(s: str) -> DepFlag:
     if s == "build":
         return BUILD

lib/spack/spack/detection/path.py

@@ -11,6 +11,7 @@
 import os.path
 import re
 import sys
+import traceback
 import warnings
 from typing import Dict, Iterable, List, Optional, Set, Tuple, Type
 
@@ -18,6 +19,7 @@
 import llnl.util.lang
 import llnl.util.tty
 
+import spack.error
 import spack.spec
 import spack.util.elf as elf_utils
 import spack.util.environment
@@ -66,6 +68,21 @@ def file_identifier(path):
     return s.st_dev, s.st_ino
 
 
+def dedupe_paths(paths: List[str]) -> List[str]:
+    """Deduplicate paths based on inode and device number. In case the list contains first a
+    symlink and then the directory it points to, the symlink is replaced with the directory path.
+    This ensures that we pick for example ``/usr/bin`` over ``/bin`` if the latter is a symlink to
+    the former`."""
+    seen: Dict[Tuple[int, int], str] = {}
+    for path in paths:
+        identifier = file_identifier(path)
+        if identifier not in seen:
+            seen[identifier] = path
+        elif not os.path.islink(path):
+            seen[identifier] = path
+    return list(seen.values())
+
+
 def executables_in_path(path_hints: List[str]) -> Dict[str, str]:
     """Get the paths of all executables available from the current PATH.
 
@@ -82,8 +99,7 @@ def executables_in_path(path_hints: List[str]) -> Dict[str, str]:
     """
     search_paths = llnl.util.filesystem.search_paths_for_executables(*path_hints)
     # Make use we don't doubly list /usr/lib and /lib etc
-    search_paths = list(llnl.util.lang.dedupe(search_paths, key=file_identifier))
-    return path_to_dict(search_paths)
+    return path_to_dict(dedupe_paths(search_paths))
 
 
 def accept_elf(path, host_compat):
@@ -144,7 +160,7 @@ def libraries_in_ld_and_system_library_path(
         search_paths = list(filter(os.path.isdir, search_paths))
 
     # Make use we don't doubly list /usr/lib and /lib etc
-    search_paths = list(llnl.util.lang.dedupe(search_paths, key=file_identifier))
+    search_paths = dedupe_paths(search_paths)
 
     try:
         host_compat = elf_utils.get_elf_compat(sys.executable)
@@ -260,8 +276,12 @@ def detect_specs(
                 )
             except Exception as e:
                 specs = []
+                if spack.error.SHOW_BACKTRACE:
+                    details = traceback.format_exc()
+                else:
+                    details = f"[{e.__class__.__name__}: {e}]"
                 warnings.warn(
-                    f'error detecting "{pkg.name}" from prefix {candidate_path} [{str(e)}]'
+                    f'error detecting "{pkg.name}" from prefix {candidate_path}: {details}'
                 )
 
             if not specs:
@@ -435,9 +455,9 @@ def by_path(
                     llnl.util.tty.debug(
                         f"[EXTERNAL DETECTION] Skipping {pkg_name}: timeout reached"
                     )
-                except Exception as e:
+                except Exception:
                     llnl.util.tty.debug(
-                        f"[EXTERNAL DETECTION] Skipping {pkg_name}: exception occured {e}"
+                        f"[EXTERNAL DETECTION] Skipping {pkg_name}: {traceback.format_exc()}"
                     )
 
     return result

lib/spack/spack/environment/depfile.py

@@ -9,11 +9,13 @@
 
 import os
 import re
+import shlex
 from enum import Enum
 from typing import List, Optional
 
 import spack.deptypes as dt
 import spack.environment.environment as ev
+import spack.paths
 import spack.spec
 import spack.traverse as traverse
 
@@ -226,6 +228,7 @@ def to_dict(self):
             "install_deps_target": self._target("install-deps"),
             "any_hash_target": self._target("%"),
             "jobserver_support": self.jobserver_support,
+            "spack_script": shlex.quote(spack.paths.spack_script),
             "adjacency_list": self.make_adjacency_list,
             "phony_convenience_targets": " ".join(self.phony_convenience_targets),
             "pkg_ids_variable": self.pkg_identifier_variable,

lib/spack/spack/environment/environment.py

@@ -42,7 +42,6 @@
 import spack.spec_list
 import spack.store
 import spack.user_environment as uenv
-import spack.util.cpus
 import spack.util.environment
 import spack.util.hash
 import spack.util.lock as lk
@@ -547,8 +546,7 @@ def _is_dev_spec_and_has_changed(spec):
     last installation"""
     # First check if this is a dev build and in the process already try to get
     # the dev_path
-    dev_path_var = spec.variants.get("dev_path", None)
-    if not dev_path_var:
+    if not spec.variants.get("dev_path", None):
         return False
 
     # Now we can check whether the code changed since the last installation
@@ -556,9 +554,10 @@ def _is_dev_spec_and_has_changed(spec):
         # Not installed -> nothing to compare against
         return False
 
-    _, record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash())
-    mtime = fs.last_modification_time_recursive(dev_path_var.value)
-    return mtime > record.installation_time
+    # hook so packages can use to write their own method for checking the dev_path
+    # use package so attributes about concretization such as variant state can be
+    # utilized
+    return spec.package.detect_dev_src_change()
 
 
 def _error_on_nonempty_view_dir(new_root):
@@ -1160,6 +1159,8 @@ def clear(self, re_read=False):
             # things that cannot be recreated from file
             self.new_specs = []  # write packages for these on write()
 
+        self.manifest.clear()
+
     @property
     def active(self):
         """True if this environment is currently active."""
@@ -1967,7 +1968,7 @@ def install_specs(self, specs: Optional[List[Spec]] = None, **install_args):
         )
         install_args["explicit"] = explicit
 
-        PackageInstaller([spec.package for spec in specs], install_args).install()
+        PackageInstaller([spec.package for spec in specs], **install_args).install()
 
     def all_specs_generator(self) -> Iterable[Spec]:
         """Returns a generator for all concrete specs"""
@@ -2164,6 +2165,13 @@ def _concrete_specs_dict(self):
             # Assumes no legacy formats, since this was just created.
             spec_dict[ht.dag_hash.name] = s.dag_hash()
             concrete_specs[s.dag_hash()] = spec_dict
+
+            if s.build_spec is not s:
+                for d in s.build_spec.traverse():
+                    build_spec_dict = d.node_dict_with_hashes(hash=ht.dag_hash)
+                    build_spec_dict[ht.dag_hash.name] = d.dag_hash()
+                    concrete_specs[d.dag_hash()] = build_spec_dict
+
         return concrete_specs
 
     def _concrete_roots_dict(self):
@@ -2323,7 +2331,7 @@ def filter_specs(self, reader, json_specs_by_hash, order_concretized):
             specs_by_hash[lockfile_key] = spec
 
         # Second pass: For each spec, get its dependencies from the node dict
-        # and add them to the spec
+        # and add them to the spec, including build specs
         for lockfile_key, node_dict in json_specs_by_hash.items():
             name, data = reader.name_and_data(node_dict)
             for _, dep_hash, deptypes, _, virtuals in reader.dependencies_from_node_dict(data):
@@ -2331,6 +2339,10 @@ def filter_specs(self, reader, json_specs_by_hash, order_concretized):
                     specs_by_hash[dep_hash], depflag=dt.canonicalize(deptypes), virtuals=virtuals
                 )
 
+            if "build_spec" in node_dict:
+                _, bhash, _ = reader.extract_build_spec_info_from_node_dict(node_dict)
+                specs_by_hash[lockfile_key]._build_spec = specs_by_hash[bhash]
+
         # Traverse the root specs one at a time in the order they appear.
         # The first time we see each DAG hash, that's the one we want to
         # keep.  This is only required as long as we support older lockfile
@@ -2790,6 +2802,11 @@ def remove_user_spec(self, user_spec: str) -> None:
             raise SpackEnvironmentError(msg) from e
         self.changed = True
 
+    def clear(self) -> None:
+        """Clear all user specs from the list of root specs"""
+        self.configuration["specs"] = []
+        self.changed = True
+
     def override_user_spec(self, user_spec: str, idx: int) -> None:
         """Overrides the user spec at index idx with the one passed as input.
 

lib/spack/spack/error.py

@@ -12,6 +12,9 @@
 #: this is module-scoped because it needs to be set very early
 debug = 0
 
+#: whether to show a backtrace when an error is printed, enabled with --backtrace.
+SHOW_BACKTRACE = False
+
 
 class SpackError(Exception):
     """This is the superclass for all Spack errors.

lib/spack/spack/filesystem_view.py

@@ -100,10 +100,12 @@ def view_copy(
 
         spack.relocate.relocate_text(files=[dst], prefixes=prefix_to_projection)
 
-    try:
-        os.chown(dst, src_stat.st_uid, src_stat.st_gid)
-    except OSError:
-        tty.debug(f"Can't change the permissions for {dst}")
+    # The os module on Windows does not have a chown function.
+    if sys.platform != "win32":
+        try:
+            os.chown(dst, src_stat.st_uid, src_stat.st_gid)
+        except OSError:
+            tty.debug(f"Can't change the permissions for {dst}")
 
 
 #: supported string values for `link_type` in an env, mapped to canonical values

lib/spack/spack/install_test.py

@@ -372,8 +372,7 @@ def phase_tests(
             builder.pkg.test_suite.current_test_spec = builder.pkg.spec
             builder.pkg.test_suite.current_base_spec = builder.pkg.spec
 
-            # TODO (post-34236): "test"->"test_" once remove deprecated methods
-            have_tests = any(name.startswith("test") for name in method_names)
+            have_tests = any(name.startswith("test_") for name in method_names)
             if have_tests:
                 copy_test_files(builder.pkg, builder.pkg.spec)
 
@@ -477,16 +476,9 @@ def write_tested_status(self):
 def test_part(pkg: Pb, test_name: str, purpose: str, work_dir: str = ".", verbose: bool = False):
     wdir = "." if work_dir is None else work_dir
     tester = pkg.tester
-    # TODO (post-34236): "test"->"test_" once remove deprecated methods
     assert test_name and test_name.startswith(
-        "test"
-    ), f"Test name must start with 'test' but {test_name} was provided"
-
-    if test_name == "test":
-        tty.warn(
-            "{}: the 'test' method is deprecated. Convert stand-alone "
-            "test(s) to methods with names starting 'test_'.".format(pkg.name)
-        )
+        "test_"
+    ), f"Test name must start with 'test_' but {test_name} was provided"
 
     title = "test: {}: {}".format(test_name, purpose or "unspecified purpose")
     with fs.working_dir(wdir, create=True):
@@ -646,28 +638,11 @@ def test_functions(
             except spack.repo.UnknownPackageError:
                 tty.debug(f"{vname}: virtual does not appear to have a package file")
 
-    # TODO (post-34236): Remove if removing empty test method check
-    def skip(line):
-        # This should match the lines in the deprecated test() method
-        ln = line.strip()
-        return ln.startswith("#") or ("warn" in ln and "deprecated" in ln)
-
-    doc_regex = r'\s+("""[\w\s\(\)\-\,\;\:]+""")'
     tests = []
     for clss in classes:
         methods = inspect.getmembers(clss, predicate=lambda x: inspect.isfunction(x))
         for name, test_fn in methods:
-            # TODO (post-34236): "test"->"test_" once remove deprecated methods
-            if not name.startswith("test"):
-                continue
-
-            # TODO (post-34236): Could remove empty method check once remove
-            # TODO (post-34236): deprecated methods though some use cases,
-            # TODO (post-34236): such as checking packages have actual, non-
-            # TODO (post-34236): empty tests, may want this check to remain.
-            source = re.sub(doc_regex, r"", inspect.getsource(test_fn)).splitlines()[1:]
-            lines = [ln.strip() for ln in source if not skip(ln)]
-            if not lines:
+            if not name.startswith("test_"):
                 continue
 
             tests.append((clss.__name__, test_fn))  # type: ignore[union-attr]

lib/spack/spack/main.py

@@ -102,9 +102,6 @@
 
 spack_ld_library_path = os.environ.get("LD_LIBRARY_PATH", "")
 
-#: Whether to print backtraces on error
-SHOW_BACKTRACE = False
-
 
 def add_all_commands(parser):
     """Add all spack subcommands to the parser."""
@@ -527,8 +524,7 @@ def setup_main_options(args):
 
     if args.debug or args.backtrace:
         spack.error.debug = True
-        global SHOW_BACKTRACE
-        SHOW_BACKTRACE = True
+        spack.error.SHOW_BACKTRACE = True
 
     if args.debug:
         spack.util.debug.register_interrupt_handler()
@@ -1021,19 +1017,19 @@ def main(argv=None):
         e.die()  # gracefully die on any SpackErrors
 
     except KeyboardInterrupt:
-        if spack.config.get("config:debug") or SHOW_BACKTRACE:
+        if spack.config.get("config:debug") or spack.error.SHOW_BACKTRACE:
             raise
         sys.stderr.write("\n")
         tty.error("Keyboard interrupt.")
         return signal.SIGINT.value
 
     except SystemExit as e:
-        if spack.config.get("config:debug") or SHOW_BACKTRACE:
+        if spack.config.get("config:debug") or spack.error.SHOW_BACKTRACE:
             traceback.print_exc()
         return e.code
 
     except Exception as e:
-        if spack.config.get("config:debug") or SHOW_BACKTRACE:
+        if spack.config.get("config:debug") or spack.error.SHOW_BACKTRACE:
             raise
         tty.error(e)
         return 3

lib/spack/spack/mirror.py

@@ -89,9 +89,8 @@ def from_url(url: str):
         """Create an anonymous mirror by URL. This method validates the URL."""
         if not urllib.parse.urlparse(url).scheme in supported_url_schemes:
             raise ValueError(
-                '"{}" is not a valid mirror URL. Scheme must be once of {}.'.format(
-                    url, ", ".join(supported_url_schemes)
-                )
+                f'"{url}" is not a valid mirror URL. '
+                f"Scheme must be one of {supported_url_schemes}."
             )
         return Mirror(url)
 
@@ -759,7 +758,7 @@ def require_mirror_name(mirror_name):
     """Find a mirror by name and raise if it does not exist"""
     mirror = spack.mirror.MirrorCollection().get(mirror_name)
     if not mirror:
-        raise ValueError('no mirror named "{0}"'.format(mirror_name))
+        raise ValueError(f'no mirror named "{mirror_name}"')
     return mirror
 
 

lib/spack/spack/modules/__init__.py

@@ -7,8 +7,16 @@
 include Tcl non-hierarchical modules, Lua hierarchical modules, and others.
 """
 
+import os
 from typing import Dict, Type
 
+import llnl.util.tty as tty
+
+import spack.repo
+import spack.spec
+import spack.store
+
+from . import common
 from .common import BaseModuleFileWriter, disable_modules
 from .lmod import LmodModulefileWriter
 from .tcl import TclModulefileWriter
@@ -19,3 +27,66 @@
     "tcl": TclModulefileWriter,
     "lmod": LmodModulefileWriter,
 }
+
+
+def get_module(
+    module_type, spec: spack.spec.Spec, get_full_path, module_set_name="default", required=True
+):
+    """Retrieve the module file for a given spec and module type.
+
+    Retrieve the module file for the given spec if it is available. If the
+    module is not available, this will raise an exception unless the module
+    is excluded or if the spec is installed upstream.
+
+    Args:
+        module_type: the type of module we want to retrieve (e.g. lmod)
+        spec: refers to the installed package that we want to retrieve a module
+            for
+        required: if the module is required but excluded, this function will
+            print a debug message. If a module is missing but not excluded,
+            then an exception is raised (regardless of whether it is required)
+        get_full_path: if ``True``, this returns the full path to the module.
+            Otherwise, this returns the module name.
+        module_set_name: the named module configuration set from modules.yaml
+            for which to retrieve the module.
+
+    Returns:
+        The module name or path. May return ``None`` if the module is not
+        available.
+    """
+    try:
+        upstream = spec.installed_upstream
+    except spack.repo.UnknownPackageError:
+        upstream, record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash())
+    if upstream:
+        module = common.upstream_module_index.upstream_module(spec, module_type)
+        if not module:
+            return None
+
+        if get_full_path:
+            return module.path
+        else:
+            return module.use_name
+    else:
+        writer = module_types[module_type](spec, module_set_name)
+        if not os.path.isfile(writer.layout.filename):
+            fmt_str = "{name}{@version}{/hash:7}"
+            if not writer.conf.excluded:
+                raise common.ModuleNotFoundError(
+                    "The module for package {} should be at {}, but it does not exist".format(
+                        spec.format(fmt_str), writer.layout.filename
+                    )
+                )
+            elif required:
+                tty.debug(
+                    "The module configuration has excluded {}: omitting it".format(
+                        spec.format(fmt_str)
+                    )
+                )
+            else:
+                return None
+
+        if get_full_path:
+            return writer.layout.filename
+        else:
+            return writer.layout.use_name

lib/spack/spack/modules/common.py

@@ -46,10 +46,8 @@
 import spack.deptypes as dt
 import spack.environment
 import spack.error
-import spack.modules
 import spack.paths
 import spack.projections as proj
-import spack.repo
 import spack.schema.environment
 import spack.spec
 import spack.store
@@ -324,67 +322,6 @@ def upstream_module(self, spec, module_type):
             return None
 
 
-def get_module(module_type, spec, get_full_path, module_set_name="default", required=True):
-    """Retrieve the module file for a given spec and module type.
-
-    Retrieve the module file for the given spec if it is available. If the
-    module is not available, this will raise an exception unless the module
-    is excluded or if the spec is installed upstream.
-
-    Args:
-        module_type: the type of module we want to retrieve (e.g. lmod)
-        spec: refers to the installed package that we want to retrieve a module
-            for
-        required: if the module is required but excluded, this function will
-            print a debug message. If a module is missing but not excluded,
-            then an exception is raised (regardless of whether it is required)
-        get_full_path: if ``True``, this returns the full path to the module.
-            Otherwise, this returns the module name.
-        module_set_name: the named module configuration set from modules.yaml
-            for which to retrieve the module.
-
-    Returns:
-        The module name or path. May return ``None`` if the module is not
-        available.
-    """
-    try:
-        upstream = spec.installed_upstream
-    except spack.repo.UnknownPackageError:
-        upstream, record = spack.store.STORE.db.query_by_spec_hash(spec.dag_hash())
-    if upstream:
-        module = upstream_module_index.upstream_module(spec, module_type)
-        if not module:
-            return None
-
-        if get_full_path:
-            return module.path
-        else:
-            return module.use_name
-    else:
-        writer = spack.modules.module_types[module_type](spec, module_set_name)
-        if not os.path.isfile(writer.layout.filename):
-            fmt_str = "{name}{@version}{/hash:7}"
-            if not writer.conf.excluded:
-                raise ModuleNotFoundError(
-                    "The module for package {} should be at {}, but it does not exist".format(
-                        spec.format(fmt_str), writer.layout.filename
-                    )
-                )
-            elif required:
-                tty.debug(
-                    "The module configuration has excluded {}: omitting it".format(
-                        spec.format(fmt_str)
-                    )
-                )
-            else:
-                return None
-
-        if get_full_path:
-            return writer.layout.filename
-        else:
-            return writer.layout.use_name
-
-
 class BaseConfiguration:
     """Manipulates the information needed to generate a module file to make
     querying easier. It needs to be sub-classed for specific module types.
@@ -590,7 +527,8 @@ def use_name(self):
         parts = name.split("/")
         name = os.path.join(*parts)
         # Add optional suffixes based on constraints
-        path_elements = [name] + self.conf.suffixes
+        path_elements = [name]
+        path_elements.extend(map(self.spec.format, self.conf.suffixes))
         return "-".join(path_elements)
 
     @property