python: urls return [None] if not pypi

Signed-off-by: Teague Sterling <teaguesterling@gmail.com>

.github/dependabot.yml

@@ -5,13 +5,10 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
-  # Requirements to build documentation
+  # Requirements to run style checks and build documentation
   - package-ecosystem: "pip"
-    directory: "/lib/spack/docs"
+    directories:
-    schedule:
+     - "/.github/workflows/requirements/style/*"
-      interval: "daily"
+     - "/lib/spack/docs"
-  # Requirements to run style checks
-  - package-ecosystem: "pip"
-    directory: "/.github/workflows/style"
     schedule:
       interval: "daily"

.github/workflows/audit.yaml

@@ -28,8 +28,8 @@ jobs:
       run:
         shell: ${{ matrix.system.shell }}
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
-    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
+    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
       with:
         python-version: ${{inputs.python_version}}
     - name: Install Python packages
@@ -44,6 +44,7 @@ jobs:
       run: |
           . share/spack/setup-env.sh
           coverage run $(which spack) audit packages
+          coverage run $(which spack) audit configs
           coverage run $(which spack) -d audit externals
           coverage combine
           coverage xml
@@ -52,6 +53,7 @@ jobs:
       run: |
           . share/spack/setup-env.sh          
           spack -d audit packages
+          spack -d audit configs
           spack -d audit externals
     - name: Package audits (without coverage)
       if: ${{ runner.os == 'Windows' }}
@@ -59,9 +61,11 @@ jobs:
           . share/spack/setup-env.sh          
           spack -d audit packages
           ./share/spack/qa/validate_last_exit.ps1
+          spack -d audit configs
+          ./share/spack/qa/validate_last_exit.ps1
           spack -d audit externals
           ./share/spack/qa/validate_last_exit.ps1
-    - uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c
+    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
       if: ${{ inputs.with_coverage == 'true' }}
       with:
         flags: unittests,audits

.github/workflows/bootstrap.yml

@@ -37,7 +37,7 @@ jobs:
               make patch unzip which xz python3 python3-devel tree \
               cmake bison
       - name: Checkout
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
           fetch-depth: 0
       - name: Bootstrap clingo
@@ -53,27 +53,33 @@ jobs:
     runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        runner: ['macos-13', 'macos-14', "ubuntu-latest"]
+        runner: ['macos-13', 'macos-14', "ubuntu-latest", "windows-latest"]
     steps:
       - name: Setup macOS
-        if: ${{ matrix.runner != 'ubuntu-latest' }}
+        if: ${{ matrix.runner != 'ubuntu-latest' && matrix.runner != 'windows-latest' }}
         run: |
           brew install cmake bison tree
       - name: Checkout
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
+      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
         with:
           python-version: "3.12"
       - name: Bootstrap clingo
+        env:
+          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
+          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
+          USER_SCOPE_PARENT_DIR: ${{ matrix.runner == 'windows-latest' && '$env:userprofile' || '$HOME' }}
+          VALIDATE_LAST_EXIT: ${{ matrix.runner == 'windows-latest' && './share/spack/qa/validate_last_exit.ps1' || '' }}
         run: |
-          source share/spack/setup-env.sh
+          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
           spack bootstrap disable github-actions-v0.5
           spack bootstrap disable github-actions-v0.4
           spack external find --not-buildable cmake bison
           spack -d solve zlib
-          tree ~/.spack/bootstrap/store/
+          ${{ env.VALIDATE_LAST_EXIT }}
+          tree ${{ env.USER_SCOPE_PARENT_DIR }}/.spack/bootstrap/store/
 
   gnupg-sources:
     runs-on: ${{ matrix.runner }}
@@ -90,7 +96,7 @@ jobs:
         if: ${{ matrix.runner == 'ubuntu-latest' }}
         run: sudo rm -rf $(command -v gpg gpg2 patchelf)
       - name: Checkout
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
           fetch-depth: 0
       - name: Bootstrap GnuPG
@@ -119,10 +125,10 @@ jobs:
         run: |
           sudo rm -rf $(which gpg) $(which gpg2) $(which patchelf)
       - name: Checkout
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
+      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
         with:
           python-version: |
             3.8
@@ -148,7 +154,7 @@ jobs:
                 not_found=0
                 old_path="$PATH"
                 export PATH="$ver_dir:$PATH"
-                ./bin/spack-tmpconfig -b ./.github/workflows/bootstrap-test.sh
+                ./bin/spack-tmpconfig -b ./.github/workflows/bin/bootstrap-test.sh
                 export PATH="$old_path"
               fi
             fi
@@ -162,4 +168,3 @@ jobs:
           source share/spack/setup-env.sh
           spack -d gpg list
           tree ~/.spack/bootstrap/store/
-

.github/workflows/build-containers.yml

@@ -40,8 +40,7 @@ jobs:
         # 1: Platforms to build for
         # 2: Base image (e.g. ubuntu:22.04)
         dockerfile: [[amazon-linux, 'linux/amd64,linux/arm64', 'amazonlinux:2'],
-                     [centos7, 'linux/amd64,linux/arm64,linux/ppc64le', 'centos:7'],
+                     [centos-stream9, 'linux/amd64,linux/arm64,linux/ppc64le', 'centos:stream9'],
-                     [centos-stream, 'linux/amd64,linux/arm64,linux/ppc64le', 'centos:stream'],
                      [leap15, 'linux/amd64,linux/arm64,linux/ppc64le', 'opensuse/leap:15'],
                      [ubuntu-focal, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:20.04'],
                      [ubuntu-jammy, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:22.04'],
@@ -56,7 +55,7 @@ jobs:
     if: github.repository == 'spack/spack'
     steps:
       - name: Checkout
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
 
       - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
         id: docker_meta
@@ -77,7 +76,7 @@ jobs:
         env:
           SPACK_YAML_OS: "${{ matrix.dockerfile[2] }}"
         run: |
-          .github/workflows/generate_spack_yaml_containerize.sh
+          .github/workflows/bin/generate_spack_yaml_containerize.sh
           . share/spack/setup-env.sh
           mkdir -p dockerfiles/${{ matrix.dockerfile[0] }}
           spack containerize --last-stage=bootstrap | tee dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile
@@ -88,19 +87,19 @@ jobs:
           fi
 
       - name: Upload Dockerfile
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
         with:
           name: dockerfiles_${{ matrix.dockerfile[0] }}
           path: dockerfiles
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db
 
       - name: Log in to GitHub Container Registry
-        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -108,13 +107,13 @@ jobs:
 
       - name: Log in to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build & Deploy ${{ matrix.dockerfile[0] }}
-        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0
+        uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755
         with:
           context: dockerfiles/${{ matrix.dockerfile[0] }}
           platforms: ${{ matrix.dockerfile[1] }}
@@ -127,7 +126,7 @@ jobs:
     needs: deploy-images
     steps:
       - name: Merge Artifacts
-        uses: actions/upload-artifact/merge@65462800fd760344b1a7b4382951275a0abb4808
+        uses: actions/upload-artifact/merge@834a144ee995460fba8ed112a2fc961b36a5ec5a
         with:
           name: dockerfiles
           pattern: dockerfiles_*

.github/workflows/ci.yaml

@@ -36,7 +36,7 @@ jobs:
       core: ${{ steps.filter.outputs.core }}
       packages: ${{ steps.filter.outputs.packages }}
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         if: ${{ github.event_name == 'push' }}
         with:
           fetch-depth: 0
@@ -53,6 +53,13 @@ jobs:
             - 'var/spack/repos/builtin/packages/clingo/**'
             - 'var/spack/repos/builtin/packages/python/**'
             - 'var/spack/repos/builtin/packages/re2c/**'
+            - 'var/spack/repos/builtin/packages/gnupg/**'
+            - 'var/spack/repos/builtin/packages/libassuan/**'
+            - 'var/spack/repos/builtin/packages/libgcrypt/**'
+            - 'var/spack/repos/builtin/packages/libgpg-error/**'
+            - 'var/spack/repos/builtin/packages/libksba/**'
+            - 'var/spack/repos/builtin/packages/npth/**'
+            - 'var/spack/repos/builtin/packages/pinentry/**'
             - 'lib/spack/**'
             - 'share/spack/**'
             - '.github/workflows/bootstrap.yml'
@@ -77,13 +84,8 @@ jobs:
     needs: [ prechecks, changes ]
     uses: ./.github/workflows/unit_tests.yaml
     secrets: inherit
-  windows:
-    if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.core == 'true' }}
-    needs: [ prechecks ]
-    uses: ./.github/workflows/windows_python.yml
-    secrets: inherit
   all:
-    needs: [ windows, unit-tests, bootstrap ]
+    needs: [ unit-tests, bootstrap ]
     runs-on: ubuntu-latest
     steps:
     - name: Success

.github/workflows/install_spack.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env sh
-. share/spack/setup-env.sh
-echo -e "config:\n  build_jobs: 2" > etc/spack/config.yaml
-spack config add "packages:all:target:[x86_64]"
-spack compiler find
-spack compiler info apple-clang
-spack debug report
-spack solve zlib

.github/workflows/nightly-win-builds.yml

@@ -14,10 +14,10 @@ jobs:
   build-paraview-deps:
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
+    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
       with:
         python-version: 3.9
     - name: Install Python packages

.github/workflows/requirements/style/requirements.txt

@@ -1,6 +1,6 @@
-black==24.4.2
+black==24.8.0
 clingo==5.7.1
-flake8==7.0.0
+flake8==7.1.1
 isort==5.13.2
 mypy==1.8.0
 types-six==1.16.21.20240513

.github/workflows/unit_tests.yaml

@@ -51,10 +51,10 @@ jobs:
           on_develop: false
 
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
+    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install System packages
@@ -72,7 +72,7 @@ jobs:
       run: |
           # Need this for the git tests to succeed.
           git --version
-          . .github/workflows/setup_git.sh
+          . .github/workflows/bin/setup_git.sh
     - name: Bootstrap clingo
       if: ${{ matrix.concretizer == 'clingo' }}
       env:
@@ -91,7 +91,7 @@ jobs:
           UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }}
       run: |
           share/spack/qa/run-unit-tests
-    - uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c
+    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
       with:
         flags: unittests,linux,${{ matrix.concretizer }}
         token: ${{ secrets.CODECOV_TOKEN }}
@@ -100,10 +100,10 @@ jobs:
   shell:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
+    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -118,13 +118,13 @@ jobs:
       run: |
           # Need this for the git tests to succeed.
           git --version
-          . .github/workflows/setup_git.sh
+          . .github/workflows/bin/setup_git.sh
     - name: Run shell tests
       env:
           COVERAGE: true
       run: |
           share/spack/qa/run-shell-tests
-    - uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c
+    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
       with:
         flags: shelltests,linux
         token: ${{ secrets.CODECOV_TOKEN }}
@@ -141,13 +141,13 @@ jobs:
           dnf install -y \
               bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
               make patch tcl unzip which xz
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
     - name: Setup repo and non-root user
       run: |
           git --version
           git config --global --add safe.directory /__w/spack/spack
           git fetch --unshallow
-          . .github/workflows/setup_git.sh
+          . .github/workflows/bin/setup_git.sh
           useradd spack-test
           chown -R spack-test .
     - name: Run unit tests
@@ -160,10 +160,10 @@ jobs:
   clingo-cffi:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
+    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -178,14 +178,14 @@ jobs:
       run: |
           # Need this for the git tests to succeed.
           git --version
-          . .github/workflows/setup_git.sh
+          . .github/workflows/bin/setup_git.sh
     - name: Run unit tests (full suite with coverage)
       env:
           COVERAGE: true
           SPACK_TEST_SOLVER: clingo
       run: |
           share/spack/qa/run-unit-tests
-    - uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c
+    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
       with:
         flags: unittests,linux,clingo
         token: ${{ secrets.CODECOV_TOKEN }}
@@ -198,10 +198,10 @@ jobs:
         os: [macos-13, macos-14]
         python-version: ["3.11"]
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
+    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Python packages
@@ -217,14 +217,45 @@ jobs:
         SPACK_TEST_PARALLEL: 4
       run: |
         git --version
-        . .github/workflows/setup_git.sh
+        . .github/workflows/bin/setup_git.sh
         . share/spack/setup-env.sh
         $(which spack) bootstrap disable spack-install
         $(which spack) solve zlib
         common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
         $(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
-    - uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c
+    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
       with:
         flags: unittests,macos
         token: ${{ secrets.CODECOV_TOKEN }}
         verbose: true
+  # Run unit tests on Windows
+  windows:
+    defaults:
+      run:
+        shell:
+          powershell Invoke-Expression -Command "./share/spack/qa/windows_test_setup.ps1"; {0}
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      with:
+        fetch-depth: 0
+    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+      with:
+        python-version: 3.9
+    - name: Install Python packages
+      run: |
+          python -m pip install --upgrade pip pywin32 setuptools pytest-cov clingo
+    - name: Create local develop
+      run: |
+        ./.github/workflows/bin/setup_git.ps1
+    - name: Unit Test
+      run: |
+        spack unit-test -x --verbose --cov --cov-config=pyproject.toml
+        ./share/spack/qa/validate_last_exit.ps1
+        coverage combine -a
+        coverage xml
+    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+      with:
+        flags: unittests,windows
+        token: ${{ secrets.CODECOV_TOKEN }}
+        verbose: true

.github/workflows/valid-style.yml

@@ -18,15 +18,15 @@ jobs:
   validate:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
-    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
+    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
       with:
         python-version: '3.11'
         cache: 'pip'
     - name: Install Python Packages
       run: |
         pip install --upgrade pip setuptools
-        pip install -r .github/workflows/style/requirements.txt
+        pip install -r .github/workflows/requirements/style/requirements.txt
     - name: vermin (Spack's Core)
       run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
     - name: vermin (Repositories)
@@ -35,22 +35,22 @@ jobs:
   style:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
+    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
       with:
         python-version: '3.11'
         cache: 'pip'
     - name: Install Python packages
       run: |
         pip install --upgrade pip setuptools
-        pip install -r .github/workflows/style/requirements.txt
+        pip install -r .github/workflows/requirements/style/requirements.txt
     - name: Setup git configuration
       run: |
         # Need this for the git tests to succeed.
         git --version
-        . .github/workflows/setup_git.sh
+        . .github/workflows/bin/setup_git.sh
     - name: Run style tests
       run: |
           share/spack/qa/run-style-tests
@@ -70,13 +70,13 @@ jobs:
           dnf install -y \
               bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
               make patch tcl unzip which xz
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       - name: Setup repo and non-root user
         run: |
           git --version
           git config --global --add safe.directory /__w/spack/spack
           git fetch --unshallow
-          . .github/workflows/setup_git.sh
+          . .github/workflows/bin/setup_git.sh
           useradd spack-test
           chown -R spack-test .
       - name: Bootstrap Spack development environment

.github/workflows/windows_python.yml

@@ -1,83 +0,0 @@
-name: windows
-
-on:
-  workflow_call:
-
-concurrency:
-  group: windows-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell:
-     powershell Invoke-Expression -Command "./share/spack/qa/windows_test_setup.ps1"; {0}
-jobs:
-  unit-tests:
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
-      with:
-        fetch-depth: 0
-    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
-      with:
-        python-version: 3.9
-    - name: Install Python packages
-      run: |
-          python -m pip install --upgrade pip pywin32 setuptools pytest-cov clingo
-    - name: Create local develop
-      run: |
-        ./.github/workflows/setup_git.ps1
-    - name: Unit Test
-      run: |
-        spack unit-test -x --verbose --cov --cov-config=pyproject.toml --ignore=lib/spack/spack/test/cmd
-        ./share/spack/qa/validate_last_exit.ps1
-        coverage combine -a
-        coverage xml
-    - uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c
-      with:
-        flags: unittests,windows
-        token: ${{ secrets.CODECOV_TOKEN }}
-        verbose: true
-  unit-tests-cmd:
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
-      with:
-        fetch-depth: 0
-    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
-      with:
-        python-version: 3.9
-    - name: Install Python packages
-      run: |
-          python -m pip install --upgrade pip pywin32 setuptools coverage pytest-cov clingo
-    - name: Create local develop
-      run: |
-        ./.github/workflows/setup_git.ps1
-    - name: Command Unit Test
-      run: |
-        spack unit-test -x --verbose --cov --cov-config=pyproject.toml lib/spack/spack/test/cmd
-        ./share/spack/qa/validate_last_exit.ps1
-        coverage combine -a
-        coverage xml
-    - uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c
-      with:
-        flags: unittests,windows
-        token: ${{ secrets.CODECOV_TOKEN }}
-        verbose: true
-  build-abseil:
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
-      with:
-        fetch-depth: 0
-    - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
-      with:
-        python-version: 3.9
-    - name: Install Python packages
-      run: |
-          python -m pip install --upgrade pip pywin32 setuptools coverage
-    - name: Build Test
-      run: |
-        spack compiler find
-        spack -d external find cmake ninja
-        spack -d install abseil-cpp

CHANGELOG.md

@@ -1,3 +1,324 @@
+
+# v0.22.0 (2024-05-12)
+
+`v0.22.0` is a major feature release.
+
+## Features in this release
+
+1. **Compiler dependencies**
+
+    We are in the process of making compilers proper dependencies in Spack, and a number
+    of changes in `v0.22` support that effort. You may notice nodes in your dependency
+    graphs for compiler runtime libraries like `gcc-runtime` or `libgfortran`, and you
+    may notice that Spack graphs now include `libc`. We've also begun moving compiler
+    configuration from `compilers.yaml` to `packages.yaml` to make it consistent with
+    other externals. We are trying to do this with the least disruption possible, so
+    your existing `compilers.yaml` files should still work. We expect to be done with
+    this transition by the `v0.23` release in November.
+
+    * #41104: Packages compiled with `%gcc` on Linux, macOS and FreeBSD now depend on a
+      new package `gcc-runtime`, which contains a copy of the shared compiler runtime
+      libraries. This enables gcc runtime libraries to be installed and relocated when
+      using a build cache. When building minimal Spack-generated container images it is
+      no longer necessary to install libgfortran, libgomp etc. using the system package
+      manager.
+
+    * #42062: Packages compiled with `%oneapi` now depend on a new package
+      `intel-oneapi-runtime`. This is similar to `gcc-runtime`, and the runtimes can
+      provide virtuals and compilers can inject dependencies on virtuals into compiled
+      packages. This allows us to model library soname compatibility and allows
+      compilers like `%oneapi` to provide virtuals like `sycl` (which can also be
+      provided by standalone libraries). Note that until we have an agreement in place
+      with intel, Intel packages are marked `redistribute(source=False, binary=False)`
+      and must be downloaded outside of Spack.
+
+    * #43272: changes to the optimization criteria of the solver improve the hit-rate of
+      buildcaches by a fair amount. The solver more relaxed compatibility rules and will
+      not try to strictly match compilers or targets of reused specs. Users can still
+      enforce the previous strict behavior with `require:` sections in `packages.yaml`.
+      Note that to enforce correct linking, Spack will *not* reuse old `%gcc` and
+      `%oneapi` specs that do not have the runtime libraries as a dependency.
+
+    * #43539: Spack will reuse specs built with compilers that are *not* explicitly
+      configured in `compilers.yaml`. Because we can now keep runtime libraries in build
+      cache, we do not require you to also have a local configured compiler to *use* the
+      runtime libraries. This improves reuse in buildcaches and avoids conflicts with OS
+      updates that happen underneath Spack.
+
+    * #43190: binary compatibility on `linux` is now based on the `libc` version,
+      instead of on the `os` tag. Spack builds now detect the host `libc` (`glibc` or
+      `musl`) and add it as an implicit external node in the dependency graph. Binaries
+      with a `libc` with the same name and a version less than or equal to that of the
+      detected `libc` can be reused. This is only on `linux`, not `macos` or `Windows`.
+
+    * #43464: each package that can provide a compiler is now detectable using `spack
+      external find`. External packages defining compiler paths are effectively used as
+      compilers, and `spack external find -t compiler` can be used as a substitute for
+      `spack compiler find`. More details on this transition are in
+      [the docs](https://spack.readthedocs.io/en/latest/getting_started.html#manual-compiler-configuration)
+
+2. **Improved `spack find` UI for Environments**
+
+   If you're working in an enviroment, you likely care about:
+
+   * What are the roots
+   * Which ones are installed / not installed
+   * What's been added that still needs to be concretized
+
+    We've tweaked `spack find` in environments to show this information much more
+    clearly. Installation status is shown next to each root, so you can see what is
+    installed. Roots are also shown in bold in the list of installed packages. There is
+    also a new option for `spack find -r` / `--only-roots` that will only show env
+    roots, if you don't want to look at all the installed specs.
+
+    More details in #42334.
+
+3. **Improved command-line string quoting**
+
+   We are making some breaking changes to how Spack parses specs on the CLI in order to
+   respect shell quoting instead of trying to fight it. If you (sadly) had to write
+   something like this on the command line:
+
+    ```
+    spack install zlib cflags=\"-O2 -g\"
+    ```
+
+    That will now result in an error, but you can now write what you probably expected
+    to work in the first place:
+
+    ```
+    spack install zlib cflags="-O2 -g"
+    ```
+
+    Quoted can also now include special characters, so you can supply flags like:
+
+    ```
+    spack intall zlib ldflags='-Wl,-rpath=$ORIGIN/_libs'
+    ```
+
+    To reduce ambiguity in parsing, we now require that you *not* put spaces around `=`
+    and `==` when for flags or variants. This would not have broken before but will now
+    result in an error:
+
+    ```
+    spack install zlib cflags = "-O2 -g"
+    ```
+
+    More details and discussion in #30634.
+
+4. **Revert default `spack install` behavior to `--reuse`**
+
+   We changed the default concretizer behavior from `--reuse` to `--reuse-deps` in
+   #30990 (in `v0.20`), which meant that *every* `spack install` invocation would
+   attempt to build a new version of the requested package / any environment roots.
+   While this is a common ask for *upgrading* and for *developer* workflows, we don't
+   think it should be the default for a package manager.
+
+   We are going to try to stick to this policy:
+   1. Prioritize reuse and build as little as possible by default.
+   2. Only upgrade or install duplicates if they are explicitly asked for, or if there
+      is a known security issue that necessitates an upgrade.
+
+   With the install command you now have three options:
+
+   * `--reuse` (default): reuse as many existing installations as possible.
+   * `--reuse-deps` / `--fresh-roots`: upgrade (freshen) roots but reuse dependencies if possible.
+   * `--fresh`: install fresh versions of requested packages (roots) and their dependencies.
+
+   We've also introduced `--fresh-roots` as an alias for `--reuse-deps` to make it more clear
+   that it may give you fresh versions. More details in #41302 and #43988.
+
+5. **More control over reused specs**
+
+   You can now control which packages to reuse and how. There is a new
+   `concretizer:reuse` config option, which accepts the following properties:
+
+   - `roots`: `true` to reuse roots, `false` to reuse just dependencies
+   - `exclude`: list of constraints used to select which specs *not* to reuse
+   - `include`: list of constraints used to select which specs *to* reuse
+   - `from`: list of sources for reused specs (some combination of `local`,
+     `buildcache`, or `external`)
+
+   For example, to reuse only specs compiled with GCC, you could write:
+
+   ```yaml
+   concretizer:
+      reuse:
+        roots: true
+        include:
+        - "%gcc"
+   ```
+
+   Or, if `openmpi` must be used from externals, and it must be the only external used:
+
+   ```yaml
+   concretizer:
+     reuse:
+       roots: true
+       from:
+       - type: local
+         exclude: ["openmpi"]
+       - type: buildcache
+         exclude: ["openmpi"]
+       - type: external
+         include: ["openmpi"]
+   ```
+
+6. **New `redistribute()` directive**
+
+   Some packages can't be redistributed in source or binary form. We need an explicit
+   way to say that in a package.
+
+   Now there is a `redistribute()` directive so that package authors can write:
+
+   ```python
+   class MyPackage(Package):
+       redistribute(source=False, binary=False)
+   ```
+
+   Like other directives, this works with `when=`:
+
+   ```python
+   class MyPackage(Package):
+       # 12.0 and higher are proprietary
+       redistribute(source=False, binary=False, when="@12.0:")
+
+       # can't redistribute when we depend on some proprietary dependency
+       redistribute(source=False, binary=False, when="^proprietary-dependency")
+   ```
+
+    More in #20185.
+
+7. **New `conflict:` and `prefer:` syntax for package preferences**
+
+   Previously, you could express conflicts and preferences in `packages.yaml` through
+   some contortions with `require:`:
+
+    ```yaml
+    packages:
+      zlib-ng:
+        require:
+        - one_of: ["%clang", "@:"]   # conflict on %clang
+        - any_of: ["+shared", "@:"]  # strong preference for +shared
+    ```
+
+    You can now use `require:` and `prefer:` for a much more readable configuration:
+
+    ```yaml
+    packages:
+      zlib-ng:
+        conflict:
+        - "%clang"
+        prefer:
+        - "+shared"
+    ```
+
+    See [the documentation](https://spack.readthedocs.io/en/latest/packages_yaml.html#conflicts-and-strong-preferences)
+    and #41832 for more details.
+
+8. **`include_concrete` in environments**
+
+   You may want to build on the *concrete* contents of another environment without
+   changing that environment.  You can now include the concrete specs from another
+   environment's `spack.lock` with `include_concrete`:
+
+   ```yaml
+      spack:
+        specs: []
+        concretizer:
+            unify: true
+        include_concrete:
+        - /path/to/environment1
+        - /path/to/environment2
+   ```
+
+   Now, when *this* environment is concretized, it will bring in the already concrete
+   specs from `environment1` and `environment2`, and build on top of them without
+   changing them. This is useful if you have phased deployments, where old deployments
+   should not be modified but you want to use as many of them as possible. More details
+   in #33768.
+
+9. **`python-venv` isolation**
+
+   Spack has unique requirements for Python because it:
+    1. installs every package in its own independent directory, and
+    2. allows users to register *external* python installations.
+
+   External installations may contain their own installed packages that can interfere
+   with Spack installations, and some distributions (Debian and Ubuntu) even change the
+   `sysconfig` in ways that alter the installation layout of installed Python packages
+   (e.g., with the addition of a `/local` prefix on Debian or Ubuntu). To isolate Spack
+   from these and other issues, we now insert a small `python-venv` package in between
+   `python` and packages that need to install Python code. This isolates Spack's build
+   environment, isolates Spack from any issues with an external python, and resolves a
+   large number of issues we've had with Python installations.
+
+   See #40773 for further details.
+
+## New commands, options, and directives
+
+* Allow packages to be pushed to build cache after install from source (#42423)
+* `spack develop`: stage build artifacts in same root as non-dev builds #41373
+  * Don't delete `spack develop` build artifacts after install (#43424)
+* `spack find`: add options for local/upstream only (#42999)
+* `spack logs`: print log files for packages (either partially built or installed) (#42202)
+* `patch`: support reversing patches (#43040)
+* `develop`: Add -b/--build-directory option to set build_directory package attribute (#39606)
+* `spack list`: add `--namesapce` / `--repo` option (#41948)
+* directives: add `checked_by` field to `license()`, add some license checks
+* `spack gc`: add options for environments and build dependencies (#41731)
+* Add `--create` to `spack env activate` (#40896)
+
+## Performance improvements
+
+* environment.py: fix excessive re-reads (#43746)
+* ruamel yaml: fix quadratic complexity bug  (#43745)
+* Refactor to improve `spec format` speed (#43712)
+* Do not acquire a write lock on the env post install if no views (#43505)
+* asp.py: fewer calls to `spec.copy()` (#43715)
+* spec.py: early return in `__str__`
+* avoid `jinja2` import at startup unless needed (#43237)
+
+## Other new features of note
+
+* `archspec`: update to `v0.2.4`: support for Windows, bugfixes for `neoverse-v1` and
+  `neoverse-v2` detection.
+* `spack config get`/`blame`: with no args, show entire config
+* `spack env create <env>`: dir if dir-like (#44024)
+* ASP-based solver: update os compatibility for macOS (#43862)
+* Add handling of custom ssl certs in urllib ops (#42953)
+* Add ability to rename environments (#43296)
+* Add config option and compiler support to reuse across OS's (#42693)
+* Support for prereleases (#43140)
+* Only reuse externals when configured (#41707)
+* Environments: Add support for including views (#42250)
+
+## Binary caches
+* Build cache: make signed/unsigned a mirror property (#41507)
+* tools stack
+
+## Removals, deprecations, and syntax changes
+* remove `dpcpp` compiler and package (#43418)
+* spack load: remove --only argument (#42120)
+
+## Notable Bugfixes
+* repo.py: drop deleted packages from provider cache (#43779)
+* Allow `+` in module file names (#41999)
+* `cmd/python`: use runpy to allow multiprocessing in scripts (#41789)
+* Show extension commands with spack -h (#41726)
+* Support environment variable expansion inside module projections (#42917)
+* Alert user to failed concretizations (#42655)
+* shell: fix zsh color formatting for PS1 in environments (#39497)
+* spack mirror create --all: include patches (#41579)
+
+## Spack community stats
+
+* 7,994 total packages; 525 since `v0.21.0`
+    * 178 new Python packages, 5 new R packages
+* 358 people contributed to this release
+    * 344 committers to packages
+    * 45 committers to core
+
 # v0.21.2 (2024-03-01)
 
 ## Bugfixes

bin/spack-python

@@ -22,4 +22,4 @@
 #
 # This is compatible across platforms.
 #
-exec /usr/bin/env spack python "$@"
+exec spack python "$@"

bin/spack.bat

@@ -188,25 +188,27 @@ if NOT "%_sp_args%"=="%_sp_args:--help=%" (
 goto :end_switch
 
 :case_load
-:: If args contain --sh, --csh, or -h/--help: just execute.
+if NOT defined _sp_args (
-if defined _sp_args (
+   exit /B 0
-    if NOT "%_sp_args%"=="%_sp_args:--help=%" (
+)
-        goto :default_case
+
-    ) else if NOT "%_sp_args%"=="%_sp_args:-h=%" (
+:: If args contain --bat, or -h/--help: just execute.
-        goto :default_case
+if NOT "%_sp_args%"=="%_sp_args:--help=%" (
-    ) else if NOT "%_sp_args%"=="%_sp_args:--bat=%" (
+    goto :default_case
-        goto :default_case
+) else if NOT "%_sp_args%"=="%_sp_args:-h=%" (
-    )
+    goto :default_case
+) else if NOT "%_sp_args%"=="%_sp_args:--bat=%" (
+    goto :default_case
+) else if NOT "%_sp_args%"=="%_sp_args:--list=%" (
+    goto :default_case
 )
 
 for /f "tokens=* USEBACKQ" %%I in (
-    `python "%spack%" %_sp_flags% %_sp_subcommand% --bat %_sp_args%`) do %%I
+    `python "%spack%" %_sp_flags% %_sp_subcommand% --bat %_sp_args%`
+    ) do %%I
 
 goto :end_switch
 
-:case_unload
-goto :case_load
-
 :default_case
 python "%spack%" %_sp_flags% %_sp_subcommand% %_sp_args%
 goto :end_switch

etc/spack/defaults/packages.yaml

@@ -20,6 +20,7 @@ packages:
       awk: [gawk]
       armci: [armcimpi]
       blas: [openblas, amdblis]
+      cxx: [gcc]
       D: [ldc]
       daal: [intel-oneapi-daal]
       elf: [elfutils]
@@ -61,6 +62,7 @@ packages:
       tbb: [intel-tbb]
       unwind: [libunwind]
       uuid: [util-linux-uuid, libuuid]
+      wasi-sdk: [wasi-sdk-prebuilt]
       xxd: [xxd-standalone, vim]
       yacc: [bison, byacc]
       ziglang: [zig]

lib/spack/docs/containers.rst

@@ -203,12 +203,9 @@ The OS that are currently supported are summarized in the table below:
    * - Ubuntu 24.04
      - ``ubuntu:24.04``
      - ``spack/ubuntu-noble``
-   * - CentOS 7
+   * - CentOS Stream9
-     - ``centos:7``
+     - ``quay.io/centos/centos:stream9``
-     - ``spack/centos7``
+     - ``spack/centos-stream9``
-   * - CentOS Stream
-     - ``quay.io/centos/centos:stream``
-     - ``spack/centos-stream``
    * - openSUSE Leap
      - ``opensuse/leap``
      - ``spack/leap15``

lib/spack/docs/environments.rst

@@ -931,32 +931,84 @@ This allows for a much-needed reduction in redundancy between packages
 and constraints.
 
 
-----------------
+-----------------
-Filesystem Views
+Environment Views
-----------------
+-----------------
 
-Spack Environments can define filesystem views, which provide a direct access point
+Spack Environments can have an associated filesystem view, which is a directory
-for software similar to the directory hierarchy that might exist under ``/usr/local``.
+with a more traditional structure ``<view>/bin``, ``<view>/lib``, ``<view>/include``
-Filesystem views are updated every time the environment is written out to the lock
+in which all files of the installed packages are linked.
-file ``spack.lock``, so the concrete environment and the view are always compatible.
+
-The files of the view's installed packages are brought into the view by symbolic or
+By default a view is created for each environment, thanks to the ``view: true``
-hard links, referencing the original Spack installation, or by copy.
+option in the ``spack.yaml`` manifest file:
+
+.. code-block:: yaml
+
+   spack:
+     specs: [perl, python]
+     view: true
+
+The view is created in a hidden directory ``.spack-env/view`` relative to the environment.
+If you've used ``spack env activate``, you may have already interacted with this view. Spack
+prepends its ``<view>/bin`` dir to ``PATH`` when the environment is activated, so that
+you can directly run executables from all installed packages in the environment.
+
+Views are highly customizable: you can control where they are put, modify their structure,
+include and exclude specs, change how files are linked, and you can even generate multiple
+views for a single environment.
 
 .. _configuring_environment_views:
 
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^
-Configuration in ``spack.yaml``
+Minimal view configuration
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The Spack Environment manifest file has a top-level keyword
+The minimal configuration
-``view``. Each entry under that heading is a **view descriptor**, headed
+
-by a name. Any number of views may be defined under the ``view`` heading.
+.. code-block:: yaml
-The view descriptor contains the root of the view, and
+
-optionally the projections for the view, ``select`` and
+   spack:
-``exclude`` lists for the view and link information via ``link`` and
+     # ...
+     view: true
+
+lets Spack generate a single view with default settings under the
+``.spack-env/view`` directory of the environment.
+
+Another short way to configure a view is to specify just where to put it:
+
+.. code-block:: yaml
+
+   spack:
+     # ...
+     view: /path/to/view
+
+Views can also be disabled by setting ``view: false``.
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Advanced view configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+One or more **view descriptors** can be defined under ``view``, keyed by a name.
+The example from the previous section with ``view: /path/to/view`` is equivalent
+to defining a view descriptor named ``default`` with a ``root`` attribute:
+
+.. code-block:: yaml
+
+   spack:
+     # ...
+     view:
+       default:  # name of the view
+         root: /path/to/view  # view descriptor attribute
+
+The ``default`` view descriptor name is special: when you ``spack env activate`` your
+environment, this view will be used to update (among other things) your ``PATH``
+variable.
+
+View descriptors must contain the root of the view, and optionally projections,
+``select`` and ``exclude`` lists and link information via ``link`` and
 ``link_type``.
 
-For example, in the following manifest
+As a more advanced example, in the following manifest
 file snippet we define a view named ``mpis``, rooted at
 ``/path/to/view`` in which all projections use the package name,
 version, and compiler name to determine the path for a given
@@ -1001,59 +1053,10 @@ of ``hardlink`` or ``copy``.
    when the environment is not activated, and linked libraries will be located
    *outside* of the view thanks to rpaths.
 
-
-There are two shorthands for environments with a single view. If the
-environment at ``/path/to/env`` has a single view, with a root at
-``/path/to/env/.spack-env/view``, with default selection and exclusion
-and the default projection, we can put ``view: True`` in the
-environment manifest. Similarly, if the environment has a view with a
-different root, but default selection, exclusion, and projections, the
-manifest can say ``view: /path/to/view``. These views are
-automatically named ``default``, so that
-
-.. code-block:: yaml
-
-   spack:
-     # ...
-     view: True
-
-is equivalent to
-
-.. code-block:: yaml
-
-   spack:
-     # ...
-     view:
-       default:
-         root: .spack-env/view
-
-and
-
-.. code-block:: yaml
-
-   spack:
-     # ...
-     view: /path/to/view
-
-is equivalent to
-
-.. code-block:: yaml
-
-   spack:
-     # ...
-     view:
-       default:
-         root: /path/to/view
-
-By default, Spack environments are configured with ``view: True`` in
-the manifest. Environments can be configured without views using
-``view: False``. For backwards compatibility reasons, environments
-with no ``view`` key are treated the same as ``view: True``.
-
 From the command line, the ``spack env create`` command takes an
 argument ``--with-view [PATH]`` that sets the path for a single, default
 view. If no path is specified, the default path is used (``view:
-True``). The argument ``--without-view`` can be used to create an
+true``). The argument ``--without-view`` can be used to create an
 environment without any view configured.
 
 The ``spack env view`` command can be used to change the manage views
@@ -1119,11 +1122,18 @@ the projection under ``all`` before reaching those entries.
 Activating environment views
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The ``spack env activate`` command will put the default view for the
+The ``spack env activate <env>`` has two effects:
-environment into the user's path, in addition to activating the
+
-environment for Spack commands. The arguments ``-v,--with-view`` and
+1. It activates the environment so that further Spack commands such
-``-V,--without-view`` can be used to tune this behavior. The default
+   as ``spack install`` will run in the context of the environment.
-behavior is to activate with the environment view if there is one.
+2. It activates the view so that environment variables such as
+   ``PATH`` are updated to include the view.
+
+Without further arguments, the ``default`` view of the environment is
+activated. If a view with a different name has to be activated,
+``spack env activate --with-view <name> <env>`` can be
+used instead. You can also activate the environment without modifying
+further environment variables using ``--without-view``.
 
 The environment variables affected by the ``spack env activate``
 command and the paths that are used to update them are determined by
@@ -1146,8 +1156,8 @@ relevant variable if the path exists. For this reason, it is not
 recommended to use non-default projections with the default view of an
 environment.
 
-The ``spack env deactivate`` command will remove the default view of
+The ``spack env deactivate`` command will remove the active view of
-the environment from the user's path.
+the Spack environment from the user's environment variables.
 
 
 .. _env-generate-depfile:
@@ -1306,7 +1316,7 @@ index once every package is pushed. Note how this target uses the generated
    example/push/%: example/install/%
    	@mkdir -p $(dir $@)
    	$(info About to push $(SPEC) to a buildcache)
-   	$(SPACK) -e . buildcache push --allow-root --only=package $(BUILDCACHE_DIR) /$(HASH)
+   	$(SPACK) -e . buildcache push --only=package $(BUILDCACHE_DIR) /$(HASH)
    	@touch $@
 
    push: $(addprefix example/push/,$(example/SPACK_PACKAGE_IDS))

lib/spack/docs/packaging_guide.rst

@@ -2344,6 +2344,27 @@ you set ``parallel`` to ``False`` at the package level, then each call
 to ``make()`` will be sequential by default, but packagers can call
 ``make(parallel=True)`` to override it.
 
+Note that the ``--jobs`` option works out of the box for all standard
+build systems. If you are using a non-standard build system instead, you
+can use the variable ``make_jobs`` to extract the number of jobs specified
+by the ``--jobs`` option:
+
+.. code-block:: python
+   :emphasize-lines: 7, 11
+   :linenos:
+ 
+   class Xios(Package):
+      ...
+      def install(self, spec, prefix):
+         ...
+         options = [
+            ...
+            '--jobs', str(make_jobs),
+        ]
+        ...
+        make_xios = Executable("./make_xios")
+        make_xios(*options)
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Install-level build parallelism
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -5173,12 +5194,6 @@ installed executable. The check is implemented as follows:
                 reframe = Executable(self.prefix.bin.reframe)
                 reframe("-l")
 
-.. warning::
-
-    The API for adding tests is not yet considered stable and may change
-    in future releases.
-
-
 """"""""""""""""""""""""""""""""
 Checking build-time test results
 """"""""""""""""""""""""""""""""
@@ -5216,38 +5231,42 @@ be left in the build stage directory as illustrated below:
 Stand-alone tests
 ^^^^^^^^^^^^^^^^^
 
-While build-time tests are integrated with the build process, stand-alone
+While build-time tests are integrated with the installation process, stand-alone
 tests are expected to run days, weeks, even months after the software is
 installed. The goal is to provide a mechanism for gaining confidence that
 packages work as installed **and** *continue* to work as the underlying
 software evolves. Packages can add and inherit stand-alone tests. The
-`spack test`` command is used to manage stand-alone testing.
+``spack test`` command is used for stand-alone testing.
 
-.. note::
+.. admonition:: Stand-alone test methods should complete within a few minutes.
 
     Execution speed is important since these tests are intended to quickly
-    assess whether installed specs work on the system. Consequently, they
+    assess whether installed specs work on the system. Spack cannot spare
-    should run relatively quickly -- as in on the order of at most a few
+    resources for more extensive testing of packages included in CI stacks.
-    minutes -- while ideally executing all, or at least key aspects of the
-    installed software.
 
-.. note::
+    Consequently, stand-alone tests should run relatively quickly -- as in
-
+    on the order of at most a few minutes -- while testing at least key aspects
-    Failing stand-alone tests indicate problems with the installation and,
+    of the installed software. Save more extensive testing for other tools.
-    therefore, there is no reason to proceed with more resource-intensive
-    tests until those have been investigated.
-
-    Passing stand-alone tests indicate that more thorough testing, such
-    as running extensive unit or regression tests, or tests that run at
-    scale can proceed without wasting resources on a problematic installation.
 
 Tests are defined in the package using methods with names beginning ``test_``.
 This allows Spack to support multiple independent checks, or parts. Files
 needed for testing, such as source, data, and expected outputs, may be saved
 from the build and or stored with the package in the repository. Regardless
 of origin, these files are automatically copied to the spec's test stage
-directory prior to execution of the test method(s). Spack also provides some
+directory prior to execution of the test method(s). Spack also provides helper
-helper functions to facilitate processing.
+functions to facilitate common processing.
+
+.. tip::
+
+    **The status of stand-alone tests can be used to guide follow-up testing efforts.**
+
+    Passing stand-alone tests justify performing more thorough testing, such
+    as running extensive unit or regression tests or tests that run at scale,
+    when available. These tests are outside of the scope of Spack packaging.
+
+    Failing stand-alone tests indicate problems with the installation and,
+    therefore, no reason to proceed with more resource-intensive tests until
+    the failures have been investigated.
 
 .. _configure-test-stage:
 
@@ -5255,30 +5274,26 @@ helper functions to facilitate processing.
 Configuring the test stage directory
 """"""""""""""""""""""""""""""""""""
 
-Stand-alone tests utilize a test stage directory for building, running,
+Stand-alone tests utilize a test stage directory to build, run, and track
-and tracking results in the same way Spack uses a build stage directory.
+tests in the same way Spack uses a build stage directory to install software.
-The default test stage root directory, ``~/.spack/test``, is defined in
+The default test stage root directory, ``$HOME/.spack/test``, is defined in
-:ref:`etc/spack/defaults/config.yaml <config-yaml>`. This location is
+:ref:`config.yaml <config-yaml>`. This location is customizable by adding or
-customizable by adding or changing the ``test_stage`` path in the high-level
+changing the ``test_stage`` path such that:
-``config`` of the appropriate ``config.yaml`` file such that:
 
 .. code-block:: yaml
 
    config:
      test_stage: /path/to/test/stage
 
-Packages can use the ``self.test_suite.stage`` property to access this setting.
+Packages can use the ``self.test_suite.stage`` property to access the path.
-Other package properties that provide access to spec-specific subdirectories
-and files are described in :ref:`accessing staged files <accessing-files>`.
 
-.. note::
+.. admonition:: Each spec being tested has its own test stage directory.
 
-   The test stage path is the root directory for the **entire suite**.
+   The ``config:test_stage`` option is the path to the root of a
-   In other words, it is the root directory for **all specs** being
+   **test suite**'s stage directories.
-   tested by the ``spack test run`` command. Each spec gets its own
-   stage subdirectory. Use ``self.test_suite.test_dir_for_spec(self.spec)``
-   to access the spec-specific test stage directory.
 
+   Other package properties that provide paths to spec-specific subdirectories
+   and files are described in :ref:`accessing-files`.
 
 .. _adding-standalone-tests:
 
@@ -5291,61 +5306,144 @@ Test recipes are defined in the package using methods with names beginning
 Each method has access to the information Spack tracks on the package, such
 as options, compilers, and dependencies, supporting the customization of tests
 to the build. Standard python ``assert`` statements and other error reporting
-mechanisms are available. Such exceptions are automatically caught and reported
+mechanisms can be used. These exceptions are automatically caught and reported
 as test failures.
 
-Each test method is an implicit test part named by the method and whose
+Each test method is an *implicit test part* named by the method. Its purpose
-purpose is the method's docstring. Providing a purpose gives context for
+is the method's docstring. Providing a meaningful purpose for the test gives
-aiding debugging. A test method may contain embedded test parts. Spack
+context that can aid debugging. Spack outputs both the name and purpose at the
-outputs the test name and purpose prior to running each test method and
+start of test execution so it's also important that the docstring/purpose be
-any embedded test parts. For example, ``MyPackage`` below provides two basic
+brief.
-examples of installation tests: ``test_always_fails`` and ``test_example``.
+
-As the name indicates, the first always fails. The second simply runs the
+.. tip::
-installed example.
+
+    We recommend naming test methods so it is clear *what* is being tested.
+    For example, if a test method is building and or running an executable
+    called ``example``, then call the method ``test_example``. This, together
+    with a similarly meaningful test purpose, will aid test comprehension,
+    debugging, and maintainability.
+
+Stand-alone tests run in an environment that provides access to information
+on the installed software, such as build options, dependencies, and compilers.
+Build options and dependencies are accessed using the same spec checks used
+by build recipes. Examples of checking :ref:`variant settings <variants>` and
+:ref:`spec constraints <testing-specs>` can be found at the provided links.
+
+.. admonition:: Spack automatically sets up the test stage directory and environment.
+
+    Spack automatically creates the test stage directory and copies
+    relevant files *prior to* running tests. It can also ensure build
+    dependencies are available **if** necessary.
+
+    The path to the test stage is configurable (see :ref:`configure-test-stage`).
+
+    Files that Spack knows to copy are those saved from the build (see
+    :ref:`cache_extra_test_sources`) and those added to the package repository
+    (see :ref:`cache_custom_files`).
+
+    Spack will use the value of the ``test_requires_compiler`` property to
+    determine whether it needs to also set up build dependencies (see
+    :ref:`test-build-tests`).
+
+The ``MyPackage`` package below provides two basic test examples: 
+``test_example`` and ``test_example2``.  The first runs the installed
+``example`` and ensures its output contains an expected string. The second
+runs ``example2`` without checking output so is only concerned with confirming
+the executable runs successfully. If the installed spec is not expected to have
+``example2``, then the check at the top of the method will raise a special
+``SkipTest`` exception, which is captured to facilitate reporting skipped test
+parts to tools like CDash.
 
 .. code-block:: python
 
    class MyPackage(Package):
        ...
 
-       def test_always_fails(self):
-           """use assert to always fail"""
-           assert False
-
        def test_example(self):
-           """run installed example"""
+           """ensure installed example works"""
+           expected = "Done."
            example = which(self.prefix.bin.example)
-           example()
+
+           # Capture stdout and stderr from running the Executable
+           # and check that the expected output was produced.
+           out = example(output=str.split, error=str.split)
+           assert expected in out, f"Expected '{expected}' in the output"
+
+       def test_example2(self):
+           """run installed example2"""
+           if self.spec.satisfies("@:1.0"):
+               # Raise SkipTest to ensure flagging the test as skipped for
+               # test reporting purposes.
+               raise SkipTest("Test is only available for v1.1 on")
+
+           example2 = which(self.prefix.bin.example2)
+           example2()
 
 Output showing the identification of each test part after running the tests
 is illustrated below.
 
 .. code-block:: console
 
-   $ spack test run --alias mypackage mypackage@1.0
+   $ spack test run --alias mypackage mypackage@2.0
    ==> Spack test mypackage
    ...
    $ spack test results -l mypackage
    ==> Results for test suite 'mypackage':
    ...
-   ==> [2023-03-10-16:03:56.625204] test: test_always_fails: use assert to always fail
+   ==> [2024-03-10-16:03:56.625439] test: test_example: ensure installed example works
    ...
-   FAILED
+   PASSED: MyPackage::test_example
-   ==> [2023-03-10-16:03:56.625439] test: test_example: run installed example
+   ==> [2024-03-10-16:03:56.625439] test: test_example2: run installed example2
    ...
-   PASSED
+   PASSED: MyPackage::test_example2
 
+.. admonition:: Do NOT implement tests that must run in the installation prefix.
 
-.. note::
+   Use of the package spec's installation prefix for building and running
+   tests is **strongly discouraged**. Doing so causes permission errors for
+   shared spack instances *and* facilities that install the software in
+   read-only file systems or directories.
 
-   If ``MyPackage`` were a recipe for a library, the tests should build
+   Instead, start these test methods by explicitly copying the needed files
-   an example or test program that is then executed.
+   from the installation prefix to the test stage directory. Note the test
+   stage directory is the current directory when the test is executed with
+   the ``spack test run`` command.
 
-A test method can include test parts using the ``test_part`` context manager.
+.. admonition:: Test methods for library packages should build test executables.
-Each part is treated as an independent check to allow subsequent test parts
-to execute even after a test part fails.
 
-.. _test-part:
+   Stand-alone tests for library packages *should* build test executables
+   that utilize the *installed* library. Doing so ensures the tests follow
+   a similar build process that users of the library would follow.
+
+   For more information on how to do this, see :ref:`test-build-tests`.
+
+.. tip::
+
+   If you want to see more examples from packages with stand-alone tests, run
+   ``spack pkg grep "def\stest" | sed "s/\/package.py.*//g" | sort -u``
+   from the command line to get a list of the packages.
+
+.. _adding-standalone-test-parts:
+
+"""""""""""""""""""""""""""""
+Adding stand-alone test parts
+"""""""""""""""""""""""""""""
+
+Sometimes dependencies between steps of a test lend themselves to being
+broken into parts. Tracking the pass/fail status of each part may aid
+debugging. Spack provides a ``test_part`` context manager for use within
+test methods.
+
+Each test part is independently run, tracked, and reported. Test parts are
+executed in the order they appear. If one fails, subsequent test parts are
+still performed even if they would also fail. This allows tools like CDash
+to track and report the status of test parts across runs. The pass/fail status
+of the enclosing test is derived from the statuses of the embedded test parts.
+
+.. admonition:: Test method and test part names **must** be unique.
+
+   Test results reporting requires that test methods and embedded test parts
+   within a package have unique names.
 
 The signature for ``test_part`` is:
 
@@ -5367,40 +5465,68 @@ where each argument has the following meaning:
 * ``work_dir`` is the path to the directory in which the test will run.
 
   The default of ``None``, or ``"."``, corresponds to the the spec's test
-  stage (i.e., ``self.test_suite.test_dir_for_spec(self.spec)``.
+  stage (i.e., ``self.test_suite.test_dir_for_spec(self.spec)``).
 
-.. admonition:: Tests should **not** run under the installation directory.
+.. admonition:: Start test part names with the name of the enclosing test.
 
-   Use of the package spec's installation directory for building and running
+   We **highly recommend** starting the names of test parts with the name
-   tests is **strongly** discouraged. Doing so causes permission errors for
+   of the enclosing test. Doing so helps with the comprehension, readability
-   shared spack instances *and* facilities that install the software in
+   and debugging of test results.
-   read-only file systems or directories.
 
-Suppose ``MyPackage`` actually installs two examples we want to use for tests.
+Suppose ``MyPackage`` installs multiple executables that need to run in a
-These checks can be implemented as separate checks or, as illustrated below,
+specific order since the outputs from one are inputs of others. Further suppose
-embedded test parts.
+we want to add an integration test that runs the executables in order. We can
+accomplish this goal by implementing a stand-alone test method consisting of
+test parts for each executable as follows:
 
 .. code-block:: python
 
    class MyPackage(Package):
        ...
 
-       def test_example(self):
+       def test_series(self):
-           """run installed examples"""
+           """run setup, perform, and report"""
-           for example in ["ex1", "ex2"]:
-               with test_part(
-                   self,
-                   f"test_example_{example}",
-                   purpose=f"run installed {example}",
-                ):
-                    exe = which(join_path(self.prefix.bin, example))
-                    exe()
 
-In this case, there will be an implicit test part for ``test_example``
+           with test_part(self, "test_series_setup", purpose="setup operation"):
-and separate sub-parts for ``ex1`` and ``ex2``. The second sub-part
+                exe = which(self.prefix.bin.setup))
-will be executed regardless of whether the first passes. The test
+                exe()
-log for a run where the first executable fails and the second passes
+
-is illustrated below.
+           with test_part(self, "test_series_run", purpose="perform operation"):
+                exe = which(self.prefix.bin.run))
+                exe()
+
+           with test_part(self, "test_series_report", purpose="generate report"):
+                exe = which(self.prefix.bin.report))
+                exe()
+
+The result is ``test_series`` runs the following executable in order: ``setup``,
+``run``, and ``report``. In this case no options are passed to any of the
+executables and no outputs from running them are checked. Consequently, the
+implementation could be simplified with a for-loop as follows:
+
+.. code-block:: python
+
+   class MyPackage(Package):
+       ...
+
+       def test_series(self):
+           """execute series setup, run, and report"""
+
+           for exe, reason in [
+               ("setup", "setup operation"),
+               ("run", "perform operation"),
+               ("report", "generate report")
+           ]:
+               with test_part(self, f"test_series_{exe}", purpose=reason):
+                   exe = which(self.prefix.bin.join(exe))
+                   exe()
+
+In both cases, since we're using a context manager, each test part in
+``test_series`` will execute regardless of the status of the other test
+parts.
+
+Now let's look at the output from running the stand-alone tests where
+the second test part, ``test_series_run``, fails.
 
 .. code-block:: console
 
@@ -5410,50 +5536,68 @@ is illustrated below.
    $ spack test results -l mypackage
    ==> Results for test suite 'mypackage':
    ...
-   ==> [2023-03-10-16:03:56.625204] test: test_example: run installed examples
+   ==> [2024-03-10-16:03:56.625204] test: test_series: execute series setup, run, and report
-   ==> [2023-03-10-16:03:56.625439] test: test_example_ex1: run installed ex1
+   ==> [2024-03-10-16:03:56.625439] test: test_series_setup: setup operation
    ...
-   FAILED
+   PASSED: MyPackage::test_series_setup
-   ==> [2023-03-10-16:03:56.625555] test: test_example_ex2: run installed ex2
+   ==> [2024-03-10-16:03:56.625555] test: test_series_run: perform operation
    ...
-   PASSED
+   FAILED: MyPackage::test_series_run
+   ==> [2024-03-10-16:03:57.003456] test: test_series_report: generate report
+   ...
+   FAILED: MyPackage::test_series_report
+   FAILED: MyPackage::test_series
    ...
 
-.. warning::
+Since test parts depended on the success of previous parts, we see that the
+failure of one results in the failure of subsequent checks and the overall
+result of the test method, ``test_series``, is failure.
 
-   Test results reporting requires that each test method and embedded
+.. tip::
-   test part for a package have a unique name.
 
-Stand-alone tests run in an environment that provides access to information
+   If you want to see more examples from packages using ``test_part``, run
-Spack has on how the software was built, such as build options, dependencies,
+   ``spack pkg grep "test_part(" | sed "s/\/package.py.*//g" | sort -u``
-and compilers. Build options and dependencies are accessed with the normal
+   from the command line to get a list of the packages.
-spec checks. Examples of checking :ref:`variant settings <variants>` and
-:ref:`spec constraints <testing-specs>` can be found at the provided links.
-Accessing compilers in stand-alone tests that are used by the build requires
-setting a package property as described :ref:`below <test-compilation>`.
 
+.. _test-build-tests:
 
-.. _test-compilation:
+"""""""""""""""""""""""""""""""""""""
+Building and running test executables
+"""""""""""""""""""""""""""""""""""""
 
-"""""""""""""""""""""""""
+.. admonition:: Re-use build-time sources and (small) input data sets when possible.
-Enabling test compilation
-"""""""""""""""""""""""""
 
-If you want to build and run binaries in tests, then you'll need to tell
+    We **highly recommend** re-using build-time test sources and pared down
-Spack to load the package's compiler configuration. This is accomplished
+    input files for testing installed software. These files are easier
-by setting the package's ``test_requires_compiler`` property to ``True``.
+    to keep synchronized with software capabilities when they reside
+    within the software's repository. More information on saving files from
+    the installation process can be found at :ref:`cache_extra_test_sources`.
 
-Setting the property to ``True`` ensures access to the compiler through
+    If that is not possible, you can add test-related files to the package
-canonical environment variables (e.g., ``CC``, ``CXX``, ``FC``, ``F77``).
+    repository (see :ref:`cache_custom_files`). It will be important to
-It also gives access to build dependencies like ``cmake`` through their
+    remember to maintain them so they work across listed or supported versions
-``spec objects`` (e.g., ``self.spec["cmake"].prefix.bin.cmake``).
+    of the package.
 
-.. note::
+Packages that build libraries are good examples of cases where you'll want
+to build test executables from the installed software before running them.
+Doing so requires you to let Spack know it needs to load the package's
+compiler configuration. This is accomplished by setting the package's
+``test_requires_compiler`` property to ``True``.
 
-   The ``test_requires_compiler`` property should be added at the top of
+.. admonition:: ``test_requires_compiler = True`` is required to build test executables.
-   the package near other attributes, such as the ``homepage`` and ``url``.
 
-Below illustrates using this feature to compile an example.
+   Setting the property to ``True`` ensures access to the compiler through
+   canonical environment variables (e.g., ``CC``, ``CXX``, ``FC``, ``F77``).
+   It also gives access to build dependencies like ``cmake`` through their
+   ``spec objects`` (e.g., ``self.spec["cmake"].prefix.bin.cmake`` for the
+   path or ``self.spec["cmake"].command`` for the ``Executable`` instance).
+
+   Be sure to add the property at the top of the package class under other
+   properties like the ``homepage``.
+
+The example below, which ignores how ``cxx-example.cpp`` is acquired,
+illustrates the basic process of compiling a test executable using the
+installed library before running it.
 
 .. code-block:: python
 
@@ -5477,28 +5621,22 @@ Below illustrates using this feature to compile an example.
            cxx_example = which(exe)
            cxx_example()
 
+Typically the files used to build and or run test executables are either
+cached from the installation (see :ref:`cache_extra_test_sources`) or added
+to the package repository (see :ref:`cache_custom_files`). There is nothing
+preventing the use of both.
 
 .. _cache_extra_test_sources:
 
-"""""""""""""""""""""""
+""""""""""""""""""""""""""""""""""""
-Saving build-time files
+Saving build- and install-time files
-"""""""""""""""""""""""
+""""""""""""""""""""""""""""""""""""
 
-.. note::
+You can use the ``cache_extra_test_sources`` helper routine to copy
-
+directories and or files from the source build stage directory to the
-    We highly recommend re-using build-time test sources and pared down
+package's installation directory. Spack will automatically copy these
-    input files for testing installed software. These files are easier
+files for you when it sets up the test stage directory and before it
-    to keep synchronized with software capabilities since they reside
+begins running the tests.
-    within the software's repository.
-
-    If that is not possible, you can add test-related files to the package
-    repository (see :ref:`adding custom files <cache_custom_files>`). It
-    will be important to maintain them so they work across listed or supported
-    versions of the package.
-
-You can use the ``cache_extra_test_sources`` helper to copy directories
-and or files from the source build stage directory to the package's
-installation directory.
 
 The signature for ``cache_extra_test_sources`` is:
 
@@ -5513,46 +5651,69 @@ where each argument has the following meaning:
 * ``srcs`` is a string *or* a list of strings corresponding to the
   paths of subdirectories and or files needed for stand-alone testing.
 
-The paths must be relative to the staged source directory. Contents of
+.. warning::
-subdirectories and files are copied to a special test cache subdirectory
-of the installation prefix. They are automatically copied to the appropriate
-relative paths under the test stage directory prior to executing stand-alone
-tests.
 
-For example, a package method for copying everything in the ``tests``
+   Paths provided in the ``srcs`` argument **must be relative** to the
-subdirectory plus the ``foo.c`` and ``bar.c`` files from ``examples``
+   staged source directory. They will be copied to the equivalent relative
-and using ``foo.c`` in a test method is illustrated below.
+   location under the test stage directory prior to test execution.
+
+Contents of subdirectories and files are copied to a special test cache
+subdirectory of the installation prefix. They are automatically copied to
+the appropriate relative paths under the test stage directory prior to
+executing stand-alone tests.
+
+.. tip:: 
+
+    *Perform test-related conversions once when copying files.*
+
+    If one or more of the copied files needs to be modified to reference
+    the installed software, it is recommended that those changes be made
+    to the cached files **once** in the post-``install`` copy method
+    **after** the call to ``cache_extra_test_sources``. This will reduce
+    the amount of unnecessary work in the test method **and** avoid problems
+    running stand-alone tests in shared instances and facility deployments.
+
+    The ``filter_file`` function can be quite useful for such changes
+    (see :ref:`file-filtering`).
+
+Below is a basic example of a test that relies on files from the installation.
+This package method re-uses the contents of the ``examples`` subdirectory,
+which is assumed to have all of the files implemented to allow ``make`` to
+compile and link ``foo.c`` and ``bar.c`` against the package's installed
+library.
 
 .. code-block:: python
 
-   class MyLibPackage(Package):
+   class MyLibPackage(MakefilePackage):
        ...
 
        @run_after("install")
        def copy_test_files(self):
-           srcs = ["tests",
+           cache_extra_test_sources(self, "examples")
-                   join_path("examples", "foo.c"),
-                   join_path("examples", "bar.c")]
-           cache_extra_test_sources(self, srcs)
 
-       def test_foo(self):
+       def test_example(self):
-           exe = "foo"
+           """build and run the examples"""
-           src_dir = self.test_suite.current_test_cache_dir.examples
+           examples_dir = self.test_suite.current_test_cache_dir.examples
-           with working_dir(src_dir):
+           with working_dir(examples_dir):
-               cc = which(os.environ["CC"])
+               make = which("make")
-               cc(
+               make()
-                   f"-L{self.prefix.lib}",
-                   f"-I{self.prefix.include}",
-                   f"{exe}.c",
-                   "-o", exe
-               )
-               foo = which(exe)
-               foo()
 
-In this case, the method copies the associated files from the build
+               for program in ["foo", "bar"]:
-stage, **after** the software is installed, to the package's test
+                   with test_part(
-cache directory. Then ``test_foo`` builds ``foo`` using ``foo.c``
+                       self,
-before running the program.
+                       f"test_example_{program}",
+                       purpose=f"ensure {program} runs"
+                   ):
+                       exe = Executable(program)
+                       exe()
+
+In this case, ``copy_test_files`` copies the associated files from the
+build stage to the package's test cache directory under the installation
+prefix. Running ``spack test run`` for the package results in Spack copying
+the directory and its contents to the the test stage directory. The
+``working_dir`` context manager ensures the commands within it are executed
+from the ``examples_dir``. The test builds the software using ``make`` before
+running each executable, ``foo`` and ``bar``, as independent test parts.
 
 .. note::
 
@@ -5561,43 +5722,18 @@ before running the program.
 
    The key to copying files for stand-alone testing at build time is use
    of the ``run_after`` directive, which ensures the associated files are
-   copied **after** the provided build stage where the files **and**
+   copied **after** the provided build stage (``install``) when the installation
-   installation prefix are available.
+   prefix **and** files are available.
 
-These paths are **automatically copied** from cache to the test stage
+   The test method uses the path contained in the package's
-directory prior to the execution of any stand-alone tests. Tests access
+   ``self.test_suite.current_test_cache_dir`` property for the root directory
-the files using the ``self.test_suite.current_test_cache_dir`` property.
+   of the copied files. In this case, that's the ``examples`` subdirectory.
-In our example above, test methods can use the following paths to reference
-the copy of each entry listed in ``srcs``, respectively:
 
-* ``self.test_suite.current_test_cache_dir.tests``
+.. tip::
-* ``join_path(self.test_suite.current_test_cache_dir.examples, "foo.c")``
-* ``join_path(self.test_suite.current_test_cache_dir.examples, "bar.c")``
-
-.. admonition:: Library packages should build stand-alone tests
-
-    Library developers will want to build the associated tests
-    against their **installed** libraries before running them.
-
-.. note::
-
-    While source and input files are generally recommended, binaries
-    **may** also be cached by the build process. Only you, as the package
-    writer or maintainer, know whether these files would be appropriate
-    for testing the installed software weeks to months later.
-
-.. note::
-
-    If one or more of the copied files needs to be modified to reference
-    the installed software, it is recommended that those changes be made
-    to the cached files **once** in the ``copy_test_sources`` method and
-    ***after** the call to ``cache_extra_test_sources()``. This will
-    reduce the amount of unnecessary work in the test method **and** avoid
-    problems testing in shared instances and facility deployments.
-
-    The ``filter_file`` function can be quite useful for such changes.
-    See :ref:`file manipulation <file-manipulation>`.
 
+   If you want to see more examples from packages that cache build files, run
+   ``spack pkg grep cache_extra_test_sources | sed "s/\/package.py.*//g" | sort -u``
+   from the command line to get a list of the packages.
 
 .. _cache_custom_files:
 
@@ -5605,8 +5741,9 @@ the copy of each entry listed in ``srcs``, respectively:
 Adding custom files
 """""""""""""""""""
 
-In some cases it can be useful to have files that can be used to build or
+Sometimes it is helpful or necessary to include custom files for building and
-check the results of tests.  Examples include:
+or checking the results of tests as part of the package. Examples of the types
+of files that might be useful are:
 
 - test source files
 - test input files
@@ -5614,17 +5751,15 @@ check the results of tests.  Examples include:
 - expected test outputs
 
 While obtaining such files from the software repository is preferred (see
-:ref:`adding build-time files  <cache_extra_test_sources>`), there are
+:ref:`cache_extra_test_sources`), there are circumstances where doing so is not
-circumstances where that is not feasible (e.g., the software is not being
+feasible such as when the software is not being actively maintained. When test
-actively maintained). When test files can't be obtained from the repository
+files cannot be obtained from the repository or there is a need to supplement
-or as a supplement to files that can, Spack supports the inclusion of
+files that can, Spack supports the inclusion of additional files under the
-additional files under the ``test`` subdirectory of the package in the
+``test`` subdirectory of the package in the Spack repository.
-Spack repository.
 
-Spack **automatically copies** the contents of that directory to the
+The following example assumes a ``custom-example.c`` is saved in ``MyLibary``
-test staging directory prior to running stand-alone tests. Test methods
+package's ``test`` subdirectory. It also assumes the program simply needs to
-access those files using the ``self.test_suite.current_test_data_dir``
+be compiled and linked against the installed ``MyLibrary`` software.
-property as shown below.
 
 .. code-block:: python
 
@@ -5634,17 +5769,29 @@ property as shown below.
        test_requires_compiler = True
        ...
 
-       def test_example(self):
+       def test_custom_example(self):
            """build and run custom-example"""
-           data_dir = self.test_suite.current_test_data_dir
+           src_dir = self.test_suite.current_test_data_dir
            exe = "custom-example"
-           src = datadir.join(f"{exe}.cpp")
-           ...
-           # TODO: Build custom-example using src and exe
-           ...
-           custom_example = which(exe)
-           custom_example()
 
+           with working_dir(src_dir):
+               cc = which(os.environ["CC"])
+               cc(
+                   f"-L{self.prefix.lib}",
+                   f"-I{self.prefix.include}",
+                   f"{exe}.cpp",
+                   "-o", exe
+               )
+
+               custom_example = Executable(exe)
+               custom_example()
+
+In this case, ``spack test run`` for the package results in Spack copying
+the contents of the ``test`` subdirectory to the test stage directory path
+in ``self.test_suite.current_test_data_dir`` before calling
+``test_custom_example``. Use of the ``working_dir`` context manager
+ensures the commands to build and run the program are performed from
+within the appropriate subdirectory of the test stage.
 
 .. _expected_test_output_from_file:
 
@@ -5653,9 +5800,8 @@ Reading expected output from a file
 """""""""""""""""""""""""""""""""""
 
 The helper function ``get_escaped_text_output`` is available for packages
-to retrieve and properly format the text from a file that contains the
+to retrieve properly formatted text from a file potentially containing
-expected output from running an executable that may contain special
+special characters.
-characters.
 
 The signature for ``get_escaped_text_output`` is:
 
@@ -5665,10 +5811,13 @@ The signature for ``get_escaped_text_output`` is:
 
 where ``filename`` is the path to the file containing the expected output.
 
-The ``filename`` for a :ref:`custom file <cache_custom_files>` can be
+The path provided to ``filename`` for one of the copied custom files
-accessed by tests using the ``self.test_suite.current_test_data_dir``
+(:ref:`custom file <cache_custom_files>`) is in the path rooted at
-property. The example below illustrates how to read a file that was
+``self.test_suite.current_test_data_dir``.
-added to the package's ``test`` subdirectory.
+
+The example below shows how to reference both the custom database
+(``packages.db``) and expected output (``dump.out``) files Spack copies
+to the test stage:
 
 .. code-block:: python
 
@@ -5690,8 +5839,9 @@ added to the package's ``test`` subdirectory.
            for exp in expected:
                assert re.search(exp, out), f"Expected '{exp}' in output"
 
-If the file was instead copied from the ``tests`` subdirectory of the staged
+If the files were instead cached from installing the software, the paths to the
-source code, the path would be obtained as shown below.
+two files would be found under the ``self.test_suite.current_test_cache_dir``
+directory as shown below:
 
 .. code-block:: python
 
@@ -5699,17 +5849,24 @@ source code, the path would be obtained as shown below.
            """check example table dump"""
            test_cache_dir = self.test_suite.current_test_cache_dir
            db_filename = test_cache_dir.join("packages.db")
+           ..
+           expected = get_escaped_text_output(test_cache_dir.join("dump.out"))
+           ...
 
-Alternatively, if the file was copied to the ``share/tests`` subdirectory
+Alternatively, if both files had been installed by the software into the
-as part of the installation process, the test could access the path as
+``share/tests`` subdirectory of the installation prefix, the paths to the
-follows:
+two files would be referenced as follows:
 
 .. code-block:: python
 
        def test_example(self):
            """check example table dump"""
-           db_filename = join_path(self.prefix.share.tests, "packages.db")
+           db_filename = self.prefix.share.tests.join("packages.db")
-
+           ..
+           expected = get_escaped_text_output(
+               self.prefix.share.tests.join("dump.out")
+           )
+           ...
 
 .. _check_outputs:
 
@@ -5717,9 +5874,9 @@ follows:
 Comparing expected to actual outputs
 """"""""""""""""""""""""""""""""""""
 
-The helper function ``check_outputs`` is available for packages to ensure
+The ``check_outputs`` helper routine is available for packages to ensure
-the expected outputs from running an executable are contained within the
+multiple expected outputs from running an executable are contained within
-actual outputs.
+the actual outputs.
 
 The signature for ``check_outputs`` is:
 
@@ -5745,11 +5902,17 @@ Invoking the method is the equivalent of:
    if errors:
        raise RuntimeError("\n ".join(errors))
 
+.. tip::
+
+   If you want to see more examples from packages that use this helper, run
+   ``spack pkg grep check_outputs | sed "s/\/package.py.*//g" | sort -u``
+   from the command line to get a list of the packages.
+
 
 .. _accessing-files:
 
 """""""""""""""""""""""""""""""""""""""""
-Accessing package- and test-related files
+Finding package- and test-related files
 """""""""""""""""""""""""""""""""""""""""
 
 You may need to access files from one or more locations when writing
@@ -5758,8 +5921,7 @@ include test source files or includes them but has no way to build the
 executables using the installed headers and libraries. In these cases
 you may need to reference the files relative to one or more root directory.
 The table below lists relevant path properties and provides additional
-examples of their use.
+examples of their use. See :ref:`expected_test_output_from_file` for
-:ref:`Reading expected output <expected_test_output_from_file>` provides
 examples of accessing files saved from the software repository, package
 repository, and installation.
 
@@ -5788,7 +5950,6 @@ repository, and installation.
      - ``self.test_suite.current_test_data_dir``
      - ``join_path(self.test_suite.current_test_data_dir, "hello.f90")``
 
-
 .. _inheriting-tests:
 
 """"""""""""""""""""""""""""
@@ -5831,7 +5992,7 @@ maintainers provide additional stand-alone tests customized to the package.
 .. warning::
 
    Any package that implements a test method with the same name as an
-   inherited method overrides the inherited method. If that is not the
+   inherited method will override the inherited method. If that is not the
    goal and you are not explicitly calling and adding functionality to
    the inherited method for the test, then make sure that all test methods
    and embedded test parts have unique test names.
@@ -5996,6 +6157,8 @@ running:
 This is already part of the boilerplate for packages created with
 ``spack create``.
 
+.. _file-filtering:
+
 ^^^^^^^^^^^^^^^^^^^
 Filtering functions
 ^^^^^^^^^^^^^^^^^^^

lib/spack/docs/pipelines.rst

@@ -253,17 +253,6 @@ can easily happen if it is not updated frequently, this behavior ensures that
 spack has a way to know for certain about the status of any concrete spec on
 the remote mirror, but can slow down pipeline generation significantly.
 
-The ``--optimize`` argument is experimental and runs the generated pipeline
-document through a series of optimization passes designed to reduce the size
-of the generated file.
-
-The ``--dependencies`` is also experimental and disables what in Gitlab is
-referred to as DAG scheduling, internally using the ``dependencies`` keyword
-rather than ``needs`` to list dependency jobs.  The drawback of using this option
-is that before any job can begin, all jobs in previous stages must first
-complete.  The benefit is that Gitlab allows more dependencies to be listed
-when using ``dependencies`` instead of ``needs``.
-
 The optional ``--output-file`` argument should be an absolute path (including
 file name) to the generated pipeline, and if not given, the default is
 ``./.gitlab-ci.yml``.

lib/spack/docs/requirements.txt

@@ -1,13 +1,13 @@
-sphinx==7.2.6
+sphinx==7.4.7
 sphinxcontrib-programoutput==0.17
-sphinx_design==0.5.0
+sphinx_design==0.6.1
 sphinx-rtd-theme==2.0.0
 python-levenshtein==0.25.1
 docutils==0.20.1
 pygments==2.18.0
-urllib3==2.2.1
+urllib3==2.2.2
-pytest==8.2.1
+pytest==8.3.2
 isort==5.13.2
-black==24.4.2
+black==24.8.0
-flake8==7.0.0
+flake8==7.1.1
-mypy==1.10.0
+mypy==1.11.1

lib/spack/env/cc

@@ -174,6 +174,46 @@ preextend() {
     unset IFS
 }
 
+execute() {
+    # dump the full command if the caller supplies SPACK_TEST_COMMAND=dump-args
+    if [ -n "${SPACK_TEST_COMMAND=}" ]; then
+        case "$SPACK_TEST_COMMAND" in
+            dump-args)
+                IFS="$lsep"
+                for arg in $full_command_list; do
+                    echo "$arg"
+                done
+                unset IFS
+                exit
+                ;;
+            dump-env-*)
+                var=${SPACK_TEST_COMMAND#dump-env-}
+                eval "printf '%s\n' \"\$0: \$var: \$$var\""
+                ;;
+            *)
+                die "Unknown test command: '$SPACK_TEST_COMMAND'"
+                ;;
+        esac
+    fi
+
+    #
+    # Write the input and output commands to debug logs if it's asked for.
+    #
+    if [ "$SPACK_DEBUG" = TRUE ]; then
+        input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_DEBUG_LOG_ID.in.log"
+        output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_DEBUG_LOG_ID.out.log"
+        echo "[$mode] $command $input_command" >> "$input_log"
+        IFS="$lsep"
+        echo "[$mode] "$full_command_list >> "$output_log"
+        unset IFS
+    fi
+
+    # Execute the full command, preserving spaces with IFS set
+    # to the alarm bell separator.
+    IFS="$lsep"; exec $full_command_list
+    exit
+}
+
 # Fail with a clear message if the input contains any bell characters.
 if eval "[ \"\${*#*${lsep}}\" != \"\$*\" ]"; then
     die "Compiler command line contains our separator ('${lsep}'). Cannot parse."
@@ -231,12 +271,17 @@ fi
 #    ld      link
 #    ccld    compile & link
 
+# Note. SPACK_ALWAYS_XFLAGS are applied for all compiler invocations,
+# including version checks (SPACK_XFLAGS variants are not applied
+# for version checks).
 command="${0##*/}"
 comp="CC"
+vcheck_flags=""
 case "$command" in
     cpp)
         mode=cpp
         debug_flags="-g"
+        vcheck_flags="${SPACK_ALWAYS_CPPFLAGS}"
         ;;
     cc|c89|c99|gcc|clang|armclang|icc|icx|pgcc|nvc|xlc|xlc_r|fcc|amdclang|cl.exe|craycc)
         command="$SPACK_CC"
@@ -244,6 +289,7 @@ case "$command" in
         comp="CC"
         lang_flags=C
         debug_flags="-g"
+        vcheck_flags="${SPACK_ALWAYS_CFLAGS}"
         ;;
     c++|CC|g++|clang++|armclang++|icpc|icpx|pgc++|nvc++|xlc++|xlc++_r|FCC|amdclang++|crayCC)
         command="$SPACK_CXX"
@@ -251,6 +297,7 @@ case "$command" in
         comp="CXX"
         lang_flags=CXX
         debug_flags="-g"
+        vcheck_flags="${SPACK_ALWAYS_CXXFLAGS}"
         ;;
     ftn|f90|fc|f95|gfortran|flang|armflang|ifort|ifx|pgfortran|nvfortran|xlf90|xlf90_r|nagfor|frt|amdflang|crayftn)
         command="$SPACK_FC"
@@ -258,6 +305,7 @@ case "$command" in
         comp="FC"
         lang_flags=F
         debug_flags="-g"
+        vcheck_flags="${SPACK_ALWAYS_FFLAGS}"
         ;;
     f77|xlf|xlf_r|pgf77)
         command="$SPACK_F77"
@@ -265,6 +313,7 @@ case "$command" in
         comp="F77"
         lang_flags=F
         debug_flags="-g"
+        vcheck_flags="${SPACK_ALWAYS_FFLAGS}"
         ;;
     ld|ld.gold|ld.lld)
         mode=ld
@@ -365,7 +414,11 @@ unset IFS
 export PATH="$new_dirs"
 
 if [ "$mode" = vcheck ]; then
-    exec "${command}" "$@"
+    full_command_list="$command"
+    args="$@"
+    extend full_command_list vcheck_flags
+    extend full_command_list args
+    execute
 fi
 
 # Darwin's linker has a -r argument that merges object files together.
@@ -722,6 +775,7 @@ case "$mode" in
     cc|ccld)
         case $lang_flags in
             F)
+                extend spack_flags_list SPACK_ALWAYS_FFLAGS
                 extend spack_flags_list SPACK_FFLAGS
                 ;;
         esac
@@ -731,6 +785,7 @@ esac
 # C preprocessor flags come before any C/CXX flags
 case "$mode" in
     cpp|as|cc|ccld)
+        extend spack_flags_list SPACK_ALWAYS_CPPFLAGS
         extend spack_flags_list SPACK_CPPFLAGS
         ;;
 esac
@@ -741,9 +796,11 @@ case "$mode" in
     cc|ccld)
         case $lang_flags in
             C)
+                extend spack_flags_list SPACK_ALWAYS_CFLAGS
                 extend spack_flags_list SPACK_CFLAGS
                 ;;
             CXX)
+                extend spack_flags_list SPACK_ALWAYS_CXXFLAGS
                 extend spack_flags_list SPACK_CXXFLAGS
                 ;;
         esac
@@ -933,39 +990,4 @@ if [ -n "$SPACK_CCACHE_BINARY" ]; then
     esac
 fi
 
-# dump the full command if the caller supplies SPACK_TEST_COMMAND=dump-args
+execute
-if [ -n "${SPACK_TEST_COMMAND=}" ]; then
-    case "$SPACK_TEST_COMMAND" in
-        dump-args)
-            IFS="$lsep"
-            for arg in $full_command_list; do
-                echo "$arg"
-            done
-            unset IFS
-            exit
-            ;;
-        dump-env-*)
-            var=${SPACK_TEST_COMMAND#dump-env-}
-            eval "printf '%s\n' \"\$0: \$var: \$$var\""
-            ;;
-        *)
-            die "Unknown test command: '$SPACK_TEST_COMMAND'"
-            ;;
-    esac
-fi
-
-#
-# Write the input and output commands to debug logs if it's asked for.
-#
-if [ "$SPACK_DEBUG" = TRUE ]; then
-    input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_DEBUG_LOG_ID.in.log"
-    output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_DEBUG_LOG_ID.out.log"
-    echo "[$mode] $command $input_command" >> "$input_log"
-    IFS="$lsep"
-    echo "[$mode] "$full_command_list >> "$output_log"
-    unset IFS
-fi
-
-# Execute the full command, preserving spaces with IFS set
-# to the alarm bell separator.
-IFS="$lsep"; exec $full_command_list

lib/spack/llnl/util/tty/log.py

@@ -33,8 +33,23 @@
     pass
 
 
+esc, bell, lbracket, bslash, newline = r"\x1b", r"\x07", r"\[", r"\\", r"\n"
+# Ansi Control Sequence Introducers (CSI) are a well-defined format
+# Standard ECMA-48: Control Functions for Character-Imaging I/O Devices, section 5.4
+# https://www.ecma-international.org/wp-content/uploads/ECMA-48_5th_edition_june_1991.pdf
+csi_pre = f"{esc}{lbracket}"
+csi_param, csi_inter, csi_post = r"[0-?]", r"[ -/]", r"[@-~]"
+ansi_csi = f"{csi_pre}{csi_param}*{csi_inter}*{csi_post}"
+# General ansi escape sequences have well-defined prefixes,
+#  but content and suffixes are less reliable.
+# Conservatively assume they end with either "<ESC>\" or "<BELL>",
+#  with no intervening "<ESC>"/"<BELL>" keys or newlines
+esc_pre = f"{esc}[@-_]"
+esc_content = f"[^{esc}{bell}{newline}]"
+esc_post = f"(?:{esc}{bslash}|{bell})"
+ansi_esc = f"{esc_pre}{esc_content}*{esc_post}"
 # Use this to strip escape sequences
-_escape = re.compile(r"\x1b[^m]*m|\x1b\[?1034h|\x1b\][0-9]+;[^\x07]*\x07")
+_escape = re.compile(f"{ansi_csi}|{ansi_esc}")
 
 # control characters for enabling/disabling echo
 #

lib/spack/spack/audit.py

@@ -351,6 +351,22 @@ def _wrongly_named_spec(error_cls):
     return errors
 
 
+@config_packages
+def _ensure_all_virtual_packages_have_default_providers(error_cls):
+    """All virtual packages must have a default provider explicitly set."""
+    configuration = spack.config.create()
+    defaults = configuration.get("packages", scope="defaults")
+    default_providers = defaults["all"]["providers"]
+    virtuals = spack.repo.PATH.provider_index.providers
+    default_providers_filename = configuration.scopes["defaults"].get_section_filename("packages")
+
+    return [
+        error_cls(f"'{virtual}' must have a default provider in {default_providers_filename}", [])
+        for virtual in virtuals
+        if virtual not in default_providers
+    ]
+
+
 def _make_config_error(config_data, summary, error_cls):
     s = io.StringIO()
     s.write("Occurring in the following file:\n")
@@ -791,7 +807,7 @@ def check_virtual_with_variants(spec, msg):
                         return
                     error = error_cls(
                         f"{pkg_name}: {msg}",
-                        f"remove variants from '{spec}' in depends_on directive in {filename}",
+                        [f"remove variants from '{spec}' in depends_on directive in {filename}"],
                     )
                     errors.append(error)
 

lib/spack/spack/bootstrap/config.py

@@ -129,10 +129,10 @@ def _bootstrap_config_scopes() -> Sequence["spack.config.ConfigScope"]:
     configuration_paths = (spack.config.CONFIGURATION_DEFAULTS_PATH, ("bootstrap", _config_path()))
     for name, path in configuration_paths:
         platform = spack.platforms.host().name
-        platform_scope = spack.config.ConfigScope(
+        platform_scope = spack.config.DirectoryConfigScope(
-            "/".join([name, platform]), os.path.join(path, platform)
+            f"{name}/{platform}", os.path.join(path, platform)
         )
-        generic_scope = spack.config.ConfigScope(name, path)
+        generic_scope = spack.config.DirectoryConfigScope(name, path)
         config_scopes.extend([generic_scope, platform_scope])
         msg = "[BOOTSTRAP CONFIG SCOPE] name={0}, path={1}"
         tty.debug(msg.format(generic_scope.name, generic_scope.path))

lib/spack/spack/build_environment.py

@@ -72,6 +72,7 @@
 import spack.store
 import spack.subprocess_context
 import spack.user_environment
+import spack.util.executable
 import spack.util.path
 import spack.util.pattern
 from spack import traverse
@@ -458,10 +459,7 @@ def set_wrapper_variables(pkg, env):
 
     # Find ccache binary and hand it to build environment
     if spack.config.get("config:ccache"):
-        ccache = Executable("ccache")
+        env.set(SPACK_CCACHE_BINARY, spack.util.executable.which_string("ccache", required=True))
-        if not ccache:
-            raise RuntimeError("No ccache binary found in PATH")
-        env.set(SPACK_CCACHE_BINARY, ccache)
 
     # Gather information about various types of dependencies
     link_deps = set(pkg.spec.traverse(root=False, deptype=("link")))
@@ -740,7 +738,9 @@ def get_rpaths(pkg):
     # Second module is our compiler mod name. We use that to get rpaths from
     # module show output.
     if pkg.compiler.modules and len(pkg.compiler.modules) > 1:
-        rpaths.append(path_from_modules([pkg.compiler.modules[1]]))
+        mod_rpath = path_from_modules([pkg.compiler.modules[1]])
+        if mod_rpath:
+            rpaths.append(mod_rpath)
     return list(dedupe(filter_system_paths(rpaths)))
 
 
@@ -1473,7 +1473,7 @@ def long_message(self):
             out.write("  {0}\n".format(self.log_name))
 
         # Also output the test log path IF it exists
-        if self.context != "test":
+        if self.context != "test" and have_log:
             test_log = join_path(os.path.dirname(self.log_name), spack_install_test_log)
             if os.path.isfile(test_log):
                 out.write("\nSee test log for details:\n")

lib/spack/spack/build_systems/cached_cmake.py

@@ -162,7 +162,9 @@ def initconfig_compiler_entries(self):
             ld_flags = " ".join(flags["ldflags"])
             ld_format_string = "CMAKE_{0}_LINKER_FLAGS"
             # CMake has separate linker arguments for types of builds.
-            for ld_type in ["EXE", "MODULE", "SHARED", "STATIC"]:
+            # 'ldflags' should not be used with CMAKE_STATIC_LINKER_FLAGS which
+            # is used by the archiver, so don't include "STATIC" in this loop:
+            for ld_type in ["EXE", "MODULE", "SHARED"]:
                 ld_string = ld_format_string.format(ld_type)
                 entries.append(cmake_cache_string(ld_string, ld_flags))
 

lib/spack/spack/build_systems/cuda.py

@@ -124,6 +124,8 @@ def cuda_flags(arch_list):
         # minimum supported versions
         conflicts("%gcc@:4", when="+cuda ^cuda@11.0:")
         conflicts("%gcc@:5", when="+cuda ^cuda@11.4:")
+        conflicts("%gcc@:7.2", when="+cuda ^cuda@12.4:")
+        conflicts("%clang@:6", when="+cuda ^cuda@12.2:")
 
         # maximum supported version
         # NOTE:
@@ -136,14 +138,14 @@ def cuda_flags(arch_list):
         conflicts("%gcc@11.2:", when="+cuda ^cuda@:11.5")
         conflicts("%gcc@12:", when="+cuda ^cuda@:11.8")
         conflicts("%gcc@13:", when="+cuda ^cuda@:12.3")
-        conflicts("%gcc@14:", when="+cuda ^cuda@:12.4")
+        conflicts("%gcc@14:", when="+cuda ^cuda@:12.5")
         conflicts("%clang@12:", when="+cuda ^cuda@:11.4.0")
         conflicts("%clang@13:", when="+cuda ^cuda@:11.5")
         conflicts("%clang@14:", when="+cuda ^cuda@:11.7")
         conflicts("%clang@15:", when="+cuda ^cuda@:12.0")
         conflicts("%clang@16:", when="+cuda ^cuda@:12.1")
         conflicts("%clang@17:", when="+cuda ^cuda@:12.3")
-        conflicts("%clang@18:", when="+cuda ^cuda@:12.4")
+        conflicts("%clang@18:", when="+cuda ^cuda@:12.5")
 
         # https://gist.github.com/ax3l/9489132#gistcomment-3860114
         conflicts("%gcc@10", when="+cuda ^cuda@:11.4.0")
@@ -211,12 +213,16 @@ def cuda_flags(arch_list):
         conflicts("%intel@19.0:", when="+cuda ^cuda@:10.0")
         conflicts("%intel@19.1:", when="+cuda ^cuda@:10.1")
         conflicts("%intel@19.2:", when="+cuda ^cuda@:11.1.0")
+        conflicts("%intel@2021:", when="+cuda ^cuda@:11.4.0")
 
         # XL is mostly relevant for ppc64le Linux
         conflicts("%xl@:12,14:", when="+cuda ^cuda@:9.1")
         conflicts("%xl@:12,14:15,17:", when="+cuda ^cuda@9.2")
         conflicts("%xl@:12,17:", when="+cuda ^cuda@:11.1.0")
 
+        # PowerPC.
+        conflicts("target=ppc64le", when="+cuda ^cuda@12.5:")
+
         # Darwin.
         # TODO: add missing conflicts for %apple-clang cuda@:10
-        conflicts("platform=darwin", when="+cuda ^cuda@11.0.2: ")
+        conflicts("platform=darwin", when="+cuda ^cuda@11.0.2:")

lib/spack/spack/build_systems/go.py

@@ -72,7 +72,7 @@ def build_directory(self):
     def build_args(self):
         """Arguments for ``go build``."""
         # Pass ldflags -s = --strip-all and -w = --no-warnings by default
-        return ["-ldflags", "-s -w", "-o", f"{self.pkg.name}"]
+        return ["-modcacherw", "-ldflags", "-s -w", "-o", f"{self.pkg.name}"]
 
     @property
     def check_args(self):

lib/spack/spack/build_systems/python.py

@@ -354,10 +354,18 @@ def homepage(cls) -> Optional[str]:  # type: ignore[override]
         return None
 
     @lang.classproperty
-    def url(cls) -> Optional[str]:
+    def urls(cls) -> Optional[List[str]]:
         if cls.pypi:
-            return f"https://files.pythonhosted.org/packages/source/{cls.pypi[0]}/{cls.pypi}"
+            urls = [f"https://files.pythonhosted.org/packages/source/{cls.pypi[0]}/{cls.pypi}"]
-        return None
+            assert cls.pypi.count("/") == 1, "PyPI class attribute must include a single slash"
+            name, file = cls.pypi.split("/")
+            name_dash_count = name.count("-")
+            if name_dash_count > 0:
+                # replace all but last dash with underscores for pypi.org listing changes
+                pypi = "/".join([name, file.replace("-", "_", name_dash_count)])
+                urls.append(f"https://files.pythonhosted.org/packages/source/{pypi[0]}/{pypi}")
+            return urls
+        return [None]
 
     @lang.classproperty
     def list_url(cls) -> Optional[str]:  # type: ignore[override]

lib/spack/spack/build_systems/sip.py

@@ -139,6 +139,10 @@ def configure(self, pkg, spec, prefix):
         args = ["--verbose", "--target-dir", inspect.getmodule(self.pkg).python_platlib]
         args.extend(self.configure_args())
 
+        # https://github.com/Python-SIP/sip/commit/cb0be6cb6e9b756b8b0db3136efb014f6fb9b766
+        if spec["py-sip"].satisfies("@6.1.0:"):
+            args.extend(["--scripts-dir", pkg.prefix.bin])
+
         sip_build = Executable(spec["py-sip"].prefix.bin.join("sip-build"))
         sip_build(*args)
 

lib/spack/spack/caches.py

@@ -34,6 +34,8 @@ def _misc_cache():
     return spack.util.file_cache.FileCache(path)
 
 
+FileCacheType = Union[spack.util.file_cache.FileCache, llnl.util.lang.Singleton]
+
 #: Spack's cache for small data
 MISC_CACHE: Union[spack.util.file_cache.FileCache, llnl.util.lang.Singleton] = (
     llnl.util.lang.Singleton(_misc_cache)

lib/spack/spack/ci.py

@@ -22,6 +22,8 @@
 from urllib.parse import urlencode
 from urllib.request import HTTPHandler, Request, build_opener
 
+import ruamel.yaml
+
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 from llnl.util.lang import memoized
@@ -69,7 +71,7 @@
 # TODO: Remove this in Spack 0.23
 SHARED_PR_MIRROR_URL = "s3://spack-binaries-prs/shared_pr_mirror"
 JOB_NAME_FORMAT = (
-    "{name}{@version} {/hash:7} {%compiler.name}{@compiler.version}{arch=architecture}"
+    "{name}{@version} {/hash:7} {%compiler.name}{@compiler.version}{ arch=architecture}"
 )
 IS_WINDOWS = sys.platform == "win32"
 spack_gpg = spack.main.SpackCommand("gpg")
@@ -551,10 +553,9 @@ def generate_gitlab_ci_yaml(
     env,
     print_summary,
     output_file,
+    *,
     prune_dag=False,
     check_index_only=False,
-    run_optimizer=False,
-    use_dependencies=False,
     artifacts_root=None,
     remote_mirror_override=None,
 ):
@@ -575,12 +576,6 @@ def generate_gitlab_ci_yaml(
             this mode results in faster yaml generation time). Otherwise, also
             check each spec directly by url (useful if there is no index or it
             might be out of date).
-        run_optimizer (bool): If True, post-process the generated yaml to try
-            try to reduce the size (attempts to collect repeated configuration
-            and replace with definitions).)
-        use_dependencies (bool): If true, use "dependencies" rather than "needs"
-            ("needs" allows DAG scheduling).  Useful if gitlab instance cannot
-            be configured to handle more than a few "needs" per job.
         artifacts_root (str): Path where artifacts like logs, environment
             files (spack.yaml, spack.lock), etc should be written.  GitLab
             requires this to be within the project directory.
@@ -814,7 +809,8 @@ def ensure_expected_target_path(path):
         cli_scopes = [
             os.path.relpath(s.path, concrete_env_dir)
             for s in cfg.scopes().values()
-            if isinstance(s, cfg.ImmutableConfigScope)
+            if not s.writable
+            and isinstance(s, (cfg.DirectoryConfigScope))
             and s.path not in env_includes
             and os.path.exists(s.path)
         ]
@@ -1271,17 +1267,6 @@ def main_script_replacements(cmd):
             with open(copy_specs_file, "w") as fd:
                 fd.write(json.dumps(buildcache_copies))
 
-        # TODO(opadron): remove this or refactor
-        if run_optimizer:
-            import spack.ci_optimization as ci_opt
-
-            output_object = ci_opt.optimizer(output_object)
-
-        # TODO(opadron): remove this or refactor
-        if use_dependencies:
-            import spack.ci_needs_workaround as cinw
-
-            output_object = cinw.needs_to_dependencies(output_object)
     else:
         # No jobs were generated
         noop_job = spack_ci_ir["jobs"]["noop"]["attributes"]
@@ -1310,8 +1295,11 @@ def main_script_replacements(cmd):
         if not rebuild_everything:
             sys.exit(1)
 
-    with open(output_file, "w") as outf:
+    # Minimize yaml output size through use of anchors
-        outf.write(syaml.dump(sorted_output, default_flow_style=True))
+    syaml.anchorify(sorted_output)
+
+    with open(output_file, "w") as f:
+        ruamel.yaml.YAML().dump(sorted_output, f)
 
 
 def _url_encode_string(input_string):

lib/spack/spack/ci_needs_workaround.py

@@ -1,34 +0,0 @@
-# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import collections.abc
-
-get_job_name = lambda needs_entry: (
-    needs_entry.get("job")
-    if (isinstance(needs_entry, collections.abc.Mapping) and needs_entry.get("artifacts", True))
-    else needs_entry if isinstance(needs_entry, str) else None
-)
-
-
-def convert_job(job_entry):
-    if not isinstance(job_entry, collections.abc.Mapping):
-        return job_entry
-
-    needs = job_entry.get("needs")
-    if needs is None:
-        return job_entry
-
-    new_job = {}
-    new_job.update(job_entry)
-    del new_job["needs"]
-
-    new_job["dependencies"] = list(
-        filter((lambda x: x is not None), (get_job_name(needs_entry) for needs_entry in needs))
-    )
-
-    return new_job
-
-
-def needs_to_dependencies(yaml):
-    return dict((k, convert_job(v)) for k, v in yaml.items())

lib/spack/spack/ci_optimization.py

@@ -1,363 +0,0 @@
-# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import collections
-import collections.abc
-import copy
-import hashlib
-
-import spack.util.spack_yaml as syaml
-
-
-def sort_yaml_obj(obj):
-    if isinstance(obj, collections.abc.Mapping):
-        return syaml.syaml_dict(
-            (k, sort_yaml_obj(v)) for k, v in sorted(obj.items(), key=(lambda item: str(item[0])))
-        )
-
-    if isinstance(obj, collections.abc.Sequence) and not isinstance(obj, str):
-        return syaml.syaml_list(sort_yaml_obj(x) for x in obj)
-
-    return obj
-
-
-def matches(obj, proto):
-    """Returns True if the test object "obj" matches the prototype object
-    "proto".
-
-    If obj and proto are mappings, obj matches proto if (key in obj) and
-    (obj[key] matches proto[key]) for every key in proto.
-
-    If obj and proto are sequences, obj matches proto if they are of the same
-    length and (a matches b) for every (a,b) in zip(obj, proto).
-
-    Otherwise, obj matches proto if obj == proto.
-
-    Precondition: proto must not have any reference cycles
-    """
-    if isinstance(obj, collections.abc.Mapping):
-        if not isinstance(proto, collections.abc.Mapping):
-            return False
-
-        return all((key in obj and matches(obj[key], val)) for key, val in proto.items())
-
-    if isinstance(obj, collections.abc.Sequence) and not isinstance(obj, str):
-        if not (isinstance(proto, collections.abc.Sequence) and not isinstance(proto, str)):
-            return False
-
-        if len(obj) != len(proto):
-            return False
-
-        return all(matches(obj[index], val) for index, val in enumerate(proto))
-
-    return obj == proto
-
-
-def subkeys(obj, proto):
-    """Returns the test mapping "obj" after factoring out the items it has in
-    common with the prototype mapping "proto".
-
-    Consider a recursive merge operation, merge(a, b) on mappings a and b, that
-    returns a mapping, m, whose keys are the union of the keys of a and b, and
-    for every such key, "k", its corresponding value is:
-
-      - merge(a[key], b[key])  if a[key] and b[key] are mappings, or
-      - b[key]                 if (key in b) and not matches(a[key], b[key]),
-                               or
-      - a[key]                 otherwise
-
-
-    If obj and proto are mappings, the returned object is the smallest object,
-    "a", such that merge(a, proto) matches obj.
-
-    Otherwise, obj is returned.
-    """
-    if not (
-        isinstance(obj, collections.abc.Mapping) and isinstance(proto, collections.abc.Mapping)
-    ):
-        return obj
-
-    new_obj = {}
-    for key, value in obj.items():
-        if key not in proto:
-            new_obj[key] = value
-            continue
-
-        if matches(value, proto[key]) and matches(proto[key], value):
-            continue
-
-        if isinstance(value, collections.abc.Mapping):
-            new_obj[key] = subkeys(value, proto[key])
-            continue
-
-        new_obj[key] = value
-
-    return new_obj
-
-
-def add_extends(yaml, key):
-    """Modifies the given object "yaml" so that it includes an "extends" key
-    whose value features "key".
-
-    If "extends" is not in yaml, then yaml is modified such that
-    yaml["extends"] == key.
-
-    If yaml["extends"] is a str, then yaml is modified such that
-    yaml["extends"] == [yaml["extends"], key]
-
-    If yaml["extends"] is a list that does not include key, then key is
-    appended to the list.
-
-    Otherwise, yaml is left unchanged.
-    """
-
-    has_key = "extends" in yaml
-    extends = yaml.get("extends")
-
-    if has_key and not isinstance(extends, (str, collections.abc.Sequence)):
-        return
-
-    if extends is None:
-        yaml["extends"] = key
-        return
-
-    if isinstance(extends, str):
-        if extends != key:
-            yaml["extends"] = [extends, key]
-        return
-
-    if key not in extends:
-        extends.append(key)
-
-
-def common_subobject(yaml, sub):
-    """Factor prototype object "sub" out of the values of mapping "yaml".
-
-    Consider a modified copy of yaml, "new", where for each key, "key" in yaml:
-
-      - If yaml[key] matches sub, then new[key] = subkeys(yaml[key], sub).
-      - Otherwise, new[key] = yaml[key].
-
-    If the above match criteria is not satisfied for any such key, then (yaml,
-    None) is returned. The yaml object is returned unchanged.
-
-    Otherwise, each matching value in new is modified as in
-    add_extends(new[key], common_key), and then new[common_key] is set to sub.
-    The common_key value is chosen such that it does not match any preexisting
-    key in new. In this case, (new, common_key) is returned.
-    """
-    match_list = set(k for k, v in yaml.items() if matches(v, sub))
-
-    if not match_list:
-        return yaml, None
-
-    common_prefix = ".c"
-    common_index = 0
-
-    while True:
-        common_key = "".join((common_prefix, str(common_index)))
-        if common_key not in yaml:
-            break
-        common_index += 1
-
-    new_yaml = {}
-
-    for key, val in yaml.items():
-        new_yaml[key] = copy.deepcopy(val)
-
-        if not matches(val, sub):
-            continue
-
-        new_yaml[key] = subkeys(new_yaml[key], sub)
-        add_extends(new_yaml[key], common_key)
-
-    new_yaml[common_key] = sub
-
-    return new_yaml, common_key
-
-
-def print_delta(name, old, new, applied=None):
-    delta = new - old
-    reldelta = (1000 * delta) // old
-    reldelta = (reldelta // 10, reldelta % 10)
-
-    if applied is None:
-        applied = new <= old
-
-    print(
-        "\n".join(
-            (
-                "{0} {1}:",
-                "  before: {2: 10d}",
-                "  after : {3: 10d}",
-                "  delta : {4:+10d} ({5:=+3d}.{6}%)",
-            )
-        ).format(name, ("+" if applied else "x"), old, new, delta, reldelta[0], reldelta[1])
-    )
-
-
-def try_optimization_pass(name, yaml, optimization_pass, *args, **kwargs):
-    """Try applying an optimization pass and return information about the
-    result
-
-    "name" is a string describing the nature of the pass. If it is a non-empty
-    string, summary statistics are also printed to stdout.
-
-    "yaml" is the object to apply the pass to.
-
-    "optimization_pass" is the function implementing the pass to be applied.
-
-    "args" and "kwargs" are the additional arguments to pass to optimization
-    pass. The pass is applied as
-
-    >>> (new_yaml, *other_results) = optimization_pass(yaml, *args, **kwargs)
-
-    The pass's results are greedily rejected if it does not modify the original
-    yaml document, or if it produces a yaml document that serializes to a
-    larger string.
-
-    Returns (new_yaml, yaml, applied, other_results) if applied, or
-    (yaml, new_yaml, applied, other_results) otherwise.
-    """
-    result = optimization_pass(yaml, *args, **kwargs)
-    new_yaml, other_results = result[0], result[1:]
-
-    if new_yaml is yaml:
-        # pass was not applied
-        return (yaml, new_yaml, False, other_results)
-
-    pre_size = len(syaml.dump_config(sort_yaml_obj(yaml), default_flow_style=True))
-    post_size = len(syaml.dump_config(sort_yaml_obj(new_yaml), default_flow_style=True))
-
-    # pass makes the size worse: not applying
-    applied = post_size <= pre_size
-    if applied:
-        yaml, new_yaml = new_yaml, yaml
-
-    if name:
-        print_delta(name, pre_size, post_size, applied)
-
-    return (yaml, new_yaml, applied, other_results)
-
-
-def build_histogram(iterator, key):
-    """Builds a histogram of values given an iterable of mappings and a key.
-
-    For each mapping "m" with key "key" in iterator, the value m[key] is
-    considered.
-
-    Returns a list of tuples (hash, count, proportion, value), where
-
-      - "hash" is a sha1sum hash of the value.
-      - "count" is the number of occurences of values that hash to "hash".
-      - "proportion" is the proportion of all values considered above that
-        hash to "hash".
-      - "value" is one of the values considered above that hash to "hash".
-        Which value is chosen when multiple values hash to the same "hash" is
-        undefined.
-
-    The list is sorted in descending order by count, yielding the most
-    frequently occuring hashes first.
-    """
-    buckets = collections.defaultdict(int)
-    values = {}
-
-    num_objects = 0
-    for obj in iterator:
-        num_objects += 1
-
-        try:
-            val = obj[key]
-        except (KeyError, TypeError):
-            continue
-
-        value_hash = hashlib.sha1()
-        value_hash.update(syaml.dump_config(sort_yaml_obj(val)).encode())
-        value_hash = value_hash.hexdigest()
-
-        buckets[value_hash] += 1
-        values[value_hash] = val
-
-    return [
-        (h, buckets[h], float(buckets[h]) / num_objects, values[h])
-        for h in sorted(buckets.keys(), key=lambda k: -buckets[k])
-    ]
-
-
-def optimizer(yaml):
-    original_size = len(syaml.dump_config(sort_yaml_obj(yaml), default_flow_style=True))
-
-    # try factoring out commonly repeated portions
-    common_job = {
-        "variables": {"SPACK_COMPILER_ACTION": "NONE"},
-        "after_script": ['rm -rf "./spack"'],
-        "artifacts": {"paths": ["jobs_scratch_dir", "cdash_report"], "when": "always"},
-    }
-
-    # look for a list of tags that appear frequently
-    _, count, proportion, tags = next(iter(build_histogram(yaml.values(), "tags")), (None,) * 4)
-
-    # If a list of tags is found, and there are more than one job that uses it,
-    # *and* the jobs that do use it represent at least 70% of all jobs, then
-    # add the list to the prototype object.
-    if tags and count > 1 and proportion >= 0.70:
-        common_job["tags"] = tags
-
-    # apply common object factorization
-    yaml, other, applied, rest = try_optimization_pass(
-        "general common object factorization", yaml, common_subobject, common_job
-    )
-
-    # look for a common script, and try factoring that out
-    _, count, proportion, script = next(
-        iter(build_histogram(yaml.values(), "script")), (None,) * 4
-    )
-
-    if script and count > 1 and proportion >= 0.70:
-        yaml, other, applied, rest = try_optimization_pass(
-            "script factorization", yaml, common_subobject, {"script": script}
-        )
-
-    # look for a common before_script, and try factoring that out
-    _, count, proportion, script = next(
-        iter(build_histogram(yaml.values(), "before_script")), (None,) * 4
-    )
-
-    if script and count > 1 and proportion >= 0.70:
-        yaml, other, applied, rest = try_optimization_pass(
-            "before_script factorization", yaml, common_subobject, {"before_script": script}
-        )
-
-    # Look specifically for the SPACK_ROOT_SPEC environment variables.
-    # Try to factor them out.
-    h = build_histogram(
-        (getattr(val, "get", lambda *args: {})("variables") for val in yaml.values()),
-        "SPACK_ROOT_SPEC",
-    )
-
-    # In this case, we try to factor out *all* instances of the SPACK_ROOT_SPEC
-    # environment variable; not just the one that appears with the greatest
-    # frequency. We only require that more than 1 job uses a given instance's
-    # value, because we expect the value to be very large, and so expect even
-    # few-to-one factorizations to yield large space savings.
-    counter = 0
-    for _, count, proportion, spec in h:
-        if count <= 1:
-            continue
-
-        counter += 1
-
-        yaml, other, applied, rest = try_optimization_pass(
-            "SPACK_ROOT_SPEC factorization ({count})".format(count=counter),
-            yaml,
-            common_subobject,
-            {"variables": {"SPACK_ROOT_SPEC": spec}},
-        )
-
-    new_size = len(syaml.dump_config(sort_yaml_obj(yaml), default_flow_style=True))
-
-    print("\n")
-    print_delta("overall summary", original_size, new_size)
-    print("\n")
-    return yaml

lib/spack/spack/cmd/__init__.py

@@ -237,7 +237,7 @@ def ensure_single_spec_or_die(spec, matching_specs):
     if len(matching_specs) <= 1:
         return
 
-    format_string = "{name}{@version}{%compiler.name}{@compiler.version}{arch=architecture}"
+    format_string = "{name}{@version}{%compiler.name}{@compiler.version}{ arch=architecture}"
     args = ["%s matches multiple packages." % spec, "Matching packages:"]
     args += [
         colorize("  @K{%s} " % s.dag_hash(7)) + s.cformat(format_string) for s in matching_specs
@@ -336,6 +336,7 @@ def display_specs(specs, args=None, **kwargs):
         groups (bool): display specs grouped by arch/compiler (default True)
         decorator (typing.Callable): function to call to decorate specs
         all_headers (bool): show headers even when arch/compiler aren't defined
+        status_fn (typing.Callable): if provided, prepend install-status info
         output (typing.IO): A file object to write to. Default is ``sys.stdout``
 
     """
@@ -359,6 +360,7 @@ def get_arg(name, default=None):
     groups = get_arg("groups", True)
     all_headers = get_arg("all_headers", False)
     output = get_arg("output", sys.stdout)
+    status_fn = get_arg("status_fn", None)
 
     decorator = get_arg("decorator", None)
     if decorator is None:
@@ -386,6 +388,13 @@ def get_arg(name, default=None):
     def fmt(s, depth=0):
         """Formatter function for all output specs"""
         string = ""
+
+        if status_fn:
+            # This was copied from spec.tree's colorization logic
+            # then shortened because it seems like status_fn should
+            # always return an InstallStatus
+            string += colorize(status_fn(s).value)
+
         if hashes:
             string += gray_hash(s, hlen) + " "
         string += depth * "    "
@@ -444,7 +453,7 @@ def format_list(specs):
 def filter_loaded_specs(specs):
     """Filter a list of specs returning only those that are
     currently loaded."""
-    hashes = os.environ.get(uenv.spack_loaded_hashes_var, "").split(":")
+    hashes = os.environ.get(uenv.spack_loaded_hashes_var, "").split(os.pathsep)
     return [x for x in specs if x.dag_hash() in hashes]
 
 

lib/spack/spack/cmd/bootstrap.py

@@ -165,7 +165,7 @@ def _reset(args):
         if not ok_to_continue:
             raise RuntimeError("Aborting")
 
-    for scope in spack.config.CONFIG.file_scopes:
+    for scope in spack.config.CONFIG.writable_scopes:
         # The default scope should stay untouched
         if scope.name == "defaults":
             continue

lib/spack/spack/cmd/buildcache.py

@@ -70,12 +70,6 @@ def setup_parser(subparser: argparse.ArgumentParser):
 
     push = subparsers.add_parser("push", aliases=["create"], help=push_fn.__doc__)
     push.add_argument("-f", "--force", action="store_true", help="overwrite tarball if it exists")
-    push.add_argument(
-        "--allow-root",
-        "-a",
-        action="store_true",
-        help="allow install root string in binary files after RPATH substitution",
-    )
     push_sign = push.add_mutually_exclusive_group(required=False)
     push_sign.add_argument(
         "--unsigned",
@@ -190,10 +184,6 @@ def setup_parser(subparser: argparse.ArgumentParser):
     keys.add_argument("-f", "--force", action="store_true", help="force new download of keys")
     keys.set_defaults(func=keys_fn)
 
-    preview = subparsers.add_parser("preview", help=preview_fn.__doc__)
-    arguments.add_common_arguments(preview, ["installed_specs"])
-    preview.set_defaults(func=preview_fn)
-
     # Check if binaries need to be rebuilt on remote mirror
     check = subparsers.add_parser("check", help=check_fn.__doc__)
     check.add_argument(
@@ -404,11 +394,6 @@ def push_fn(args):
     else:
         roots = spack.cmd.require_active_env(cmd_name="buildcache push").concrete_roots()
 
-    if args.allow_root:
-        tty.warn(
-            "The flag `--allow-root` is the default in Spack 0.21, will be removed in Spack 0.22"
-        )
-
     mirror: spack.mirror.Mirror = args.mirror
 
     # Check if this is an OCI image.
@@ -960,14 +945,6 @@ def keys_fn(args):
     bindist.get_keys(args.install, args.trust, args.force)
 
 
-def preview_fn(args):
-    """analyze an installed spec and reports whether executables and libraries are relocatable"""
-    tty.warn(
-        "`spack buildcache preview` is deprecated since `spack buildcache push --allow-root` is "
-        "now the default. This command will be removed in Spack 0.22"
-    )
-
-
 def check_fn(args: argparse.Namespace):
     """check specs against remote binary mirror(s) to see if any need to be rebuilt
 

lib/spack/spack/cmd/ci.py

@@ -6,6 +6,7 @@
 import json
 import os
 import shutil
+import warnings
 from urllib.parse import urlparse, urlunparse
 
 import llnl.util.filesystem as fs
@@ -73,7 +74,7 @@ def setup_parser(subparser):
         "--optimize",
         action="store_true",
         default=False,
-        help="(experimental) optimize the gitlab yaml file for size\n\n"
+        help="(DEPRECATED) optimize the gitlab yaml file for size\n\n"
         "run the generated document through a series of optimization passes "
         "designed to reduce the size of the generated file",
     )
@@ -81,7 +82,7 @@ def setup_parser(subparser):
         "--dependencies",
         action="store_true",
         default=False,
-        help="(experimental) disable DAG scheduling (use 'plain' dependencies)",
+        help="(DEPRECATED) disable DAG scheduling (use 'plain' dependencies)",
     )
     generate.add_argument(
         "--buildcache-destination",
@@ -200,6 +201,18 @@ def ci_generate(args):
     before invoking this command. the value must be the CDash authorization token needed to create
     a build group and register all generated jobs under it
     """
+    if args.optimize:
+        warnings.warn(
+            "The --optimize option has been deprecated, and currently has no effect. "
+            "It will be removed in Spack v0.24."
+        )
+
+    if args.dependencies:
+        warnings.warn(
+            "The --dependencies option has been deprecated, and currently has no effect. "
+            "It will be removed in Spack v0.24."
+        )
+
     env = spack.cmd.require_active_env(cmd_name="ci generate")
 
     if args.copy_to:
@@ -212,8 +225,6 @@ def ci_generate(args):
 
     output_file = args.output_file
     copy_yaml_to = args.copy_to
-    run_optimizer = args.optimize
-    use_dependencies = args.dependencies
     prune_dag = args.prune_dag
     index_only = args.index_only
     artifacts_root = args.artifacts_root
@@ -234,8 +245,6 @@ def ci_generate(args):
         output_file,
         prune_dag=prune_dag,
         check_index_only=index_only,
-        run_optimizer=run_optimizer,
-        use_dependencies=use_dependencies,
         artifacts_root=artifacts_root,
         remote_mirror_override=buildcache_destination,
     )

lib/spack/spack/cmd/commands.py

@@ -11,7 +11,6 @@
 from argparse import ArgumentParser, Namespace
 from typing import IO, Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
 
-import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 from llnl.util.argparsewriter import ArgparseRstWriter, ArgparseWriter, Command
 from llnl.util.tty.colify import colify
@@ -867,9 +866,6 @@ def _commands(parser: ArgumentParser, args: Namespace) -> None:
             prepend_header(args, f)
             formatter(args, f)
 
-        if args.update_completion:
-            fs.set_executable(args.update)
-
     else:
         prepend_header(args, sys.stdout)
         formatter(args, sys.stdout)

lib/spack/spack/cmd/concretize.py

@@ -3,6 +3,9 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import llnl.util.tty as tty
+from llnl.string import plural
+
 import spack.cmd
 import spack.cmd.common.arguments
 import spack.environment as ev
@@ -43,5 +46,9 @@ def concretize(parser, args):
     with env.write_transaction():
         concretized_specs = env.concretize(force=args.force, tests=tests)
         if not args.quiet:
-            ev.display_specs(concretized_specs)
+            if concretized_specs:
+                tty.msg(f"Concretized {plural(len(concretized_specs), 'spec')}:")
+                ev.display_specs([concrete for _, concrete in concretized_specs])
+            else:
+                tty.msg("No new specs to concretize.")
         env.write()

lib/spack/spack/cmd/config.py

@@ -156,7 +156,7 @@ def print_flattened_configuration(*, blame: bool) -> None:
     """
     env = ev.active_environment()
     if env is not None:
-        pristine = env.manifest.pristine_yaml_content
+        pristine = env.manifest.yaml_content
         flattened = pristine.copy()
         flattened[spack.schema.env.TOP_LEVEL_KEY] = pristine[spack.schema.env.TOP_LEVEL_KEY].copy()
     else:
@@ -264,7 +264,9 @@ def config_remove(args):
 def _can_update_config_file(scope: spack.config.ConfigScope, cfg_file):
     if isinstance(scope, spack.config.SingleFileScope):
         return fs.can_access(cfg_file)
-    return fs.can_write_to_dir(scope.path) and fs.can_access(cfg_file)
+    elif isinstance(scope, spack.config.DirectoryConfigScope):
+        return fs.can_write_to_dir(scope.path) and fs.can_access(cfg_file)
+    return False
 
 
 def _config_change_requires_scope(path, spec, scope, match_spec=None):
@@ -362,14 +364,11 @@ def config_change(args):
 def config_update(args):
     # Read the configuration files
     spack.config.CONFIG.get_config(args.section, scope=args.scope)
-    updates: List[spack.config.ConfigScope] = list(
+    updates: List[spack.config.ConfigScope] = [
-        filter(
+        x
-            lambda s: not isinstance(
+        for x in spack.config.CONFIG.format_updates[args.section]
-                s, (spack.config.InternalConfigScope, spack.config.ImmutableConfigScope)
+        if not isinstance(x, spack.config.InternalConfigScope) and x.writable
-            ),
+    ]
-            spack.config.CONFIG.format_updates[args.section],
-        )
-    )
 
     cannot_overwrite, skip_system_scope = [], False
     for scope in updates:
@@ -447,7 +446,7 @@ def _can_revert_update(scope_dir, cfg_file, bkp_file):
 
 
 def config_revert(args):
-    scopes = [args.scope] if args.scope else [x.name for x in spack.config.CONFIG.file_scopes]
+    scopes = [args.scope] if args.scope else [x.name for x in spack.config.CONFIG.writable_scopes]
 
     # Search for backup files in the configuration scopes
     Entry = collections.namedtuple("Entry", ["scope", "cfg", "bkp"])

lib/spack/spack/cmd/create.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
 import os
 import re
 import sys
@@ -934,7 +933,7 @@ def get_repository(args, name):
     # Figure out where the new package should live
     repo_path = args.repo
     if repo_path is not None:
-        repo = spack.repo.Repo(repo_path)
+        repo = spack.repo.from_path(repo_path)
         if spec.namespace and spec.namespace != repo.namespace:
             tty.die(
                 "Can't create package with namespace {0} in repo with "
@@ -942,9 +941,7 @@ def get_repository(args, name):
             )
     else:
         if spec.namespace:
-            repo = spack.repo.PATH.get_repo(spec.namespace, None)
+            repo = spack.repo.PATH.get_repo(spec.namespace)
-            if not repo:
-                tty.die("Unknown namespace: '{0}'".format(spec.namespace))
         else:
             repo = spack.repo.PATH.first_repo()
 

lib/spack/spack/cmd/dependents.py

@@ -47,16 +47,6 @@ def inverted_dependencies():
     dependents of, e.g., `mpi`, but virtuals are not included as
     actual dependents.
     """
-    dag = {}
-    for pkg_cls in spack.repo.PATH.all_package_classes():
-        dag.setdefault(pkg_cls.name, set())
-        for dep in pkg_cls.dependencies_by_name():
-            deps = [dep]
-
-            # expand virtuals if necessary
-            if spack.repo.PATH.is_virtual(dep):
-                deps += [s.name for s in spack.repo.PATH.providers_for(dep)]
-
     dag = collections.defaultdict(set)
     for pkg_cls in spack.repo.PATH.all_package_classes():
         for _, deps_by_name in pkg_cls.dependencies.items():

lib/spack/spack/cmd/develop.py

@@ -9,6 +9,8 @@
 
 import spack.cmd
 import spack.config
+import spack.fetch_strategy
+import spack.repo
 import spack.spec
 import spack.util.path
 import spack.version
@@ -69,13 +71,15 @@ def _retrieve_develop_source(spec, abspath):
     # We construct a package class ourselves, rather than asking for
     # Spec.package, since Spec only allows this when it is concrete
     package = pkg_cls(spec)
-    if isinstance(package.stage[0].fetcher, spack.fetch_strategy.GitFetchStrategy):
+    source_stage = package.stage[0]
-        package.stage[0].fetcher.get_full_repo = True
+    if isinstance(source_stage.fetcher, spack.fetch_strategy.GitFetchStrategy):
+        source_stage.fetcher.get_full_repo = True
         # If we retrieved this version before and cached it, we may have
         # done so without cloning the full git repo; likewise, any
         # mirror might store an instance with truncated history.
-        package.stage[0].disable_mirrors()
+        source_stage.disable_mirrors()
 
+    source_stage.fetcher.set_package(package)
     package.stage.steal_source(abspath)
 
 

lib/spack/spack/cmd/edit.py

@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import errno
 import glob
 import os
 
@@ -11,43 +12,13 @@
 import spack.cmd
 import spack.paths
 import spack.repo
-from spack.spec import Spec
+import spack.util.editor
-from spack.util.editor import editor
 
 description = "open package files in $EDITOR"
 section = "packaging"
 level = "short"
 
 
-def edit_package(name, repo_path, namespace):
-    """Opens the requested package file in your favorite $EDITOR.
-
-    Args:
-        name (str): The name of the package
-        repo_path (str): The path to the repository containing this package
-        namespace (str): A valid namespace registered with Spack
-    """
-    # Find the location of the package
-    if repo_path:
-        repo = spack.repo.Repo(repo_path)
-    elif namespace:
-        repo = spack.repo.PATH.get_repo(namespace)
-    else:
-        repo = spack.repo.PATH
-    path = repo.filename_for_package_name(name)
-
-    spec = Spec(name)
-    if os.path.exists(path):
-        if not os.path.isfile(path):
-            tty.die("Something is wrong. '{0}' is not a file!".format(path))
-        if not os.access(path, os.R_OK):
-            tty.die("Insufficient permissions on '%s'!" % path)
-    else:
-        raise spack.repo.UnknownPackageError(spec.name)
-
-    editor(path)
-
-
 def setup_parser(subparser):
     excl_args = subparser.add_mutually_exclusive_group()
 
@@ -98,41 +69,67 @@ def setup_parser(subparser):
     excl_args.add_argument("-r", "--repo", default=None, help="path to repo to edit package in")
     excl_args.add_argument("-N", "--namespace", default=None, help="namespace of package to edit")
 
-    subparser.add_argument("package", nargs="?", default=None, help="package name")
+    subparser.add_argument("package", nargs="*", default=None, help="package name")
+
+
+def locate_package(name: str, repo: spack.repo.Repo) -> str:
+    path = repo.filename_for_package_name(name)
+
+    try:
+        with open(path, "r"):
+            return path
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            raise spack.repo.UnknownPackageError(name) from e
+        tty.die(f"Cannot edit package: {e}")
+
+
+def locate_file(name: str, path: str) -> str:
+    # convert command names to python module name
+    if path == spack.paths.command_path:
+        name = spack.cmd.python_name(name)
+
+    file_path = os.path.join(path, name)
+
+    # Try to open direct match.
+    try:
+        with open(file_path, "r"):
+            return file_path
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            tty.die(f"Cannot edit file: {e}")
+        pass
+
+    # Otherwise try to find a file that starts with the name
+    candidates = glob.glob(file_path + "*")
+    exclude_list = [".pyc", "~"]  # exclude binaries and backups
+    files = [f for f in candidates if not any(f.endswith(ext) for ext in exclude_list)]
+    if len(files) > 1:
+        tty.die(
+            f"Multiple files start with `{name}`:\n"
+            + "\n".join(f"        {os.path.basename(f)}" for f in files)
+        )
+    elif not files:
+        tty.die(f"No file for '{name}' was found in {path}")
+    return files[0]
 
 
 def edit(parser, args):
-    name = args.package
+    names = args.package
-
-    # By default, edit package files
-    path = spack.paths.packages_path
 
     # If `--command`, `--test`, or `--module` is chosen, edit those instead
     if args.path:
-        path = args.path
+        paths = [locate_file(name, args.path) for name in names] if names else [args.path]
-        if name:
+        spack.util.editor.editor(*paths)
-            # convert command names to python module name
+    elif names:
-            if path == spack.paths.command_path:
+        if args.repo:
-                name = spack.cmd.python_name(name)
+            repo = spack.repo.from_path(args.repo)
-
+        elif args.namespace:
-            path = os.path.join(path, name)
+            repo = spack.repo.PATH.get_repo(args.namespace)
-            if not os.path.exists(path):
+        else:
-                files = glob.glob(path + "*")
+            repo = spack.repo.PATH
-                exclude_list = [".pyc", "~"]  # exclude binaries and backups
+        paths = [locate_package(name, repo) for name in names]
-                files = list(filter(lambda x: all(s not in x for s in exclude_list), files))
+        spack.util.editor.editor(*paths)
-                if len(files) > 1:
-                    m = "Multiple files exist with the name {0}.".format(name)
-                    m += " Please specify a suffix. Files are:\n\n"
-                    for f in files:
-                        m += "        " + os.path.basename(f) + "\n"
-                    tty.die(m)
-                if not files:
-                    tty.die("No file for '{0}' was found in {1}".format(name, path))
-                path = files[0]  # already confirmed only one entry in files
-
-        editor(path)
-    elif name:
-        edit_package(name, args.repo, args.namespace)
     else:
         # By default open the directory where packages live
-        editor(path)
+        spack.util.editor.editor(spack.paths.packages_path)

lib/spack/spack/cmd/external.py

@@ -7,7 +7,7 @@
 import os
 import re
 import sys
-from typing import List, Optional
+from typing import List, Optional, Set
 
 import llnl.util.tty as tty
 import llnl.util.tty.colify as colify
@@ -19,6 +19,7 @@
 import spack.detection
 import spack.error
 import spack.repo
+import spack.spec
 import spack.util.environment
 from spack.cmd.common import arguments
 
@@ -138,14 +139,26 @@ def external_find(args):
         candidate_packages, path_hints=args.path, max_workers=args.jobs
     )
 
-    new_entries = spack.detection.update_configuration(
+    new_specs = spack.detection.update_configuration(
         detected_packages, scope=args.scope, buildable=not args.not_buildable
     )
-    if new_entries:
+
+    # If the user runs `spack external find --not-buildable mpich` we also mark `mpi` non-buildable
+    # to avoid that the concretizer picks a different mpi provider.
+    if new_specs and args.not_buildable:
+        virtuals: Set[str] = {
+            virtual.name
+            for new_spec in new_specs
+            for virtual_specs in spack.repo.PATH.get_pkg_class(new_spec.name).provided.values()
+            for virtual in virtual_specs
+        }
+        new_virtuals = spack.detection.set_virtuals_nonbuildable(virtuals, scope=args.scope)
+        new_specs.extend(spack.spec.Spec(name) for name in new_virtuals)
+
+    if new_specs:
         path = spack.config.CONFIG.get_config_filename(args.scope, "packages")
-        msg = "The following specs have been detected on this system and added to {0}"
+        tty.msg(f"The following specs have been detected on this system and added to {path}")
-        tty.msg(msg.format(path))
+        spack.cmd.display_specs(new_specs)
-        spack.cmd.display_specs(new_entries)
     else:
         tty.msg("No new external packages detected")
 

lib/spack/spack/cmd/find.py

@@ -46,6 +46,10 @@ def setup_parser(subparser):
         help="output specs as machine-readable json records",
     )
 
+    subparser.add_argument(
+        "-I", "--install-status", action="store_true", help="show install status of packages"
+    )
+
     subparser.add_argument(
         "-d", "--deps", action="store_true", help="output dependencies along with found specs"
     )
@@ -293,25 +297,24 @@ def root_decorator(spec, string):
         )
         print()
 
-    if args.show_concretized:
-        tty.msg("Concretized roots")
-        cmd.display_specs(env.specs_by_hash.values(), args, decorator=decorator)
-        print()
-
-    # Display a header for the installed packages section IF there are installed
-    # packages. If there aren't any, we'll just end up printing "0 installed packages"
-    # later.
-    if results and not args.only_roots:
-        tty.msg("Installed packages")
-
 
 def find(parser, args):
-    q_args = query_arguments(args)
-    results = args.specs(**q_args)
-
     env = ev.active_environment()
+
     if not env and args.only_roots:
         tty.die("-r / --only-roots requires an active environment")
+    if not env and args.show_concretized:
+        tty.die("-c / --show-concretized requires an active environment")
+
+    if env:
+        if args.constraint:
+            init_specs = spack.cmd.parse_specs(args.constraint)
+            results = env.all_matching_specs(*init_specs)
+        else:
+            results = env.all_specs()
+    else:
+        q_args = query_arguments(args)
+        results = args.specs(**q_args)
 
     decorator = make_env_decorator(env) if env else lambda s, f: f
 
@@ -332,6 +335,11 @@ def find(parser, args):
     if args.loaded:
         results = spack.cmd.filter_loaded_specs(results)
 
+    if args.install_status or args.show_concretized:
+        status_fn = spack.spec.Spec.install_status
+    else:
+        status_fn = None
+
     # Display the result
     if args.json:
         cmd.display_specs_as_json(results, deps=args.deps)
@@ -340,12 +348,34 @@ def find(parser, args):
             if env:
                 display_env(env, args, decorator, results)
 
-        count_suffix = " (not shown)"
         if not args.only_roots:
-            cmd.display_specs(results, args, decorator=decorator, all_headers=True)
+            display_results = results
-            count_suffix = ""
+            if not args.show_concretized:
+                display_results = list(x for x in results if x.installed)
+            cmd.display_specs(
+                display_results, args, decorator=decorator, all_headers=True, status_fn=status_fn
+            )
 
         # print number of installed packages last (as the list may be long)
         if sys.stdout.isatty() and args.groups:
+            installed_suffix = ""
+            concretized_suffix = " to be installed"
+
+            if args.only_roots:
+                installed_suffix += " (not shown)"
+                concretized_suffix += " (not shown)"
+            else:
+                if env and not args.show_concretized:
+                    concretized_suffix += " (show with `spack find -c`)"
+
             pkg_type = "loaded" if args.loaded else "installed"
-            spack.cmd.print_how_many_pkgs(results, pkg_type, suffix=count_suffix)
+            spack.cmd.print_how_many_pkgs(
+                list(x for x in results if x.installed), pkg_type, suffix=installed_suffix
+            )
+
+            if env:
+                spack.cmd.print_how_many_pkgs(
+                    list(x for x in results if not x.installed),
+                    "concretized",
+                    suffix=concretized_suffix,
+                )

lib/spack/spack/cmd/gc.py

@@ -56,7 +56,6 @@ def roots_from_environments(args, active_env):
 
     # -e says "also preserve things needed by this particular env"
     for env_name_or_dir in args.except_environment:
-        print("HMM", env_name_or_dir)
         if ev.exists(env_name_or_dir):
             env = ev.read(env_name_or_dir)
         elif ev.is_env_dir(env_name_or_dir):

lib/spack/spack/cmd/install.py

@@ -10,6 +10,7 @@
 from typing import List
 
 import llnl.util.filesystem as fs
+from llnl.string import plural
 from llnl.util import lang, tty
 
 import spack.build_environment
@@ -375,7 +376,9 @@ def _maybe_add_and_concretize(args, env, specs):
         # `spack concretize`
         tests = compute_tests_install_kwargs(env.user_specs, args.test)
         concretized_specs = env.concretize(tests=tests)
-        ev.display_specs(concretized_specs)
+        if concretized_specs:
+            tty.msg(f"Concretized {plural(len(concretized_specs), 'spec')}")
+            ev.display_specs([concrete for _, concrete in concretized_specs])
 
         # save view regeneration for later, so that we only do it
         # once, as it can be slow.

lib/spack/spack/cmd/pkg.py

@@ -169,7 +169,9 @@ def pkg_hash(args):
 
 def get_grep(required=False):
     """Get a grep command to use with ``spack pkg grep``."""
-    return exe.which(os.environ.get("SPACK_GREP") or "grep", required=required)
+    grep = exe.which(os.environ.get("SPACK_GREP") or "grep", required=required)
+    grep.ignore_quotes = True  # allow `spack pkg grep '"quoted string"'` without warning
+    return grep
 
 
 def pkg_grep(args, unknown_args):

lib/spack/spack/cmd/repo.py

@@ -91,7 +91,7 @@ def repo_add(args):
         tty.die("Not a Spack repository: %s" % path)
 
     # Make sure it's actually a spack repository by constructing it.
-    repo = spack.repo.Repo(canon_path)
+    repo = spack.repo.from_path(canon_path)
 
     # If that succeeds, finally add it to the configuration.
     repos = spack.config.get("repos", scope=args.scope)
@@ -124,7 +124,7 @@ def repo_remove(args):
     # If it is a namespace, remove corresponding repo
     for path in repos:
         try:
-            repo = spack.repo.Repo(path)
+            repo = spack.repo.from_path(path)
             if repo.namespace == namespace_or_path:
                 repos.remove(path)
                 spack.config.set("repos", repos, args.scope)
@@ -142,7 +142,7 @@ def repo_list(args):
     repos = []
     for r in roots:
         try:
-            repos.append(spack.repo.Repo(r))
+            repos.append(spack.repo.from_path(r))
         except spack.repo.RepoError:
             continue
 

lib/spack/spack/cmd/solve.py

@@ -114,15 +114,16 @@ def _process_result(result, show, required_format, kwargs):
 
     # dump the solutions as concretized specs
     if "solutions" in show:
-        for spec in result.specs:
+        if required_format:
-            # With -y, just print YAML to output.
+            for spec in result.specs:
-            if required_format == "yaml":
+                # With -y, just print YAML to output.
-                # use write because to_yaml already has a newline.
+                if required_format == "yaml":
-                sys.stdout.write(spec.to_yaml(hash=ht.dag_hash))
+                    # use write because to_yaml already has a newline.
-            elif required_format == "json":
+                    sys.stdout.write(spec.to_yaml(hash=ht.dag_hash))
-                sys.stdout.write(spec.to_json(hash=ht.dag_hash))
+                elif required_format == "json":
-            else:
+                    sys.stdout.write(spec.to_json(hash=ht.dag_hash))
-                sys.stdout.write(spec.tree(color=sys.stdout.isatty(), **kwargs))
+        else:
+            sys.stdout.write(spack.spec.tree(result.specs, color=sys.stdout.isatty(), **kwargs))
         print()
 
     if result.unsolved_specs and "solutions" in show:

lib/spack/spack/cmd/spec.py

@@ -105,11 +105,19 @@ def spec(parser, args):
         if env:
             env.concretize()
             specs = env.concretized_specs()
+
+            # environments are printed together in a combined tree() invocation,
+            # except when using --yaml or --json, which we print spec by spec below.
+            if not args.format:
+                tree_kwargs["key"] = spack.traverse.by_dag_hash
+                tree_kwargs["hashes"] = args.long or args.very_long
+                print(spack.spec.tree([concrete for _, concrete in specs], **tree_kwargs))
+                return
         else:
             tty.die("spack spec requires at least one spec or an active environment")
 
     for input, output in specs:
-        # With -y, just print YAML to output.
+        # With --yaml or --json, just print the raw specs to output
         if args.format:
             if args.format == "yaml":
                 # use write because to_yaml already has a newline.

lib/spack/spack/cmd/unload.py

@@ -71,7 +71,7 @@ def unload(parser, args):
             "Cannot specify specs on command line when unloading all specs with '--all'"
         )
 
-    hashes = os.environ.get(uenv.spack_loaded_hashes_var, "").split(":")
+    hashes = os.environ.get(uenv.spack_loaded_hashes_var, "").split(os.pathsep)
     if args.specs:
         specs = [
             spack.cmd.disambiguate_spec_from_hashes(spec, hashes)

lib/spack/spack/cmd/url.py

@@ -339,7 +339,7 @@ def add(self, pkg_name, fetcher):
     for pkg_cls in spack.repo.PATH.all_package_classes():
         npkgs += 1
 
-        for v in pkg_cls.versions:
+        for v in list(pkg_cls.versions):
             try:
                 pkg = pkg_cls(spack.spec.Spec(pkg_cls.name))
                 fetcher = fs.for_package_version(pkg, v)

lib/spack/spack/cmd/view.py

@@ -38,10 +38,10 @@
 
 import spack.cmd
 import spack.environment as ev
+import spack.filesystem_view as fsv
 import spack.schema.projections
 import spack.store
 from spack.config import validate
-from spack.filesystem_view import YamlFilesystemView, view_func_parser
 from spack.util import spack_yaml as s_yaml
 
 description = "project packages to a compact naming scheme on the filesystem"
@@ -193,17 +193,13 @@ def view(parser, args):
         ordered_projections = {}
 
     # What method are we using for this view
-    if args.action in actions_link:
+    link_type = args.action if args.action in actions_link else "symlink"
-        link_fn = view_func_parser(args.action)
+    view = fsv.YamlFilesystemView(
-    else:
-        link_fn = view_func_parser("symlink")
-
-    view = YamlFilesystemView(
         path,
         spack.store.STORE.layout,
         projections=ordered_projections,
         ignore_conflicts=getattr(args, "ignore_conflicts", False),
-        link=link_fn,
+        link_type=link_type,
         verbose=args.verbose,
     )
 

lib/spack/spack/compiler.py

@@ -18,7 +18,6 @@
 import llnl.util.tty as tty
 from llnl.util.filesystem import path_contains_subdirectory, paths_containing_libs
 
-import spack.compilers
 import spack.error
 import spack.schema.environment
 import spack.spec
@@ -279,11 +278,6 @@ def debug_flags(self):
     def opt_flags(self):
         return ["-O", "-O0", "-O1", "-O2", "-O3"]
 
-    # Cray PrgEnv name that can be used to load this compiler
-    PrgEnv: Optional[str] = None
-    # Name of module used to switch versions of this compiler
-    PrgEnv_compiler: Optional[str] = None
-
     def __init__(
         self,
         cspec,

lib/spack/spack/compilers/__init__.py

@@ -260,7 +260,7 @@ def _init_compiler_config(
 def compiler_config_files():
     config_files = list()
     config = spack.config.CONFIG
-    for scope in config.file_scopes:
+    for scope in config.writable_scopes:
         name = scope.name
         compiler_config = config.get("compilers", scope=name)
         if compiler_config:
@@ -488,7 +488,7 @@ def supported_compilers_for_host_platform() -> List[str]:
     return supported_compilers_for_platform(host_plat)
 
 
-def supported_compilers_for_platform(platform: spack.platforms.Platform) -> List[str]:
+def supported_compilers_for_platform(platform: "spack.platforms.Platform") -> List[str]:
     """Return a set of compiler class objects supported by Spack
     that are also supported by the provided platform
 

lib/spack/spack/compilers/aocc.py

@@ -25,9 +25,6 @@ class Aocc(Compiler):
     # Subclasses use possible names of Fortran 90 compiler
     fc_names = ["flang"]
 
-    PrgEnv = "PrgEnv-aocc"
-    PrgEnv_compiler = "aocc"
-
     version_argument = "--version"
 
     @property

lib/spack/spack/compilers/cce.py

@@ -34,12 +34,9 @@ def __init__(self, *args, **kwargs):
     # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
     suffixes = [r"-mp-\d\.\d"]
 
-    PrgEnv = "PrgEnv-cray"
-    PrgEnv_compiler = "cce"
-
     @property
     def link_paths(self):
-        if any(self.PrgEnv in m for m in self.modules):
+        if any("PrgEnv-cray" in m for m in self.modules):
             # Old module-based interface to cray compilers
             return {
                 "cc": os.path.join("cce", "cc"),

lib/spack/spack/compilers/gcc.py

@@ -40,9 +40,6 @@ class Gcc(spack.compiler.Compiler):
         "fc": os.path.join("gcc", "gfortran"),
     }
 
-    PrgEnv = "PrgEnv-gnu"
-    PrgEnv_compiler = "gcc"
-
     @property
     def verbose_flag(self):
         return "-v"

lib/spack/spack/compilers/intel.py

@@ -31,9 +31,6 @@ class Intel(Compiler):
         "fc": os.path.join("intel", "ifort"),
     }
 
-    PrgEnv = "PrgEnv-intel"
-    PrgEnv_compiler = "intel"
-
     if sys.platform == "win32":
         version_argument = "/QV"
     else:
@@ -126,3 +123,14 @@ def fc_pic_flag(self):
     @property
     def stdcxx_libs(self):
         return ("-cxxlib",)
+
+    def setup_custom_environment(self, pkg, env):
+        # Edge cases for Intel's oneAPI compilers when using the legacy classic compilers:
+        # Always pass flags to disable deprecation warnings, since these warnings can
+        # confuse tools that parse the output of compiler commands (e.g. version checks).
+        if self.cc and self.cc.endswith("icc") and self.real_version >= Version("2021"):
+            env.append_flags("SPACK_ALWAYS_CFLAGS", "-diag-disable=10441")
+        if self.cxx and self.cxx.endswith("icpc") and self.real_version >= Version("2021"):
+            env.append_flags("SPACK_ALWAYS_CXXFLAGS", "-diag-disable=10441")
+        if self.fc and self.fc.endswith("ifort") and self.real_version >= Version("2021"):
+            env.append_flags("SPACK_ALWAYS_FFLAGS", "-diag-disable=10448")

lib/spack/spack/compilers/msvc.py

@@ -231,24 +231,55 @@ def msvc_version(self):
 
     @property
     def short_msvc_version(self):
+        """This is the shorthand VCToolset version of form
+        MSVC<short-ver>
         """
-        This is the shorthand VCToolset version of form
+        return "MSVC" + self.vc_toolset_ver
-        MSVC<short-ver> *NOT* the full version, for that see
+
+    @property
+    def vc_toolset_ver(self):
+        """
+        The toolset version is the version of the combined set of cl and link
+        This typically relates directly to VS version i.e. VS 2022 is v143
+        VS 19 is v142, etc.
+        This value is defined by the first three digits of the major + minor
+        version of the VS toolset (143 for 14.3x.bbbbb). Traditionally the
+        minor version has remained a static two digit number for a VS release
+        series, however, as of VS22, this is no longer true, both
+        14.4x.bbbbb and 14.3x.bbbbb are considered valid VS22 VC toolset
+        versions due to a change in toolset minor version sentiment.
+
+        This is *NOT* the full version, for that see
         Msvc.msvc_version or MSVC.platform_toolset_ver for the
         raw platform toolset version
+
         """
-        ver = self.platform_toolset_ver
+        ver = self.msvc_version[:2].joined.string[:3]
-        return "MSVC" + ver
+        return ver
 
     @property
     def platform_toolset_ver(self):
         """
         This is the platform toolset version of current MSVC compiler
-        i.e. 142.
+        i.e. 142. The platform toolset is the targeted MSVC library/compiler
+        versions by compilation (this is different from the VC Toolset)
+
+
         This is different from the VC toolset version as established
-        by `short_msvc_version`
+        by `short_msvc_version`, but typically are represented by the same
+        three digit value
         """
-        return self.msvc_version[:2].joined.string[:3]
+        # Typically VS toolset version and platform toolset versions match
+        # VS22 introduces the first divergence of VS toolset version
+        # (144 for "recent" releases) and platform toolset version (143)
+        # so it needs additional handling until MS releases v144
+        # (assuming v144 is also for VS22)
+        # or adds better support for detection
+        # TODO: (johnwparent) Update this logic for the next platform toolset
+        # or VC toolset version update
+        toolset_ver = self.vc_toolset_ver
+        vs22_toolset = Version(toolset_ver) > Version("142")
+        return toolset_ver if not vs22_toolset else "143"
 
     def _compiler_version(self, compiler):
         """Returns version object for given compiler"""

lib/spack/spack/compilers/nvhpc.py

@@ -29,9 +29,6 @@ class Nvhpc(Compiler):
         "fc": os.path.join("nvhpc", "nvfortran"),
     }
 
-    PrgEnv = "PrgEnv-nvhpc"
-    PrgEnv_compiler = "nvhpc"
-
     version_argument = "--version"
     version_regex = r"nv[^ ]* (?:[^ ]+ Dev-r)?([0-9.]+)(?:-[0-9]+)?"
 

lib/spack/spack/compilers/oneapi.py

@@ -4,11 +4,12 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import os
-from os.path import dirname
+from os.path import dirname, join
 
 from llnl.util import tty
 
 from spack.compiler import Compiler
+from spack.version import Version
 
 
 class Oneapi(Compiler):
@@ -32,9 +33,6 @@ class Oneapi(Compiler):
         "fc": os.path.join("oneapi", "ifx"),
     }
 
-    PrgEnv = "PrgEnv-oneapi"
-    PrgEnv_compiler = "oneapi"
-
     version_argument = "--version"
     version_regex = r"(?:(?:oneAPI DPC\+\+(?:\/C\+\+)? Compiler)|(?:\(IFORT\))|(?:\(IFX\))) (\S+)"
 
@@ -135,8 +133,22 @@ def setup_custom_environment(self, pkg, env):
         # It is located in the same directory as the driver. Error message:
         #   clang++: error: unable to execute command:
         #   Executable "sycl-post-link" doesn't exist!
-        if self.cxx:
+        # also ensures that shared objects and libraries required by the compiler,
+        # e.g. libonnx, can be found succesfully
+        # due to a fix, this is no longer required for OneAPI versions >= 2024.2
+        if self.cxx and pkg.spec.satisfies("%oneapi@:2024.1"):
             env.prepend_path("PATH", dirname(self.cxx))
+            env.prepend_path("LD_LIBRARY_PATH", join(dirname(dirname(self.cxx)), "lib"))
+
+        # Edge cases for Intel's oneAPI compilers when using the legacy classic compilers:
+        # Always pass flags to disable deprecation warnings, since these warnings can
+        # confuse tools that parse the output of compiler commands (e.g. version checks).
+        if self.cc and self.cc.endswith("icc") and self.real_version >= Version("2021"):
+            env.append_flags("SPACK_ALWAYS_CFLAGS", "-diag-disable=10441")
+        if self.cxx and self.cxx.endswith("icpc") and self.real_version >= Version("2021"):
+            env.append_flags("SPACK_ALWAYS_CXXFLAGS", "-diag-disable=10441")
+        if self.fc and self.fc.endswith("ifort") and self.real_version >= Version("2021"):
+            env.append_flags("SPACK_ALWAYS_FFLAGS", "-diag-disable=10448")
 
         # 2024 release bumped the libsycl version because of an ABI
         # change, 2024 compilers are required.  You will see this

lib/spack/spack/compilers/pgi.py

@@ -30,9 +30,6 @@ class Pgi(Compiler):
         "fc": os.path.join("pgi", "pgfortran"),
     }
 
-    PrgEnv = "PrgEnv-pgi"
-    PrgEnv_compiler = "pgi"
-
     version_argument = "-V"
     ignore_version_errors = [2]  # `pgcc -V` on PowerPC annoyingly returns 2
     version_regex = r"pg[^ ]* ([0-9.]+)-[0-9]+ (LLVM )?[^ ]+ target on "

lib/spack/spack/compilers/rocmcc.py

@@ -23,9 +23,6 @@ class Rocmcc(spack.compilers.clang.Clang):
     # Subclasses use possible names of Fortran 90 compiler
     fc_names = ["amdflang"]
 
-    PrgEnv = "PrgEnv-amd"
-    PrgEnv_compiler = "amd"
-
     @property
     def link_paths(self):
         link_paths = {

lib/spack/spack/config.py

@@ -35,11 +35,10 @@
 import os
 import re
 import sys
-from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Type, Union
+from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
 
 from llnl.util import filesystem, lang, tty
 
-import spack.compilers
 import spack.paths
 import spack.platforms
 import spack.schema
@@ -117,21 +116,39 @@
 
 
 class ConfigScope:
-    """This class represents a configuration scope.
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self.writable = False
+        self.sections = syaml.syaml_dict()
 
-    A scope is one directory containing named configuration files.
+    def get_section_filename(self, section: str) -> str:
-    Each file is a config "section" (e.g., mirrors, compilers, etc.).
+        raise NotImplementedError
-    """
 
-    def __init__(self, name, path) -> None:
+    def get_section(self, section: str) -> Optional[YamlConfigDict]:
-        self.name = name  # scope name.
+        raise NotImplementedError
-        self.path = path  # path to directory containing configs.
+
-        self.sections = syaml.syaml_dict()  # sections read from config files.
+    def _write_section(self, section: str) -> None:
+        raise NotImplementedError
 
     @property
     def is_platform_dependent(self) -> bool:
-        """Returns true if the scope name is platform specific"""
+        return False
-        return os.sep in self.name
+
+    def clear(self) -> None:
+        """Empty cached config information."""
+        self.sections = syaml.syaml_dict()
+
+    def __repr__(self) -> str:
+        return f"<ConfigScope: {self.name}>"
+
+
+class DirectoryConfigScope(ConfigScope):
+    """Config scope backed by a directory containing one file per section."""
+
+    def __init__(self, name: str, path: str, *, writable: bool = True) -> None:
+        super().__init__(name)
+        self.path = path
+        self.writable = writable
 
     def get_section_filename(self, section: str) -> str:
         """Returns the filename associated with a given section"""
@@ -148,14 +165,15 @@ def get_section(self, section: str) -> Optional[YamlConfigDict]:
         return self.sections[section]
 
     def _write_section(self, section: str) -> None:
+        if not self.writable:
+            raise ConfigError(f"Cannot write to immutable scope {self}")
+
         filename = self.get_section_filename(section)
         data = self.get_section(section)
         if data is None:
             return
 
-        # We copy data here to avoid adding defaults at write time
+        validate(data, SECTION_SCHEMAS[section])
-        validate_data = copy.deepcopy(data)
-        validate(validate_data, SECTION_SCHEMAS[section])
 
         try:
             filesystem.mkdirp(self.path)
@@ -164,19 +182,23 @@ def _write_section(self, section: str) -> None:
         except (syaml.SpackYAMLError, OSError) as e:
             raise ConfigFileError(f"cannot write to '{filename}'") from e
 
-    def clear(self) -> None:
+    @property
-        """Empty cached config information."""
+    def is_platform_dependent(self) -> bool:
-        self.sections = syaml.syaml_dict()
+        """Returns true if the scope name is platform specific"""
-
+        return "/" in self.name
-    def __repr__(self) -> str:
-        return f"<ConfigScope: {self.name}: {self.path}>"
 
 
 class SingleFileScope(ConfigScope):
     """This class represents a configuration scope in a single YAML file."""
 
     def __init__(
-        self, name: str, path: str, schema: YamlConfigDict, yaml_path: Optional[List[str]] = None
+        self,
+        name: str,
+        path: str,
+        schema: YamlConfigDict,
+        *,
+        yaml_path: Optional[List[str]] = None,
+        writable: bool = True,
     ) -> None:
         """Similar to ``ConfigScope`` but can be embedded in another schema.
 
@@ -195,15 +217,13 @@ def __init__(
                        config:
                          install_tree: $spack/opt/spack
         """
-        super().__init__(name, path)
+        super().__init__(name)
         self._raw_data: Optional[YamlConfigDict] = None
         self.schema = schema
+        self.path = path
+        self.writable = writable
         self.yaml_path = yaml_path or []
 
-    @property
-    def is_platform_dependent(self) -> bool:
-        return False
-
     def get_section_filename(self, section) -> str:
         return self.path
 
@@ -257,6 +277,8 @@ def get_section(self, section: str) -> Optional[YamlConfigDict]:
         return self.sections.get(section, None)
 
     def _write_section(self, section: str) -> None:
+        if not self.writable:
+            raise ConfigError(f"Cannot write to immutable scope {self}")
         data_to_write: Optional[YamlConfigDict] = self._raw_data
 
         # If there is no existing data, this section SingleFileScope has never
@@ -301,19 +323,6 @@ def __repr__(self) -> str:
         return f"<SingleFileScope: {self.name}: {self.path}>"
 
 
-class ImmutableConfigScope(ConfigScope):
-    """A configuration scope that cannot be written to.
-
-    This is used for ConfigScopes passed on the command line.
-    """
-
-    def _write_section(self, section) -> None:
-        raise ConfigError(f"Cannot write to immutable scope {self}")
-
-    def __repr__(self) -> str:
-        return f"<ImmutableConfigScope: {self.name}: {self.path}>"
-
-
 class InternalConfigScope(ConfigScope):
     """An internal configuration scope that is not persisted to a file.
 
@@ -323,7 +332,7 @@ class InternalConfigScope(ConfigScope):
     """
 
     def __init__(self, name: str, data: Optional[YamlConfigDict] = None) -> None:
-        super().__init__(name, None)
+        super().__init__(name)
         self.sections = syaml.syaml_dict()
 
         if data is not None:
@@ -333,9 +342,6 @@ def __init__(self, name: str, data: Optional[YamlConfigDict] = None) -> None:
                 validate({section: dsec}, SECTION_SCHEMAS[section])
                 self.sections[section] = _mark_internal(syaml.syaml_dict({section: dsec}), name)
 
-    def get_section_filename(self, section: str) -> str:
-        raise NotImplementedError("Cannot get filename for InternalConfigScope.")
-
     def get_section(self, section: str) -> Optional[YamlConfigDict]:
         """Just reads from an internal dictionary."""
         if section not in self.sections:
@@ -440,27 +446,21 @@ def remove_scope(self, scope_name: str) -> Optional[ConfigScope]:
         return scope
 
     @property
-    def file_scopes(self) -> List[ConfigScope]:
+    def writable_scopes(self) -> Generator[ConfigScope, None, None]:
-        """List of writable scopes with an associated file."""
+        """Generator of writable scopes with an associated file."""
-        return [
+        return (s for s in self.scopes.values() if s.writable)
-            s
-            for s in self.scopes.values()
-            if (type(s) is ConfigScope or type(s) is SingleFileScope)
-        ]
 
     def highest_precedence_scope(self) -> ConfigScope:
-        """Non-internal scope with highest precedence."""
+        """Writable scope with highest precedence."""
-        return next(reversed(self.file_scopes))
+        return next(s for s in reversed(self.scopes.values()) if s.writable)  # type: ignore
 
     def highest_precedence_non_platform_scope(self) -> ConfigScope:
-        """Non-internal non-platform scope with highest precedence
+        """Writable non-platform scope with highest precedence"""
-
+        return next(
-        Platform-specific scopes are of the form scope/platform"""
+            s
-        generator = reversed(self.file_scopes)
+            for s in reversed(self.scopes.values())  # type: ignore
-        highest = next(generator)
+            if s.writable and not s.is_platform_dependent
-        while highest and highest.is_platform_dependent:
+        )
-            highest = next(generator)
-        return highest
 
     def matching_scopes(self, reg_expr) -> List[ConfigScope]:
         """
@@ -755,13 +755,14 @@ def override(
 
 
 def _add_platform_scope(
-    cfg: Union[Configuration, lang.Singleton], scope_type: Type[ConfigScope], name: str, path: str
+    cfg: Union[Configuration, lang.Singleton], name: str, path: str, writable: bool = True
 ) -> None:
     """Add a platform-specific subdirectory for the current platform."""
     platform = spack.platforms.host().name
-    plat_name = os.path.join(name, platform)
+    scope = DirectoryConfigScope(
-    plat_path = os.path.join(path, platform)
+        f"{name}/{platform}", os.path.join(path, platform), writable=writable
-    cfg.push_scope(scope_type(plat_name, plat_path))
+    )
+    cfg.push_scope(scope)
 
 
 def config_paths_from_entry_points() -> List[Tuple[str, str]]:
@@ -792,22 +793,27 @@ def config_paths_from_entry_points() -> List[Tuple[str, str]]:
 def _add_command_line_scopes(
     cfg: Union[Configuration, lang.Singleton], command_line_scopes: List[str]
 ) -> None:
-    """Add additional scopes from the --config-scope argument.
+    """Add additional scopes from the --config-scope argument, either envs or dirs."""
+    import spack.environment.environment as env  # circular import
 
-    Command line scopes are named after their position in the arg list.
-    """
     for i, path in enumerate(command_line_scopes):
-        # We ensure that these scopes exist and are readable, as they are
+        name = f"cmd_scope_{i}"
-        # provided on the command line by the user.
-        if not os.path.isdir(path):
-            raise ConfigError(f"config scope is not a directory: '{path}'")
-        elif not os.access(path, os.R_OK):
-            raise ConfigError(f"config scope is not readable: '{path}'")
 
-        # name based on order on the command line
+        if env.exists(path):  # managed environment
-        name = f"cmd_scope_{i:d}"
+            manifest = env.EnvironmentManifestFile(env.root(path))
-        cfg.push_scope(ImmutableConfigScope(name, path))
+        elif env.is_env_dir(path):  # anonymous environment
-        _add_platform_scope(cfg, ImmutableConfigScope, name, path)
+            manifest = env.EnvironmentManifestFile(path)
+        elif os.path.isdir(path):  # directory with config files
+            cfg.push_scope(DirectoryConfigScope(name, path, writable=False))
+            _add_platform_scope(cfg, name, path, writable=False)
+            continue
+        else:
+            raise ConfigError(f"Invalid configuration scope: {path}")
+
+        for scope in manifest.env_config_scopes:
+            scope.name = f"{name}:{scope.name}"
+            scope.writable = False
+            cfg.push_scope(scope)
 
 
 def create() -> Configuration:
@@ -851,10 +857,10 @@ def create() -> Configuration:
 
     # add each scope and its platform-specific directory
     for name, path in configuration_paths:
-        cfg.push_scope(ConfigScope(name, path))
+        cfg.push_scope(DirectoryConfigScope(name, path))
 
         # Each scope can have per-platfom overrides in subdirectories
-        _add_platform_scope(cfg, ConfigScope, name, path)
+        _add_platform_scope(cfg, name, path)
 
     # add command-line scopes
     _add_command_line_scopes(cfg, COMMAND_LINE_SCOPES)
@@ -969,7 +975,7 @@ def set(path: str, value: Any, scope: Optional[str] = None) -> None:
 def add_default_platform_scope(platform: str) -> None:
     plat_name = os.path.join("defaults", platform)
     plat_path = os.path.join(CONFIGURATION_DEFAULTS_PATH[1], platform)
-    CONFIG.push_scope(ConfigScope(plat_name, plat_path))
+    CONFIG.push_scope(DirectoryConfigScope(plat_name, plat_path))
 
 
 def scopes() -> Dict[str, ConfigScope]:
@@ -978,19 +984,10 @@ def scopes() -> Dict[str, ConfigScope]:
 
 
 def writable_scopes() -> List[ConfigScope]:
-    """
+    """Return list of writable scopes. Higher-priority scopes come first in the list."""
-    Return list of writable scopes. Higher-priority scopes come first in the
+    scopes = [x for x in CONFIG.scopes.values() if x.writable]
-    list.
+    scopes.reverse()
-    """
+    return scopes
-    return list(
-        reversed(
-            list(
-                x
-                for x in CONFIG.scopes.values()
-                if not isinstance(x, (InternalConfigScope, ImmutableConfigScope))
-            )
-        )
-    )
 
 
 def writable_scope_names() -> List[str]:
@@ -1080,11 +1077,8 @@ def validate(
     """
     import jsonschema
 
-    # Validate a copy to avoid adding defaults
-    # This allows us to round-trip data without adding to it.
-    test_data = syaml.deepcopy(data)
     try:
-        spack.schema.Validator(schema).validate(test_data)
+        spack.schema.Validator(schema).validate(data)
     except jsonschema.ValidationError as e:
         if hasattr(e.instance, "lc"):
             line_number = e.instance.lc.line + 1
@@ -1093,7 +1087,7 @@ def validate(
         raise ConfigFormatError(e, data, filename, line_number) from e
     # return the validated data so that we can access the raw data
     # mostly relevant for environments
-    return test_data
+    return data
 
 
 def read_config_file(
@@ -1599,7 +1593,7 @@ def _config_from(scopes_or_paths: List[Union[ConfigScope, str]]) -> Configuratio
         path = os.path.normpath(scope_or_path)
         assert os.path.isdir(path), f'"{path}" must be a directory'
         name = os.path.basename(path)
-        scopes.append(ConfigScope(name, path))
+        scopes.append(DirectoryConfigScope(name, path))
 
     configuration = Configuration(*scopes)
     return configuration

lib/spack/spack/container/images.json

@@ -78,24 +78,17 @@
         "image": "quay.io/almalinuxorg/almalinux:8"
       }
     },
-    "centos:stream": {
+    "centos:stream9": {
       "bootstrap": {
-        "template": "container/centos_stream.dockerfile",
+        "template": "container/centos_stream9.dockerfile",
-        "image": "quay.io/centos/centos:stream"
+        "image": "quay.io/centos/centos:stream9"
       },
       "os_package_manager": "dnf_epel",
-      "build": "spack/centos-stream",
+      "build": "spack/centos-stream9",
       "final": {
-        "image": "quay.io/centos/centos:stream"
+        "image": "quay.io/centos/centos:stream9"
       }
     },
-    "centos:7": {
-      "bootstrap": {
-        "template": "container/centos_7.dockerfile"
-      },
-      "os_package_manager": "yum",
-      "build": "spack/centos7"
-    },
     "opensuse/leap:15": {
       "bootstrap": {
         "template": "container/leap-15.dockerfile"

lib/spack/spack/detection/__init__.py

@@ -2,7 +2,12 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-from .common import DetectedPackage, executable_prefix, update_configuration
+from .common import (
+    DetectedPackage,
+    executable_prefix,
+    set_virtuals_nonbuildable,
+    update_configuration,
+)
 from .path import by_path, executables_in_path
 from .test import detection_tests
 
@@ -12,5 +17,6 @@
     "executables_in_path",
     "executable_prefix",
     "update_configuration",
+    "set_virtuals_nonbuildable",
     "detection_tests",
 ]

lib/spack/spack/detection/common.py

@@ -252,6 +252,27 @@ def update_configuration(
     return all_new_specs
 
 
+def set_virtuals_nonbuildable(virtuals: Set[str], scope: Optional[str] = None) -> List[str]:
+    """Update packages:virtual:buildable:False for the provided virtual packages, if the property
+    is not set by the user. Returns the list of virtual packages that have been updated."""
+    packages = spack.config.get("packages")
+    new_config = {}
+    for virtual in virtuals:
+        # If the user has set the buildable prop do not override it
+        if virtual in packages and "buildable" in packages[virtual]:
+            continue
+        new_config[virtual] = {"buildable": False}
+
+    # Update the provided scope
+    spack.config.set(
+        "packages",
+        spack.config.merge_yaml(spack.config.get("packages", scope=scope), new_config),
+        scope=scope,
+    )
+
+    return list(new_config.keys())
+
+
 def _windows_drive() -> str:
     """Return Windows drive string extracted from the PROGRAMFILES environment variable,
     which is guaranteed to be defined for all logins.

lib/spack/spack/detection/path.py

@@ -12,7 +12,7 @@
 import re
 import sys
 import warnings
-from typing import Dict, List, Optional, Set, Tuple
+from typing import Dict, List, Optional, Set, Tuple, Type
 
 import llnl.util.filesystem
 import llnl.util.lang
@@ -200,7 +200,7 @@ class Finder:
     def default_path_hints(self) -> List[str]:
         return []
 
-    def search_patterns(self, *, pkg: "spack.package_base.PackageBase") -> List[str]:
+    def search_patterns(self, *, pkg: Type["spack.package_base.PackageBase"]) -> List[str]:
         """Returns the list of patterns used to match candidate files.
 
         Args:
@@ -226,7 +226,7 @@ def prefix_from_path(self, *, path: str) -> str:
         raise NotImplementedError("must be implemented by derived classes")
 
     def detect_specs(
-        self, *, pkg: "spack.package_base.PackageBase", paths: List[str]
+        self, *, pkg: Type["spack.package_base.PackageBase"], paths: List[str]
     ) -> List[DetectedPackage]:
         """Given a list of files matching the search patterns, returns a list of detected specs.
 
@@ -327,7 +327,7 @@ class ExecutablesFinder(Finder):
     def default_path_hints(self) -> List[str]:
         return spack.util.environment.get_path("PATH")
 
-    def search_patterns(self, *, pkg: "spack.package_base.PackageBase") -> List[str]:
+    def search_patterns(self, *, pkg: Type["spack.package_base.PackageBase"]) -> List[str]:
         result = []
         if hasattr(pkg, "executables") and hasattr(pkg, "platform_executables"):
             result = pkg.platform_executables()
@@ -356,7 +356,7 @@ class LibrariesFinder(Finder):
     DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH, and standard system library paths
     """
 
-    def search_patterns(self, *, pkg: "spack.package_base.PackageBase") -> List[str]:
+    def search_patterns(self, *, pkg: Type["spack.package_base.PackageBase"]) -> List[str]:
         result = []
         if hasattr(pkg, "libraries"):
             result = pkg.libraries

lib/spack/spack/directives.py

@@ -81,7 +81,17 @@ class OpenMpi(Package):
 ]
 
 #: These are variant names used by Spack internally; packages can't use them
-reserved_names = ["patches", "dev_path"]
+reserved_names = [
+    "arch",
+    "architecture",
+    "dev_path",
+    "namespace",
+    "operating_system",
+    "os",
+    "patches",
+    "platform",
+    "target",
+]
 
 #: Names of possible directives. This list is mostly populated using the @directive decorator.
 #: Some directives leverage others and in that case are not automatically added.
@@ -90,14 +100,14 @@ class OpenMpi(Package):
 _patch_order_index = 0
 
 
-SpecType = Union["spack.spec.Spec", str]
+SpecType = str
 DepType = Union[Tuple[str, ...], str]
 WhenType = Optional[Union["spack.spec.Spec", str, bool]]
 Patcher = Callable[[Union["spack.package_base.PackageBase", Dependency]], None]
 PatchesType = Optional[Union[Patcher, str, List[Union[Patcher, str]]]]
 
 
-SUPPORTED_LANGUAGES = ("fortran", "cxx")
+SUPPORTED_LANGUAGES = ("fortran", "cxx", "c")
 
 
 def _make_when_spec(value: WhenType) -> Optional["spack.spec.Spec"]:
@@ -475,7 +485,7 @@ def _execute_version(pkg, ver, **kwargs):
 
 def _depends_on(
     pkg: "spack.package_base.PackageBase",
-    spec: SpecType,
+    spec: "spack.spec.Spec",
     *,
     when: WhenType = None,
     type: DepType = dt.DEFAULT_TYPES,
@@ -485,11 +495,10 @@ def _depends_on(
     if not when_spec:
         return
 
-    dep_spec = spack.spec.Spec(spec)
+    if not spec.name:
-    if not dep_spec.name:
+        raise DependencyError(f"Invalid dependency specification in package '{pkg.name}':", spec)
-        raise DependencyError("Invalid dependency specification in package '%s':" % pkg.name, spec)
+    if pkg.name == spec.name:
-    if pkg.name == dep_spec.name:
+        raise CircularReferenceError(f"Package '{pkg.name}' cannot depend on itself.")
-        raise CircularReferenceError("Package '%s' cannot depend on itself." % pkg.name)
 
     depflag = dt.canonicalize(type)
 
@@ -505,7 +514,7 @@ def _depends_on(
     # ensure `Spec.virtual` is a valid thing to call in a directive.
     # For now, we comment out the following check to allow for virtual packages
     # with package files.
-    # if patches and dep_spec.virtual:
+    # if patches and spec.virtual:
     #     raise DependencyPatchError("Cannot patch a virtual dependency.")
 
     # ensure patches is a list
@@ -520,13 +529,13 @@ def _depends_on(
 
     # this is where we actually add the dependency to this package
     deps_by_name = pkg.dependencies.setdefault(when_spec, {})
-    dependency = deps_by_name.get(dep_spec.name)
+    dependency = deps_by_name.get(spec.name)
 
     if not dependency:
-        dependency = Dependency(pkg, dep_spec, depflag=depflag)
+        dependency = Dependency(pkg, spec, depflag=depflag)
-        deps_by_name[dep_spec.name] = dependency
+        deps_by_name[spec.name] = dependency
     else:
-        dependency.spec.constrain(dep_spec, deps=False)
+        dependency.spec.constrain(spec, deps=False)
         dependency.depflag |= depflag
 
     # apply patches to the dependency
@@ -591,12 +600,13 @@ def depends_on(
     @see The section "Dependency specs" in the Spack Packaging Guide.
 
     """
-    if spack.spec.Spec(spec).name in SUPPORTED_LANGUAGES:
+    dep_spec = spack.spec.Spec(spec)
+    if dep_spec.name in SUPPORTED_LANGUAGES:
         assert type == "build", "languages must be of 'build' type"
         return _language(lang_spec_str=spec, when=when)
 
     def _execute_depends_on(pkg: "spack.package_base.PackageBase"):
-        _depends_on(pkg, spec, when=when, type=type, patches=patches)
+        _depends_on(pkg, dep_spec, when=when, type=type, patches=patches)
 
     return _execute_depends_on
 
@@ -666,25 +676,24 @@ def extends(spec, when=None, type=("build", "run"), patches=None):
 
     keyword arguments can be passed to extends() so that extension
     packages can pass parameters to the extendee's extension
-    mechanism.
+    mechanism."""
-
-    """
 
     def _execute_extends(pkg):
         when_spec = _make_when_spec(when)
         if not when_spec:
             return
 
-        _depends_on(pkg, spec, when=when, type=type, patches=patches)
+        dep_spec = spack.spec.Spec(spec)
-        spec_obj = spack.spec.Spec(spec)
+
+        _depends_on(pkg, dep_spec, when=when, type=type, patches=patches)
 
         # When extending python, also add a dependency on python-venv. This is done so that
         # Spack environment views are Python virtual environments.
-        if spec_obj.name == "python" and not pkg.name == "python-venv":
+        if dep_spec.name == "python" and not pkg.name == "python-venv":
-            _depends_on(pkg, "python-venv", when=when, type=("build", "run"))
+            _depends_on(pkg, spack.spec.Spec("python-venv"), when=when, type=("build", "run"))
 
         # TODO: the values of the extendees dictionary are not used. Remove in next refactor.
-        pkg.extendees[spec_obj.name] = (spec_obj, None)
+        pkg.extendees[dep_spec.name] = (dep_spec, None)
 
     return _execute_extends
 

lib/spack/spack/environment/environment.py

@@ -5,7 +5,7 @@
 import collections
 import collections.abc
 import contextlib
-import copy
+import errno
 import os
 import pathlib
 import re
@@ -24,12 +24,15 @@
 from llnl.util.link_tree import ConflictingSpecsError
 from llnl.util.symlink import readlink, symlink
 
+import spack.caches
+import spack.cmd
 import spack.compilers
 import spack.concretize
 import spack.config
 import spack.deptypes as dt
 import spack.error
 import spack.fetch_strategy
+import spack.filesystem_view as fsv
 import spack.hash_types as ht
 import spack.hooks
 import spack.main
@@ -52,7 +55,6 @@
 import spack.util.url
 import spack.version
 from spack import traverse
-from spack.filesystem_view import SimpleFilesystemView, inverse_view_func_parser, view_func_parser
 from spack.installer import PackageInstaller
 from spack.schema.env import TOP_LEVEL_KEY
 from spack.spec import Spec
@@ -267,9 +269,7 @@ def root(name):
 
 def exists(name):
     """Whether an environment with this name exists or not."""
-    if not valid_env_name(name):
+    return valid_env_name(name) and os.path.isdir(_root(name))
-        return False
-    return os.path.isdir(root(name))
 
 
 def active(name):
@@ -528,8 +528,8 @@ def _read_yaml(str_or_file):
         )
 
     filename = getattr(str_or_file, "name", None)
-    default_data = spack.config.validate(data, spack.schema.env.schema, filename)
+    spack.config.validate(data, spack.schema.env.schema, filename)
-    return data, default_data
+    return data
 
 
 def _write_yaml(data, str_or_file):
@@ -606,7 +606,7 @@ def __init__(
         self.projections = projections
         self.select = select
         self.exclude = exclude
-        self.link_type = view_func_parser(link_type)
+        self.link_type = fsv.canonicalize_link_type(link_type)
         self.link = link
 
     def select_fn(self, spec):
@@ -640,7 +640,7 @@ def to_dict(self):
         if self.exclude:
             ret["exclude"] = self.exclude
         if self.link_type:
-            ret["link_type"] = inverse_view_func_parser(self.link_type)
+            ret["link_type"] = self.link_type
         if self.link != default_view_link:
             ret["link"] = self.link
         return ret
@@ -690,7 +690,7 @@ def get_projection_for_spec(self, spec):
         to exist on the filesystem."""
         return self._view(self.root).get_projection_for_spec(spec)
 
-    def view(self, new: Optional[str] = None) -> SimpleFilesystemView:
+    def view(self, new: Optional[str] = None) -> fsv.SimpleFilesystemView:
         """
         Returns a view object for the *underlying* view directory. This means that the
         self.root symlink is followed, and that the view has to exist on the filesystem
@@ -710,14 +710,14 @@ def view(self, new: Optional[str] = None) -> SimpleFilesystemView:
             )
         return self._view(path)
 
-    def _view(self, root: str) -> SimpleFilesystemView:
+    def _view(self, root: str) -> fsv.SimpleFilesystemView:
         """Returns a view object for a given root dir."""
-        return SimpleFilesystemView(
+        return fsv.SimpleFilesystemView(
             root,
             spack.store.STORE.layout,
             ignore_conflicts=True,
             projections=self.projections,
-            link=self.link_type,
+            link_type=self.link_type,
         )
 
     def __contains__(self, spec):
@@ -789,6 +789,23 @@ def regenerate(self, concrete_roots: List[Spec]) -> None:
         root_dirname = os.path.dirname(self.root)
         tmp_symlink_name = os.path.join(root_dirname, "._view_link")
 
+        # Remove self.root if is it an empty dir, since we need a symlink there. Note that rmdir
+        # fails if self.root is a symlink.
+        try:
+            os.rmdir(self.root)
+        except (FileNotFoundError, NotADirectoryError):
+            pass
+        except OSError as e:
+            if e.errno == errno.ENOTEMPTY:
+                msg = "it is a non-empty directory"
+            elif e.errno == errno.EACCES:
+                msg = "of insufficient permissions"
+            else:
+                raise
+            raise SpackEnvironmentViewError(
+                f"The environment view in {self.root} cannot not be created because {msg}."
+            ) from e
+
         # Create a new view
         try:
             fs.mkdirp(new_root)
@@ -920,7 +937,7 @@ def __init__(self, manifest_dir: Union[str, pathlib.Path]) -> None:
     def _load_manifest_file(self):
         """Instantiate and load the manifest file contents into memory."""
         with lk.ReadTransaction(self.txlock):
-            self.manifest = EnvironmentManifestFile(self.path)
+            self.manifest = EnvironmentManifestFile(self.path, self.name)
             with self.manifest.use_config():
                 self._read()
 
@@ -957,18 +974,25 @@ def write_transaction(self):
         """Get a write lock context manager for use in a `with` block."""
         return lk.WriteTransaction(self.txlock, acquire=self._re_read)
 
-    def _process_definition(self, item):
+    def _process_definition(self, entry):
         """Process a single spec definition item."""
-        entry = copy.deepcopy(item)
+        when_string = entry.get("when")
-        when = _eval_conditional(entry.pop("when", "True"))
+        if when_string is not None:
-        assert len(entry) == 1
+            when = _eval_conditional(when_string)
+            assert len([x for x in entry if x != "when"]) == 1
+        else:
+            when = True
+            assert len(entry) == 1
+
         if when:
-            name, spec_list = next(iter(entry.items()))
+            for name, spec_list in entry.items():
-            user_specs = SpecList(name, spec_list, self.spec_lists.copy())
+                if name == "when":
-            if name in self.spec_lists:
+                    continue
-                self.spec_lists[name].extend(user_specs)
+                user_specs = SpecList(name, spec_list, self.spec_lists.copy())
-            else:
+                if name in self.spec_lists:
-                self.spec_lists[name] = user_specs
+                    self.spec_lists[name].extend(user_specs)
+                else:
+                    self.spec_lists[name] = user_specs
 
     def _process_view(self, env_view: Optional[Union[bool, str, Dict]]):
         """Process view option(s), which can be boolean, string, or None.
@@ -2473,27 +2497,21 @@ def _equiv_dict(first, second):
     return same_values and same_keys_with_same_overrides
 
 
-def display_specs(concretized_specs):
+def display_specs(specs):
-    """Displays the list of specs returned by `Environment.concretize()`.
+    """Displays a list of specs traversed breadth-first, covering nodes, with install status.
 
     Args:
-        concretized_specs (list): list of specs returned by
+        specs (list): list of specs
-            `Environment.concretize()`
     """
-
+    tree_string = spack.spec.tree(
-    def _tree_to_display(spec):
+        specs,
-        return spec.tree(
+        format=spack.spec.DISPLAY_FORMAT,
-            recurse_dependencies=True,
+        hashes=True,
-            format=spack.spec.DISPLAY_FORMAT,
+        hashlen=7,
-            status_fn=spack.spec.Spec.install_status,
+        status_fn=spack.spec.Spec.install_status,
-            hashlen=7,
+        key=traverse.by_dag_hash,
-            hashes=True,
+    )
-        )
+    print(tree_string)
-
-    for user_spec, concrete_spec in concretized_specs:
-        tty.msg("Concretized {0}".format(user_spec))
-        sys.stdout.write(_tree_to_display(concrete_spec))
-        print("")
 
 
 def _concretize_from_constraints(spec_constraints, tests=False):
@@ -2547,7 +2565,7 @@ def _concretize_task(packed_arguments) -> Tuple[int, Spec, float]:
 
 def make_repo_path(root):
     """Make a RepoPath from the repo subdirectories in an environment."""
-    path = spack.repo.RepoPath()
+    path = spack.repo.RepoPath(cache=spack.caches.MISC_CACHE)
 
     if os.path.isdir(root):
         for repo_root in os.listdir(root):
@@ -2556,7 +2574,7 @@ def make_repo_path(root):
             if not os.path.isdir(repo_root):
                 continue
 
-            repo = spack.repo.Repo(repo_root)
+            repo = spack.repo.from_path(repo_root)
             path.put_last(repo)
 
     return path
@@ -2757,10 +2775,11 @@ def from_lockfile(manifest_dir: Union[pathlib.Path, str]) -> "EnvironmentManifes
         manifest.flush()
         return manifest
 
-    def __init__(self, manifest_dir: Union[pathlib.Path, str]) -> None:
+    def __init__(self, manifest_dir: Union[pathlib.Path, str], name: Optional[str] = None) -> None:
         self.manifest_dir = pathlib.Path(manifest_dir)
+        self.name = name or str(manifest_dir)
         self.manifest_file = self.manifest_dir / manifest_name
-        self.scope_name = f"env:{environment_name(self.manifest_dir)}"
+        self.scope_name = f"env:{self.name}"
         self.config_stage_dir = os.path.join(env_subdir_path(manifest_dir), "config")
 
         #: Configuration scopes associated with this environment. Note that these are not
@@ -2772,12 +2791,8 @@ def __init__(self, manifest_dir: Union[pathlib.Path, str]) -> None:
             raise SpackEnvironmentError(msg)
 
         with self.manifest_file.open() as f:
-            raw, with_defaults_added = _read_yaml(f)
+            self.yaml_content = _read_yaml(f)
 
-        #: Pristine YAML content, without defaults being added
-        self.pristine_yaml_content = raw
-        #: YAML content with defaults added by Spack, if they're missing
-        self.yaml_content = with_defaults_added
         self.changed = False
 
     def _all_matches(self, user_spec: str) -> List[str]:
@@ -2791,7 +2806,7 @@ def _all_matches(self, user_spec: str) -> List[str]:
             ValueError: if no equivalent match is found
         """
         result = []
-        for yaml_spec_str in self.pristine_configuration["specs"]:
+        for yaml_spec_str in self.configuration["specs"]:
             if Spec(yaml_spec_str) == Spec(user_spec):
                 result.append(yaml_spec_str)
 
@@ -2806,7 +2821,6 @@ def add_user_spec(self, user_spec: str) -> None:
         Args:
             user_spec: user spec to be appended
         """
-        self.pristine_configuration.setdefault("specs", []).append(user_spec)
         self.configuration.setdefault("specs", []).append(user_spec)
         self.changed = True
 
@@ -2821,7 +2835,6 @@ def remove_user_spec(self, user_spec: str) -> None:
         """
         try:
             for key in self._all_matches(user_spec):
-                self.pristine_configuration["specs"].remove(key)
                 self.configuration["specs"].remove(key)
         except ValueError as e:
             msg = f"cannot remove {user_spec} from {self}, no such spec exists"
@@ -2839,7 +2852,6 @@ def override_user_spec(self, user_spec: str, idx: int) -> None:
             SpackEnvironmentError: when the user spec cannot be overridden
         """
         try:
-            self.pristine_configuration["specs"][idx] = user_spec
             self.configuration["specs"][idx] = user_spec
         except ValueError as e:
             msg = f"cannot override {user_spec} from {self}"
@@ -2852,10 +2864,10 @@ def set_include_concrete(self, include_concrete: List[str]) -> None:
         Args:
             include_concrete: list of already existing concrete environments to include
         """
-        self.pristine_configuration[included_concrete_name] = []
+        self.configuration[included_concrete_name] = []
 
         for env_path in include_concrete:
-            self.pristine_configuration[included_concrete_name].append(env_path)
+            self.configuration[included_concrete_name].append(env_path)
 
         self.changed = True
 
@@ -2869,14 +2881,13 @@ def add_definition(self, user_spec: str, list_name: str) -> None:
         Raises:
             SpackEnvironmentError: is no valid definition exists already
         """
-        defs = self.pristine_configuration.get("definitions", [])
+        defs = self.configuration.get("definitions", [])
         msg = f"cannot add {user_spec} to the '{list_name}' definition, no valid list exists"
 
         for idx, item in self._iterate_on_definitions(defs, list_name=list_name, err_msg=msg):
             item[list_name].append(user_spec)
             break
 
-        self.configuration["definitions"][idx][list_name].append(user_spec)
         self.changed = True
 
     def remove_definition(self, user_spec: str, list_name: str) -> None:
@@ -2890,7 +2901,7 @@ def remove_definition(self, user_spec: str, list_name: str) -> None:
             SpackEnvironmentError: if the user spec cannot be removed from the list,
                 or the list does not exist
         """
-        defs = self.pristine_configuration.get("definitions", [])
+        defs = self.configuration.get("definitions", [])
         msg = (
             f"cannot remove {user_spec} from the '{list_name}' definition, "
             f"no valid list exists"
@@ -2903,7 +2914,6 @@ def remove_definition(self, user_spec: str, list_name: str) -> None:
             except ValueError:
                 pass
 
-        self.configuration["definitions"][idx][list_name].remove(user_spec)
         self.changed = True
 
     def override_definition(self, user_spec: str, *, override: str, list_name: str) -> None:
@@ -2918,7 +2928,7 @@ def override_definition(self, user_spec: str, *, override: str, list_name: str)
         Raises:
             SpackEnvironmentError: if the user spec cannot be overridden
         """
-        defs = self.pristine_configuration.get("definitions", [])
+        defs = self.configuration.get("definitions", [])
         msg = f"cannot override {user_spec} with {override} in the '{list_name}' definition"
 
         for idx, item in self._iterate_on_definitions(defs, list_name=list_name, err_msg=msg):
@@ -2929,7 +2939,6 @@ def override_definition(self, user_spec: str, *, override: str, list_name: str)
             except ValueError:
                 pass
 
-        self.configuration["definitions"][idx][list_name][sub_index] = override
         self.changed = True
 
     def _iterate_on_definitions(self, definitions, *, list_name, err_msg):
@@ -2961,7 +2970,6 @@ def set_default_view(self, view: Union[bool, str, pathlib.Path, Dict[str, str]])
                 True the default view is used for the environment, if False there's no view.
         """
         if isinstance(view, dict):
-            self.pristine_configuration["view"][default_view_name].update(view)
             self.configuration["view"][default_view_name].update(view)
             self.changed = True
             return
@@ -2969,15 +2977,13 @@ def set_default_view(self, view: Union[bool, str, pathlib.Path, Dict[str, str]])
         if not isinstance(view, bool):
             view = str(view)
 
-        self.pristine_configuration["view"] = view
         self.configuration["view"] = view
         self.changed = True
 
     def remove_default_view(self) -> None:
         """Removes the default view from the manifest file"""
-        view_data = self.pristine_configuration.get("view")
+        view_data = self.configuration.get("view")
         if isinstance(view_data, collections.abc.Mapping):
-            self.pristine_configuration["view"].pop(default_view_name)
             self.configuration["view"].pop(default_view_name)
             self.changed = True
             return
@@ -2990,17 +2996,12 @@ def flush(self) -> None:
             return
 
         with fs.write_tmp_and_move(os.path.realpath(self.manifest_file)) as f:
-            _write_yaml(self.pristine_yaml_content, f)
+            _write_yaml(self.yaml_content, f)
         self.changed = False
 
-    @property
-    def pristine_configuration(self):
-        """Return the dictionaries in the pristine YAML, without the top level attribute"""
-        return self.pristine_yaml_content[TOP_LEVEL_KEY]
-
     @property
     def configuration(self):
-        """Return the dictionaries in the YAML, without the top level attribute"""
+        """Return the dictionaries in the pristine YAML, without the top level attribute"""
         return self.yaml_content[TOP_LEVEL_KEY]
 
     def __len__(self):
@@ -3032,12 +3033,11 @@ def included_config_scopes(self) -> List[spack.config.ConfigScope]:
             SpackEnvironmentError: if the manifest includes a remote file but
                 no configuration stage directory has been identified
         """
-        scopes = []
+        scopes: List[spack.config.ConfigScope] = []
 
         # load config scopes added via 'include:', in reverse so that
         # highest-precedence scopes are last.
         includes = self[TOP_LEVEL_KEY].get("include", [])
-        env_name = environment_name(self.manifest_dir)
         missing = []
         for i, config_path in enumerate(reversed(includes)):
             # allow paths to contain spack config/environment variables, etc.
@@ -3100,24 +3100,22 @@ def included_config_scopes(self) -> List[spack.config.ConfigScope]:
 
             if os.path.isdir(config_path):
                 # directories are treated as regular ConfigScopes
-                config_name = "env:%s:%s" % (env_name, os.path.basename(config_path))
+                config_name = f"env:{self.name}:{os.path.basename(config_path)}"
-                tty.debug("Creating ConfigScope {0} for '{1}'".format(config_name, config_path))
+                tty.debug(f"Creating DirectoryConfigScope {config_name} for '{config_path}'")
-                scope = spack.config.ConfigScope(config_name, config_path)
+                scopes.append(spack.config.DirectoryConfigScope(config_name, config_path))
             elif os.path.exists(config_path):
                 # files are assumed to be SingleFileScopes
-                config_name = "env:%s:%s" % (env_name, config_path)
+                config_name = f"env:{self.name}:{config_path}"
-                tty.debug(
+                tty.debug(f"Creating SingleFileScope {config_name} for '{config_path}'")
-                    "Creating SingleFileScope {0} for '{1}'".format(config_name, config_path)
+                scopes.append(
-                )
+                    spack.config.SingleFileScope(
-                scope = spack.config.SingleFileScope(
+                        config_name, config_path, spack.schema.merged.schema
-                    config_name, config_path, spack.schema.merged.schema
+                    )
                 )
             else:
                 missing.append(config_path)
                 continue
 
-            scopes.append(scope)
-
         if missing:
             msg = "Detected {0} missing include path(s):".format(len(missing))
             msg += "\n   {0}".format("\n   ".join(missing))
@@ -3134,7 +3132,10 @@ def env_config_scopes(self) -> List[spack.config.ConfigScope]:
         scopes: List[spack.config.ConfigScope] = [
             *self.included_config_scopes,
             spack.config.SingleFileScope(
-                self.scope_name, str(self.manifest_file), spack.schema.env.schema, [TOP_LEVEL_KEY]
+                self.scope_name,
+                str(self.manifest_file),
+                spack.schema.env.schema,
+                yaml_path=[TOP_LEVEL_KEY],
             ),
         ]
         ensure_no_disallowed_env_config_mods(scopes)

lib/spack/spack/filesystem_view.py

@@ -10,8 +10,9 @@
 import shutil
 import stat
 import sys
-from typing import Optional
+from typing import Callable, Dict, Optional
 
+from llnl.string import comma_or
 from llnl.util import tty
 from llnl.util.filesystem import (
     mkdirp,
@@ -49,19 +50,20 @@
 _projections_path = ".spack/projections.yaml"
 
 
-def view_symlink(src, dst, **kwargs):
+LinkCallbackType = Callable[[str, str, "FilesystemView", Optional["spack.spec.Spec"]], None]
-    # keyword arguments are irrelevant
+
-    # here to fit required call signature
+
+def view_symlink(src: str, dst: str, *args, **kwargs) -> None:
     symlink(src, dst)
 
 
-def view_hardlink(src, dst, **kwargs):
+def view_hardlink(src: str, dst: str, *args, **kwargs) -> None:
-    # keyword arguments are irrelevant
-    # here to fit required call signature
     os.link(src, dst)
 
 
-def view_copy(src: str, dst: str, view, spec: Optional[spack.spec.Spec] = None):
+def view_copy(
+    src: str, dst: str, view: "FilesystemView", spec: Optional["spack.spec.Spec"] = None
+) -> None:
     """
     Copy a file from src to dst.
 
@@ -104,27 +106,40 @@ def view_copy(src: str, dst: str, view, spec: Optional[spack.spec.Spec] = None):
         tty.debug(f"Can't change the permissions for {dst}")
 
 
-def view_func_parser(parsed_name):
+#: supported string values for `link_type` in an env, mapped to canonical values
-    # What method are we using for this view
+_LINK_TYPES = {
-    if parsed_name in ("hardlink", "hard"):
+    "hardlink": "hardlink",
+    "hard": "hardlink",
+    "copy": "copy",
+    "relocate": "copy",
+    "add": "symlink",
+    "symlink": "symlink",
+    "soft": "symlink",
+}
+
+_VALID_LINK_TYPES = sorted(set(_LINK_TYPES.values()))
+
+
+def canonicalize_link_type(link_type: str) -> str:
+    """Return canonical"""
+    canonical = _LINK_TYPES.get(link_type)
+    if not canonical:
+        raise ValueError(
+            f"Invalid link type: '{link_type}. Must be one of {comma_or(_VALID_LINK_TYPES)}'"
+        )
+    return canonical
+
+
+def function_for_link_type(link_type: str) -> LinkCallbackType:
+    link_type = canonicalize_link_type(link_type)
+    if link_type == "hardlink":
         return view_hardlink
-    elif parsed_name in ("copy", "relocate"):
+    elif link_type == "symlink":
-        return view_copy
-    elif parsed_name in ("add", "symlink", "soft"):
         return view_symlink
-    else:
+    elif link_type == "copy":
-        raise ValueError(f"invalid link type for view: '{parsed_name}'")
+        return view_copy
 
-
+    assert False, "invalid link type"  # need mypy Literal values
-def inverse_view_func_parser(view_type):
-    # get string based on view type
-    if view_type is view_hardlink:
-        link_name = "hardlink"
-    elif view_type is view_copy:
-        link_name = "copy"
-    else:
-        link_name = "symlink"
-    return link_name
 
 
 class FilesystemView:
@@ -140,7 +155,16 @@ class FilesystemView:
     directory structure.
     """
 
-    def __init__(self, root, layout, **kwargs):
+    def __init__(
+        self,
+        root: str,
+        layout: "spack.directory_layout.DirectoryLayout",
+        *,
+        projections: Optional[Dict] = None,
+        ignore_conflicts: bool = False,
+        verbose: bool = False,
+        link_type: str = "symlink",
+    ):
         """
         Initialize a filesystem view under the given `root` directory with
         corresponding directory `layout`.
@@ -149,15 +173,14 @@ def __init__(self, root, layout, **kwargs):
         """
         self._root = root
         self.layout = layout
+        self.projections = {} if projections is None else projections
 
-        self.projections = kwargs.get("projections", {})
+        self.ignore_conflicts = ignore_conflicts
-
+        self.verbose = verbose
-        self.ignore_conflicts = kwargs.get("ignore_conflicts", False)
-        self.verbose = kwargs.get("verbose", False)
 
         # Setup link function to include view
-        link_func = kwargs.get("link", view_symlink)
+        self.link_type = link_type
-        self.link = ft.partial(link_func, view=self)
+        self.link = ft.partial(function_for_link_type(link_type), view=self)
 
     def add_specs(self, *specs, **kwargs):
         """
@@ -255,8 +278,24 @@ class YamlFilesystemView(FilesystemView):
     Filesystem view to work with a yaml based directory layout.
     """
 
-    def __init__(self, root, layout, **kwargs):
+    def __init__(
-        super().__init__(root, layout, **kwargs)
+        self,
+        root: str,
+        layout: "spack.directory_layout.DirectoryLayout",
+        *,
+        projections: Optional[Dict] = None,
+        ignore_conflicts: bool = False,
+        verbose: bool = False,
+        link_type: str = "symlink",
+    ):
+        super().__init__(
+            root,
+            layout,
+            projections=projections,
+            ignore_conflicts=ignore_conflicts,
+            verbose=verbose,
+            link_type=link_type,
+        )
 
         # Super class gets projections from the kwargs
         # YAML specific to get projections from YAML file
@@ -638,9 +677,6 @@ class SimpleFilesystemView(FilesystemView):
     """A simple and partial implementation of FilesystemView focused on performance and immutable
     views, where specs cannot be removed after they were added."""
 
-    def __init__(self, root, layout, **kwargs):
-        super().__init__(root, layout, **kwargs)
-
     def _sanity_check_view_projection(self, specs):
         """A very common issue is that we end up with two specs of the same package, that project
         to the same prefix. We want to catch that as early as possible and give a sensible error to

lib/spack/spack/installer.py

@@ -582,7 +582,7 @@ def dump_packages(spec: "spack.spec.Spec", path: str) -> None:
 
             # Create a source repo and get the pkg directory out of it.
             try:
-                source_repo = spack.repo.Repo(source_repo_root)
+                source_repo = spack.repo.from_path(source_repo_root)
                 source_pkg_dir = source_repo.dirname_for_package_name(node.name)
             except spack.repo.RepoError as err:
                 tty.debug(f"Failed to create source repo for {node.name}: {str(err)}")
@@ -593,7 +593,7 @@ def dump_packages(spec: "spack.spec.Spec", path: str) -> None:
         dest_repo_root = os.path.join(path, node.namespace)
         if not os.path.exists(dest_repo_root):
             spack.repo.create_repo(dest_repo_root)
-        repo = spack.repo.Repo(dest_repo_root)
+        repo = spack.repo.from_path(dest_repo_root)
 
         # Get the location of the package in the dest repo.
         dest_pkg_dir = repo.dirname_for_package_name(node.name)
@@ -1542,17 +1542,6 @@ def _add_tasks(self, request: BuildRequest, all_deps):
             tty.warn(f"Installation request refused: {str(err)}")
             return
 
-        # Skip out early if the spec is not being installed locally (i.e., if
-        # external or upstream).
-        #
-        # External and upstream packages need to get flagged as installed to
-        # ensure proper status tracking for environment build.
-        explicit = request.pkg.spec.dag_hash() in request.install_args.get("explicit", [])
-        not_local = _handle_external_and_upstream(request.pkg, explicit)
-        if not_local:
-            self._flag_installed(request.pkg)
-            return
-
         install_compilers = spack.config.get("config:install_missing_compilers", False)
 
         install_deps = request.install_args.get("install_deps")
@@ -2029,11 +2018,10 @@ def install(self) -> None:
             # Skip the installation if the spec is not being installed locally
             # (i.e., if external or upstream) BUT flag it as installed since
             # some package likely depends on it.
-            if not task.explicit:
+            if _handle_external_and_upstream(pkg, task.explicit):
-                if _handle_external_and_upstream(pkg, False):
+                term_status.clear()
-                    term_status.clear()
+                self._flag_installed(pkg, task.dependents)
-                    self._flag_installed(pkg, task.dependents)
+                continue
-                    continue
 
             # Flag a failed spec.  Do not need an (install) prefix lock since
             # assume using a separate (failed) prefix lock file.

lib/spack/spack/main.py

@@ -444,8 +444,9 @@ def make_argument_parser(**kwargs):
         "--config-scope",
         dest="config_scopes",
         action="append",
-        metavar="DIR",
+        metavar="DIR|ENV",
-        help="add a custom configuration scope",
+        help="add directory or environment as read-only configuration scope, without activating "
+        "the environment.",
     )
     parser.add_argument(
         "-d",

lib/spack/spack/operating_systems/mac_os.py

@@ -143,6 +143,7 @@ def __init__(self):
             "12": "monterey",
             "13": "ventura",
             "14": "sonoma",
+            "15": "sequoia",
         }
 
         version = macos_version()

lib/spack/spack/package_base.py

@@ -35,6 +35,7 @@
 
 import spack.compilers
 import spack.config
+import spack.dependency
 import spack.deptypes as dt
 import spack.directives
 import spack.directory_layout
@@ -199,10 +200,10 @@ def __init__(cls, name, bases, attr_dict):
         # assumed to be detectable
         if hasattr(cls, "executables") or hasattr(cls, "libraries"):
             # Append a tag to each detectable package, so that finding them is faster
-            if hasattr(cls, "tags"):
+            if not hasattr(cls, "tags"):
-                getattr(cls, "tags").append(DetectablePackageMeta.TAG)
-            else:
                 setattr(cls, "tags", [DetectablePackageMeta.TAG])
+            elif DetectablePackageMeta.TAG not in cls.tags:
+                cls.tags.append(DetectablePackageMeta.TAG)
 
             @classmethod
             def platform_executables(cls):
@@ -621,10 +622,6 @@ class PackageBase(WindowsRPath, PackageViewMixin, RedistributionMixin, metaclass
     #: By default do not run tests within package's install()
     run_tests = False
 
-    #: Keep -Werror flags, matches config:flags:keep_werror to override config
-    # NOTE: should be type Optional[Literal['all', 'specific', 'none']] in 3.8+
-    keep_werror: Optional[str] = None
-
     #: Most packages are NOT extendable. Set to True if you want extensions.
     extendable = False
 
@@ -752,11 +749,6 @@ def __init__(self, spec):
         self._fetch_time = 0.0
 
         self.win_rpath = fsys.WindowsSimulatedRPath(self)
-
-        if self.is_extension:
-            pkg_cls = spack.repo.PATH.get_pkg_class(self.extendee_spec.name)
-            pkg_cls(self.extendee_spec)._check_extendable()
-
         super().__init__()
 
     @classmethod
@@ -930,6 +922,32 @@ def global_license_file(self):
             self.global_license_dir, self.name, os.path.basename(self.license_files[0])
         )
 
+    # NOTE: return type should be Optional[Literal['all', 'specific', 'none']] in
+    # Python 3.8+, but we still support 3.6.
+    @property
+    def keep_werror(self) -> Optional[str]:
+        """Keep ``-Werror`` flags, matches ``config:flags:keep_werror`` to override config.
+
+        Valid return values are:
+        * ``"all"``: keep all ``-Werror`` flags.
+        * ``"specific"``: keep only ``-Werror=specific-warning`` flags.
+        * ``"none"``: filter out all ``-Werror*`` flags.
+        * ``None``: respect the user's configuration (``"none"`` by default).
+        """
+        if self.spec.satisfies("%nvhpc@:23.3") or self.spec.satisfies("%pgi"):
+            # Filtering works by replacing -Werror with -Wno-error, but older nvhpc and
+            # PGI do not understand -Wno-error, so we disable filtering.
+            return "all"
+
+        elif self.spec.satisfies("%nvhpc@23.4:"):
+            # newer nvhpc supports -Wno-error but can't disable specific warnings with
+            # -Wno-error=warning. Skip -Werror=warning, but still filter -Werror.
+            return "specific"
+
+        else:
+            # use -Werror disablement by default for other compilers
+            return None
+
     @property
     def version(self):
         if not self.spec.versions.concrete:
@@ -1119,10 +1137,9 @@ def _make_stage(self):
             if not link_format:
                 link_format = "build-{arch}-{hash:7}"
             stage_link = self.spec.format_path(link_format)
-            return DevelopStage(compute_stage_name(self.spec), dev_path, stage_link)
+            source_stage = DevelopStage(compute_stage_name(self.spec), dev_path, stage_link)
-
+        else:
-        # To fetch the current version
+            source_stage = self._make_root_stage(self.fetcher)
-        source_stage = self._make_root_stage(self.fetcher)
 
         # all_stages is source + resources + patches
         all_stages = StageComposite()
@@ -1451,10 +1468,8 @@ def do_fetch(self, mirror_only=False):
             return
 
         checksum = spack.config.get("config:checksum")
-        fetch = self.stage.needs_fetching
         if (
             checksum
-            and fetch
             and (self.version not in self.versions)
             and (not isinstance(self.version, GitVersion))
         ):
@@ -1561,13 +1576,11 @@ def do_patch(self):
                 tty.debug("Patching failed last time. Restaging.")
                 self.stage.restage()
             else:
-                # develop specs/ DIYStages may have patch failures but
+                # develop specs may have patch failures but should never be restaged
-                # should never be restaged
+                tty.warn(
-                msg = (
+                    f"A patch failure was detected in {self.name}."
-                    "A patch failure was detected in %s." % self.name
+                    " Build errors may occur due to this."
-                    + " Build errors may occur due to this."
                 )
-                tty.warn(msg)
                 return
 
         # If this file exists, then we already applied all the patches.
@@ -2371,10 +2384,6 @@ def do_deprecate(self, deprecator, link_fn):
         PackageBase.uninstall_by_spec(spec, force=True, deprecator=deprecator)
         link_fn(deprecator.prefix, spec.prefix)
 
-    def _check_extendable(self):
-        if not self.extendable:
-            raise ValueError("Package %s is not extendable!" % self.name)
-
     def view(self):
         """Create a view with the prefix of this package as the root.
         Extensions added to this view will modify the installation prefix of
@@ -2417,9 +2426,8 @@ def all_urls(self) -> List[str]:
         if hasattr(self, "url") and self.url:
             urls.append(self.url)
 
-        # fetch from first entry in urls to save time
         if hasattr(self, "urls") and self.urls:
-            urls.append(self.urls[0])
+            urls.extend(self.urls)
 
         for args in self.versions.values():
             if "url" in args:

lib/spack/spack/parser.py

@@ -328,19 +328,26 @@ def next_spec(
         if not self.ctx.next_token:
             return initial_spec
 
+        def add_dependency(dep, **edge_properties):
+            """wrapper around root_spec._add_dependency"""
+            try:
+                root_spec._add_dependency(dep, **edge_properties)
+            except spack.error.SpecError as e:
+                raise SpecParsingError(str(e), self.ctx.current_token, self.literal_str) from e
+
         initial_spec = initial_spec or spack.spec.Spec()
-        root_spec = SpecNodeParser(self.ctx).parse(initial_spec)
+        root_spec = SpecNodeParser(self.ctx, self.literal_str).parse(initial_spec)
         while True:
             if self.ctx.accept(TokenType.START_EDGE_PROPERTIES):
                 edge_properties = EdgeAttributeParser(self.ctx, self.literal_str).parse()
                 edge_properties.setdefault("depflag", 0)
                 edge_properties.setdefault("virtuals", ())
                 dependency = self._parse_node(root_spec)
-                root_spec._add_dependency(dependency, **edge_properties)
+                add_dependency(dependency, **edge_properties)
 
             elif self.ctx.accept(TokenType.DEPENDENCY):
                 dependency = self._parse_node(root_spec)
-                root_spec._add_dependency(dependency, depflag=0, virtuals=())
+                add_dependency(dependency, depflag=0, virtuals=())
 
             else:
                 break
@@ -348,7 +355,7 @@ def next_spec(
         return root_spec
 
     def _parse_node(self, root_spec):
-        dependency = SpecNodeParser(self.ctx).parse()
+        dependency = SpecNodeParser(self.ctx, self.literal_str).parse()
         if dependency is None:
             msg = (
                 "the dependency sigil and any optional edge attributes must be followed by a "
@@ -367,10 +374,11 @@ def all_specs(self) -> List["spack.spec.Spec"]:
 class SpecNodeParser:
     """Parse a single spec node from a stream of tokens"""
 
-    __slots__ = "ctx", "has_compiler", "has_version"
+    __slots__ = "ctx", "has_compiler", "has_version", "literal_str"
 
-    def __init__(self, ctx):
+    def __init__(self, ctx, literal_str):
         self.ctx = ctx
+        self.literal_str = literal_str
         self.has_compiler = False
         self.has_version = False
 
@@ -388,7 +396,8 @@ def parse(
         if not self.ctx.next_token or self.ctx.expect(TokenType.DEPENDENCY):
             return initial_spec
 
-        initial_spec = initial_spec or spack.spec.Spec()
+        if initial_spec is None:
+            initial_spec = spack.spec.Spec()
 
         # If we start with a package name we have a named spec, we cannot
         # accept another package name afterwards in a node
@@ -405,12 +414,21 @@ def parse(
         elif self.ctx.accept(TokenType.FILENAME):
             return FileParser(self.ctx).parse(initial_spec)
 
+        def raise_parsing_error(string: str, cause: Optional[Exception] = None):
+            """Raise a spec parsing error with token context."""
+            raise SpecParsingError(string, self.ctx.current_token, self.literal_str) from cause
+
+        def add_flag(name: str, value: str, propagate: bool):
+            """Wrapper around ``Spec._add_flag()`` that adds parser context to errors raised."""
+            try:
+                initial_spec._add_flag(name, value, propagate)
+            except Exception as e:
+                raise_parsing_error(str(e), e)
+
         while True:
             if self.ctx.accept(TokenType.COMPILER):
                 if self.has_compiler:
-                    raise spack.spec.DuplicateCompilerSpecError(
+                    raise_parsing_error("Spec cannot have multiple compilers")
-                        f"{initial_spec} cannot have multiple compilers"
-                    )
 
                 compiler_name = self.ctx.current_token.value[1:]
                 initial_spec.compiler = spack.spec.CompilerSpec(compiler_name.strip(), ":")
@@ -418,9 +436,7 @@ def parse(
 
             elif self.ctx.accept(TokenType.COMPILER_AND_VERSION):
                 if self.has_compiler:
-                    raise spack.spec.DuplicateCompilerSpecError(
+                    raise_parsing_error("Spec cannot have multiple compilers")
-                        f"{initial_spec} cannot have multiple compilers"
-                    )
 
                 compiler_name, compiler_version = self.ctx.current_token.value[1:].split("@")
                 initial_spec.compiler = spack.spec.CompilerSpec(
@@ -434,9 +450,8 @@ def parse(
                 or self.ctx.accept(TokenType.VERSION)
             ):
                 if self.has_version:
-                    raise spack.spec.MultipleVersionError(
+                    raise_parsing_error("Spec cannot have multiple versions")
-                        f"{initial_spec} cannot have multiple versions"
+
-                    )
                 initial_spec.versions = spack.version.VersionList(
                     [spack.version.from_string(self.ctx.current_token.value[1:])]
                 )
@@ -445,29 +460,25 @@ def parse(
 
             elif self.ctx.accept(TokenType.BOOL_VARIANT):
                 variant_value = self.ctx.current_token.value[0] == "+"
-                initial_spec._add_flag(
+                add_flag(self.ctx.current_token.value[1:].strip(), variant_value, propagate=False)
-                    self.ctx.current_token.value[1:].strip(), variant_value, propagate=False
-                )
 
             elif self.ctx.accept(TokenType.PROPAGATED_BOOL_VARIANT):
                 variant_value = self.ctx.current_token.value[0:2] == "++"
-                initial_spec._add_flag(
+                add_flag(self.ctx.current_token.value[2:].strip(), variant_value, propagate=True)
-                    self.ctx.current_token.value[2:].strip(), variant_value, propagate=True
-                )
 
             elif self.ctx.accept(TokenType.KEY_VALUE_PAIR):
                 match = SPLIT_KVP.match(self.ctx.current_token.value)
                 assert match, "SPLIT_KVP and KEY_VALUE_PAIR do not agree."
 
-                name, delim, value = match.groups()
+                name, _, value = match.groups()
-                initial_spec._add_flag(name, strip_quotes_and_unescape(value), propagate=False)
+                add_flag(name, strip_quotes_and_unescape(value), propagate=False)
 
             elif self.ctx.accept(TokenType.PROPAGATED_KEY_VALUE_PAIR):
                 match = SPLIT_KVP.match(self.ctx.current_token.value)
                 assert match, "SPLIT_KVP and PROPAGATED_KEY_VALUE_PAIR do not agree."
 
-                name, delim, value = match.groups()
+                name, _, value = match.groups()
-                initial_spec._add_flag(name, strip_quotes_and_unescape(value), propagate=True)
+                add_flag(name, strip_quotes_and_unescape(value), propagate=True)
 
             elif self.ctx.expect(TokenType.DAG_HASH):
                 if initial_spec.abstract_hash:

lib/spack/spack/patch.py

@@ -9,7 +9,7 @@
 import os.path
 import pathlib
 import sys
-from typing import Any, Dict, Optional, Tuple, Type
+from typing import Any, Dict, Optional, Tuple, Type, Union
 
 import llnl.util.filesystem
 from llnl.url import allowed_archive
@@ -65,6 +65,9 @@ def apply_patch(
         patch(*args)
 
 
+PatchPackageType = Union["spack.package_base.PackageBase", Type["spack.package_base.PackageBase"]]
+
+
 class Patch:
     """Base class for patches.
 
@@ -77,7 +80,7 @@ class Patch:
 
     def __init__(
         self,
-        pkg: "spack.package_base.PackageBase",
+        pkg: PatchPackageType,
         path_or_url: str,
         level: int,
         working_dir: str,
@@ -159,7 +162,7 @@ class FilePatch(Patch):
 
     def __init__(
         self,
-        pkg: "spack.package_base.PackageBase",
+        pkg: PatchPackageType,
         relative_path: str,
         level: int,
         working_dir: str,
@@ -183,7 +186,7 @@ def __init__(
         abs_path: Optional[str] = None
         # At different times we call FilePatch on instances and classes
         pkg_cls = pkg if inspect.isclass(pkg) else pkg.__class__
-        for cls in inspect.getmro(pkg_cls):
+        for cls in inspect.getmro(pkg_cls):  # type: ignore
             if not hasattr(cls, "module"):
                 # We've gone too far up the MRO
                 break
@@ -242,7 +245,7 @@ class UrlPatch(Patch):
 
     def __init__(
         self,
-        pkg: "spack.package_base.PackageBase",
+        pkg: PatchPackageType,
         url: str,
         level: int = 1,
         *,
@@ -361,8 +364,9 @@ def from_dict(
     """
     repository = repository or spack.repo.PATH
     owner = dictionary.get("owner")
-    if "owner" not in dictionary:
+    if owner is None:
-        raise ValueError("Invalid patch dictionary: %s" % dictionary)
+        raise ValueError(f"Invalid patch dictionary: {dictionary}")
+    assert isinstance(owner, str)
     pkg_cls = repository.get_pkg_class(owner)
 
     if "url" in dictionary:

lib/spack/spack/repo.py

@@ -25,7 +25,8 @@
 import traceback
 import types
 import uuid
-from typing import Any, Dict, List, Set, Tuple, Union
+import warnings
+from typing import Any, Dict, Generator, List, Optional, Set, Tuple, Type, Union
 
 import llnl.path
 import llnl.util.filesystem as fs
@@ -126,11 +127,35 @@ def exec_module(self, module):
 
 
 class ReposFinder:
-    """MetaPathFinder class that loads a Python module corresponding to a Spack package
+    """MetaPathFinder class that loads a Python module corresponding to a Spack package.
 
-    Return a loader based on the inspection of the current global repository list.
+    Returns a loader based on the inspection of the current repository list.
     """
 
+    def __init__(self):
+        self._repo_init = _path
+        self._repo = None
+
+    @property
+    def current_repository(self):
+        if self._repo is None:
+            self._repo = self._repo_init()
+        return self._repo
+
+    @current_repository.setter
+    def current_repository(self, value):
+        self._repo = value
+
+    @contextlib.contextmanager
+    def switch_repo(self, substitute: "RepoType"):
+        """Switch the current repository list for the duration of the context manager."""
+        old = self.current_repository
+        try:
+            self.current_repository = substitute
+            yield
+        finally:
+            self.current_repository = old
+
     def find_spec(self, fullname, python_path, target=None):
         # "target" is not None only when calling importlib.reload()
         if target is not None:
@@ -149,9 +174,14 @@ def compute_loader(self, fullname):
         # namespaces are added to repo, and package modules are leaves.
         namespace, dot, module_name = fullname.rpartition(".")
 
-        # If it's a module in some repo, or if it is the repo's
+        # If it's a module in some repo, or if it is the repo's namespace, let the repo handle it.
-        # namespace, let the repo handle it.
+        is_repo_path = isinstance(self.current_repository, RepoPath)
-        for repo in PATH.repos:
+        if is_repo_path:
+            repos = self.current_repository.repos
+        else:
+            repos = [self.current_repository]
+
+        for repo in repos:
             # We are using the namespace of the repo and the repo contains the package
             if namespace == repo.full_namespace:
                 # With 2 nested conditionals we can call "repo.real_name" only once
@@ -165,7 +195,7 @@ def compute_loader(self, fullname):
 
         # No repo provides the namespace, but it is a valid prefix of
         # something in the RepoPath.
-        if PATH.by_namespace.is_prefix(fullname):
+        if is_repo_path and self.current_repository.by_namespace.is_prefix(fullname):
             return SpackNamespaceLoader()
 
         return None
@@ -560,7 +590,7 @@ def __init__(
         self,
         package_checker: FastPackageChecker,
         namespace: str,
-        cache: spack.util.file_cache.FileCache,
+        cache: "spack.caches.FileCacheType",
     ):
         self.checker = package_checker
         self.packages_path = self.checker.packages_path
@@ -645,33 +675,39 @@ class RepoPath:
     repository.
 
     Args:
-        repos (list): list Repo objects or paths to put in this RepoPath
+        repos: list Repo objects or paths to put in this RepoPath
+        cache: file cache associated with this repository
+        overrides: dict mapping package name to class attribute overrides for that package
     """
 
-    def __init__(self, *repos, **kwargs):
+    def __init__(
-        cache = kwargs.get("cache", spack.caches.MISC_CACHE)
+        self,
-        self.repos = []
+        *repos: Union[str, "Repo"],
+        cache: "spack.caches.FileCacheType",
+        overrides: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        self.repos: List[Repo] = []
         self.by_namespace = nm.NamespaceTrie()
-
+        self._provider_index: Optional[spack.provider_index.ProviderIndex] = None
-        self._provider_index = None
+        self._patch_index: Optional[spack.patch.PatchCache] = None
-        self._patch_index = None
+        self._tag_index: Optional[spack.tag.TagIndex] = None
-        self._tag_index = None
 
         # Add each repo to this path.
         for repo in repos:
             try:
                 if isinstance(repo, str):
-                    repo = Repo(repo, cache=cache)
+                    repo = Repo(repo, cache=cache, overrides=overrides)
+                repo.finder(self)
                 self.put_last(repo)
             except RepoError as e:
                 tty.warn(
-                    "Failed to initialize repository: '%s'." % repo,
+                    f"Failed to initialize repository: '{repo}'.",
                     e.message,
                     "To remove the bad repository, run this command:",
-                    "    spack repo rm %s" % repo,
+                    f"    spack repo rm {repo}",
                 )
 
-    def put_first(self, repo):
+    def put_first(self, repo: "Repo") -> None:
         """Add repo first in the search path."""
         if isinstance(repo, RepoPath):
             for r in reversed(repo.repos):
@@ -699,50 +735,34 @@ def remove(self, repo):
         if repo in self.repos:
             self.repos.remove(repo)
 
-    def get_repo(self, namespace, default=NOT_PROVIDED):
+    def get_repo(self, namespace: str) -> "Repo":
-        """Get a repository by namespace.
+        """Get a repository by namespace."""
-
-        Arguments:
-
-            namespace:
-
-                Look up this namespace in the RepoPath, and return it if found.
-
-        Optional Arguments:
-
-            default:
-
-                If default is provided, return it when the namespace
-                isn't found.  If not, raise an UnknownNamespaceError.
-        """
         full_namespace = python_package_for_repo(namespace)
         if full_namespace not in self.by_namespace:
-            if default == NOT_PROVIDED:
+            raise UnknownNamespaceError(namespace)
-                raise UnknownNamespaceError(namespace)
-            return default
         return self.by_namespace[full_namespace]
 
-    def first_repo(self):
+    def first_repo(self) -> Optional["Repo"]:
         """Get the first repo in precedence order."""
         return self.repos[0] if self.repos else None
 
     @llnl.util.lang.memoized
-    def _all_package_names_set(self, include_virtuals):
+    def _all_package_names_set(self, include_virtuals) -> Set[str]:
         return {name for repo in self.repos for name in repo.all_package_names(include_virtuals)}
 
     @llnl.util.lang.memoized
-    def _all_package_names(self, include_virtuals):
+    def _all_package_names(self, include_virtuals: bool) -> List[str]:
         """Return all unique package names in all repositories."""
         return sorted(self._all_package_names_set(include_virtuals), key=lambda n: n.lower())
 
-    def all_package_names(self, include_virtuals=False):
+    def all_package_names(self, include_virtuals: bool = False) -> List[str]:
         return self._all_package_names(include_virtuals)
 
-    def package_path(self, name):
+    def package_path(self, name: str) -> str:
         """Get path to package.py file for this repo."""
         return self.repo_for_pkg(name).package_path(name)
 
-    def all_package_paths(self):
+    def all_package_paths(self) -> Generator[str, None, None]:
         for name in self.all_package_names():
             yield self.package_path(name)
 
@@ -758,53 +778,52 @@ def packages_with_tags(self, *tags: str, full: bool = False) -> Set[str]:
             for pkg in repo.packages_with_tags(*tags)
         }
 
-    def all_package_classes(self):
+    def all_package_classes(self) -> Generator[Type["spack.package_base.PackageBase"], None, None]:
         for name in self.all_package_names():
             yield self.get_pkg_class(name)
 
     @property
-    def provider_index(self):
+    def provider_index(self) -> spack.provider_index.ProviderIndex:
         """Merged ProviderIndex from all Repos in the RepoPath."""
         if self._provider_index is None:
             self._provider_index = spack.provider_index.ProviderIndex(repository=self)
             for repo in reversed(self.repos):
                 self._provider_index.merge(repo.provider_index)
-
         return self._provider_index
 
     @property
-    def tag_index(self):
+    def tag_index(self) -> spack.tag.TagIndex:
         """Merged TagIndex from all Repos in the RepoPath."""
         if self._tag_index is None:
             self._tag_index = spack.tag.TagIndex(repository=self)
             for repo in reversed(self.repos):
                 self._tag_index.merge(repo.tag_index)
-
         return self._tag_index
 
     @property
-    def patch_index(self):
+    def patch_index(self) -> spack.patch.PatchCache:
         """Merged PatchIndex from all Repos in the RepoPath."""
         if self._patch_index is None:
             self._patch_index = spack.patch.PatchCache(repository=self)
             for repo in reversed(self.repos):
                 self._patch_index.update(repo.patch_index)
-
         return self._patch_index
 
     @autospec
-    def providers_for(self, vpkg_spec):
+    def providers_for(self, virtual_spec: "spack.spec.Spec") -> List["spack.spec.Spec"]:
         providers = [
             spec
-            for spec in self.provider_index.providers_for(vpkg_spec)
+            for spec in self.provider_index.providers_for(virtual_spec)
             if spec.name in self._all_package_names_set(include_virtuals=False)
         ]
         if not providers:
-            raise UnknownPackageError(vpkg_spec.fullname)
+            raise UnknownPackageError(virtual_spec.fullname)
         return providers
 
     @autospec
-    def extensions_for(self, extendee_spec):
+    def extensions_for(
+        self, extendee_spec: "spack.spec.Spec"
+    ) -> List["spack.package_base.PackageBase"]:
         return [
             pkg_cls(spack.spec.Spec(pkg_cls.name))
             for pkg_cls in self.all_package_classes()
@@ -815,7 +834,7 @@ def last_mtime(self):
         """Time a package file in this repo was last updated."""
         return max(repo.last_mtime() for repo in self.repos)
 
-    def repo_for_pkg(self, spec):
+    def repo_for_pkg(self, spec: Union[str, "spack.spec.Spec"]) -> "Repo":
         """Given a spec, get the repository for its package."""
         # We don't @_autospec this function b/c it's called very frequently
         # and we want to avoid parsing str's into Specs unnecessarily.
@@ -840,17 +859,20 @@ def repo_for_pkg(self, spec):
                 return repo
 
         # If the package isn't in any repo, return the one with
-        # highest precedence.  This is for commands like `spack edit`
+        # highest precedence. This is for commands like `spack edit`
         # that can operate on packages that don't exist yet.
-        return self.first_repo()
+        selected = self.first_repo()
+        if selected is None:
+            raise UnknownPackageError(name)
+        return selected
 
-    def get(self, spec):
+    def get(self, spec: "spack.spec.Spec") -> "spack.package_base.PackageBase":
         """Returns the package associated with the supplied spec."""
         msg = "RepoPath.get can only be called on concrete specs"
         assert isinstance(spec, spack.spec.Spec) and spec.concrete, msg
         return self.repo_for_pkg(spec).get(spec)
 
-    def get_pkg_class(self, pkg_name):
+    def get_pkg_class(self, pkg_name: str) -> Type["spack.package_base.PackageBase"]:
         """Find a class for the spec's package and return the class object."""
         return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name)
 
@@ -863,26 +885,26 @@ def dump_provenance(self, spec, path):
         """
         return self.repo_for_pkg(spec).dump_provenance(spec, path)
 
-    def dirname_for_package_name(self, pkg_name):
+    def dirname_for_package_name(self, pkg_name: str) -> str:
         return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name)
 
-    def filename_for_package_name(self, pkg_name):
+    def filename_for_package_name(self, pkg_name: str) -> str:
         return self.repo_for_pkg(pkg_name).filename_for_package_name(pkg_name)
 
-    def exists(self, pkg_name):
+    def exists(self, pkg_name: str) -> bool:
         """Whether package with the give name exists in the path's repos.
 
         Note that virtual packages do not "exist".
         """
         return any(repo.exists(pkg_name) for repo in self.repos)
 
-    def _have_name(self, pkg_name):
+    def _have_name(self, pkg_name: str) -> bool:
         have_name = pkg_name is not None
         if have_name and not isinstance(pkg_name, str):
-            raise ValueError("is_virtual(): expected package name, got %s" % type(pkg_name))
+            raise ValueError(f"is_virtual(): expected package name, got {type(pkg_name)}")
         return have_name
 
-    def is_virtual(self, pkg_name):
+    def is_virtual(self, pkg_name: str) -> bool:
         """Return True if the package with this name is virtual, False otherwise.
 
         This function use the provider index. If calling from a code block that
@@ -894,7 +916,7 @@ def is_virtual(self, pkg_name):
         have_name = self._have_name(pkg_name)
         return have_name and pkg_name in self.provider_index
 
-    def is_virtual_safe(self, pkg_name):
+    def is_virtual_safe(self, pkg_name: str) -> bool:
         """Return True if the package with this name is virtual, False otherwise.
 
         This function doesn't use the provider index.
@@ -915,18 +937,28 @@ class Repo:
     Each package repository must have a top-level configuration file
     called `repo.yaml`.
 
-    Currently, `repo.yaml` this must define:
+    Currently, `repo.yaml` must define:
 
     `namespace`:
         A Python namespace where the repository's packages should live.
 
+    `subdirectory`:
+        An optional subdirectory name where packages are placed
     """
 
-    def __init__(self, root, cache=None):
+    def __init__(
+        self,
+        root: str,
+        *,
+        cache: "spack.caches.FileCacheType",
+        overrides: Optional[Dict[str, Any]] = None,
+    ) -> None:
         """Instantiate a package repository from a filesystem path.
 
         Args:
             root: the root directory of the repository
+            cache: file cache associated with this repository
+            overrides: dict mapping package name to class attribute overrides for that package
         """
         # Root directory, containing _repo.yaml and package dirs
         # Allow roots to by spack-relative by starting with '$spack'
@@ -939,20 +971,20 @@ def check(condition, msg):
 
         # Validate repository layout.
         self.config_file = os.path.join(self.root, repo_config_name)
-        check(os.path.isfile(self.config_file), "No %s found in '%s'" % (repo_config_name, root))
+        check(os.path.isfile(self.config_file), f"No {repo_config_name} found in '{root}'")
 
         # Read configuration and validate namespace
         config = self._read_config()
         check(
             "namespace" in config,
-            "%s must define a namespace." % os.path.join(root, repo_config_name),
+            f"{os.path.join(root, repo_config_name)} must define a namespace.",
         )
 
         self.namespace = config["namespace"]
         check(
             re.match(r"[a-zA-Z][a-zA-Z0-9_.]+", self.namespace),
-            ("Invalid namespace '%s' in repo '%s'. " % (self.namespace, self.root))
+            f"Invalid namespace '{self.namespace}' in repo '{self.root}'. "
-            + "Namespaces must be valid python identifiers separated by '.'",
+            "Namespaces must be valid python identifiers separated by '.'",
         )
 
         # Set up 'full_namespace' to include the super-namespace
@@ -964,23 +996,26 @@ def check(condition, msg):
         packages_dir = config.get("subdirectory", packages_dir_name)
         self.packages_path = os.path.join(self.root, packages_dir)
         check(
-            os.path.isdir(self.packages_path),
+            os.path.isdir(self.packages_path), f"No directory '{packages_dir}' found in '{root}'"
-            "No directory '%s' found in '%s'" % (packages_dir, root),
         )
 
-        # These are internal cache variables.
+        # Class attribute overrides by package name
-        self._modules = {}
+        self.overrides = overrides or {}
-        self._classes = {}
+
-        self._instances = {}
+        # Optional reference to a RepoPath to influence module import from spack.pkg
+        self._finder: Optional[RepoPath] = None
 
         # Maps that goes from package name to corresponding file stat
-        self._fast_package_checker = None
+        self._fast_package_checker: Optional[FastPackageChecker] = None
 
         # Indexes for this repository, computed lazily
-        self._repo_index = None
+        self._repo_index: Optional[RepoIndex] = None
-        self._cache = cache or spack.caches.MISC_CACHE
+        self._cache = cache
 
-    def real_name(self, import_name):
+    def finder(self, value: RepoPath) -> None:
+        self._finder = value
+
+    def real_name(self, import_name: str) -> Optional[str]:
         """Allow users to import Spack packages using Python identifiers.
 
         A python identifier might map to many different Spack package
@@ -999,18 +1034,21 @@ def real_name(self, import_name):
             return import_name
 
         options = nm.possible_spack_module_names(import_name)
-        options.remove(import_name)
+        try:
+            options.remove(import_name)
+        except ValueError:
+            pass
         for name in options:
             if name in self:
                 return name
         return None
 
-    def is_prefix(self, fullname):
+    def is_prefix(self, fullname: str) -> bool:
         """True if fullname is a prefix of this Repo's namespace."""
         parts = fullname.split(".")
         return self._names[: len(parts)] == parts
 
-    def _read_config(self):
+    def _read_config(self) -> Dict[str, str]:
         """Check for a YAML config file in this db's root directory."""
         try:
             with open(self.config_file) as reponame_file:
@@ -1021,14 +1059,14 @@ def _read_config(self):
                     or "repo" not in yaml_data
                     or not isinstance(yaml_data["repo"], dict)
                 ):
-                    tty.die("Invalid %s in repository %s" % (repo_config_name, self.root))
+                    tty.die(f"Invalid {repo_config_name} in repository {self.root}")
 
                 return yaml_data["repo"]
 
         except IOError:
-            tty.die("Error reading %s when opening %s" % (self.config_file, self.root))
+            tty.die(f"Error reading {self.config_file} when opening {self.root}")
 
-    def get(self, spec):
+    def get(self, spec: "spack.spec.Spec") -> "spack.package_base.PackageBase":
         """Returns the package associated with the supplied spec."""
         msg = "Repo.get can only be called on concrete specs"
         assert isinstance(spec, spack.spec.Spec) and spec.concrete, msg
@@ -1049,16 +1087,13 @@ def get(self, spec):
             # pass these through as their error messages will be fine.
             raise
         except Exception as e:
-            tty.debug(e)
-
             # Make sure other errors in constructors hit the error
             # handler by wrapping them
-            if spack.config.get("config:debug"):
+            tty.debug(e)
-                sys.excepthook(*sys.exc_info())
+            raise FailedConstructorError(spec.fullname, *sys.exc_info()) from e
-            raise FailedConstructorError(spec.fullname, *sys.exc_info())
 
     @autospec
-    def dump_provenance(self, spec, path):
+    def dump_provenance(self, spec: "spack.spec.Spec", path: str) -> None:
         """Dump provenance information for a spec to a particular path.
 
         This dumps the package file and any associated patch files.
@@ -1066,7 +1101,7 @@ def dump_provenance(self, spec, path):
         """
         if spec.namespace and spec.namespace != self.namespace:
             raise UnknownPackageError(
-                "Repository %s does not contain package %s." % (self.namespace, spec.fullname)
+                f"Repository {self.namespace} does not contain package {spec.fullname}."
             )
 
         package_path = self.filename_for_package_name(spec.name)
@@ -1083,17 +1118,13 @@ def dump_provenance(self, spec, path):
                     if os.path.exists(patch.path):
                         fs.install(patch.path, path)
                     else:
-                        tty.warn("Patch file did not exist: %s" % patch.path)
+                        warnings.warn(f"Patch file did not exist: {patch.path}")
 
         # Install the package.py file itself.
         fs.install(self.filename_for_package_name(spec.name), path)
 
-    def purge(self):
-        """Clear entire package instance cache."""
-        self._instances.clear()
-
     @property
-    def index(self):
+    def index(self) -> RepoIndex:
         """Construct the index for this repo lazily."""
         if self._repo_index is None:
             self._repo_index = RepoIndex(self._pkg_checker, self.namespace, cache=self._cache)
@@ -1103,42 +1134,40 @@ def index(self):
         return self._repo_index
 
     @property
-    def provider_index(self):
+    def provider_index(self) -> spack.provider_index.ProviderIndex:
         """A provider index with names *specific* to this repo."""
         return self.index["providers"]
 
     @property
-    def tag_index(self):
+    def tag_index(self) -> spack.tag.TagIndex:
         """Index of tags and which packages they're defined on."""
         return self.index["tags"]
 
     @property
-    def patch_index(self):
+    def patch_index(self) -> spack.patch.PatchCache:
         """Index of patches and packages they're defined on."""
         return self.index["patches"]
 
     @autospec
-    def providers_for(self, vpkg_spec):
+    def providers_for(self, vpkg_spec: "spack.spec.Spec") -> List["spack.spec.Spec"]:
         providers = self.provider_index.providers_for(vpkg_spec)
         if not providers:
             raise UnknownPackageError(vpkg_spec.fullname)
         return providers
 
     @autospec
-    def extensions_for(self, extendee_spec):
+    def extensions_for(
-        return [
+        self, extendee_spec: "spack.spec.Spec"
-            pkg_cls(spack.spec.Spec(pkg_cls.name))
+    ) -> List["spack.package_base.PackageBase"]:
-            for pkg_cls in self.all_package_classes()
+        result = [pkg_cls(spack.spec.Spec(pkg_cls.name)) for pkg_cls in self.all_package_classes()]
-            if pkg_cls(spack.spec.Spec(pkg_cls.name)).extends(extendee_spec)
+        return [x for x in result if x.extends(extendee_spec)]
-        ]
 
-    def dirname_for_package_name(self, pkg_name):
+    def dirname_for_package_name(self, pkg_name: str) -> str:
-        """Get the directory name for a particular package.  This is the
+        """Given a package name, get the directory containing its package.py file."""
-        directory that contains its package.py file."""
         _, unqualified_name = self.partition_package_name(pkg_name)
         return os.path.join(self.packages_path, unqualified_name)
 
-    def filename_for_package_name(self, pkg_name):
+    def filename_for_package_name(self, pkg_name: str) -> str:
         """Get the filename for the module we should load for a particular
         package.  Packages for a Repo live in
         ``$root/<package_name>/package.py``
@@ -1151,23 +1180,23 @@ def filename_for_package_name(self, pkg_name):
         return os.path.join(pkg_dir, package_file_name)
 
     @property
-    def _pkg_checker(self):
+    def _pkg_checker(self) -> FastPackageChecker:
         if self._fast_package_checker is None:
             self._fast_package_checker = FastPackageChecker(self.packages_path)
         return self._fast_package_checker
 
-    def all_package_names(self, include_virtuals=False):
+    def all_package_names(self, include_virtuals: bool = False) -> List[str]:
         """Returns a sorted list of all package names in the Repo."""
         names = sorted(self._pkg_checker.keys())
         if include_virtuals:
             return names
         return [x for x in names if not self.is_virtual(x)]
 
-    def package_path(self, name):
+    def package_path(self, name: str) -> str:
         """Get path to package.py file for this repo."""
         return os.path.join(self.packages_path, name, package_file_name)
 
-    def all_package_paths(self):
+    def all_package_paths(self) -> Generator[str, None, None]:
         for name in self.all_package_names():
             yield self.package_path(name)
 
@@ -1176,7 +1205,7 @@ def packages_with_tags(self, *tags: str) -> Set[str]:
         v.intersection_update(*(self.tag_index[tag.lower()] for tag in tags))
         return v
 
-    def all_package_classes(self):
+    def all_package_classes(self) -> Generator[Type["spack.package_base.PackageBase"], None, None]:
         """Iterator over all package *classes* in the repository.
 
         Use this with care, because loading packages is slow.
@@ -1184,7 +1213,7 @@ def all_package_classes(self):
         for name in self.all_package_names():
             yield self.get_pkg_class(name)
 
-    def exists(self, pkg_name):
+    def exists(self, pkg_name: str) -> bool:
         """Whether a package with the supplied name exists."""
         if pkg_name is None:
             return False
@@ -1201,28 +1230,22 @@ def last_mtime(self):
         """Time a package file in this repo was last updated."""
         return self._pkg_checker.last_mtime()
 
-    def is_virtual(self, pkg_name):
+    def is_virtual(self, pkg_name: str) -> bool:
         """Return True if the package with this name is virtual, False otherwise.
 
         This function use the provider index. If calling from a code block that
         is used to construct the provider index use the ``is_virtual_safe`` function.
-
-        Args:
-            pkg_name (str): name of the package we want to check
         """
         return pkg_name in self.provider_index
 
-    def is_virtual_safe(self, pkg_name):
+    def is_virtual_safe(self, pkg_name: str) -> bool:
         """Return True if the package with this name is virtual, False otherwise.
 
         This function doesn't use the provider index.
-
-        Args:
-            pkg_name (str): name of the package we want to check
         """
         return not self.exists(pkg_name) or self.get_pkg_class(pkg_name).virtual
 
-    def get_pkg_class(self, pkg_name):
+    def get_pkg_class(self, pkg_name: str) -> Type["spack.package_base.PackageBase"]:
         """Get the class for the package out of its module.
 
         First loads (or fetches from cache) a module for the
@@ -1234,7 +1257,8 @@ def get_pkg_class(self, pkg_name):
         fullname = f"{self.full_namespace}.{pkg_name}"
 
         try:
-            module = importlib.import_module(fullname)
+            with REPOS_FINDER.switch_repo(self._finder or self):
+                module = importlib.import_module(fullname)
         except ImportError:
             raise UnknownPackageError(fullname)
         except Exception as e:
@@ -1245,26 +1269,21 @@ def get_pkg_class(self, pkg_name):
         if not inspect.isclass(cls):
             tty.die(f"{pkg_name}.{class_name} is not a class")
 
-        new_cfg_settings = (
+        # Clear any prior changes to class attributes in case the class was loaded from the
-            spack.config.get("packages").get(pkg_name, {}).get("package_attributes", {})
+        # same repo, but with different overrides
-        )
-
         overridden_attrs = getattr(cls, "overridden_attrs", {})
         attrs_exclusively_from_config = getattr(cls, "attrs_exclusively_from_config", [])
-        # Clear any prior changes to class attributes in case the config has
-        # since changed
         for key, val in overridden_attrs.items():
             setattr(cls, key, val)
         for key in attrs_exclusively_from_config:
             delattr(cls, key)
 
-        # Keep track of every class attribute that is overridden by the config:
+        # Keep track of every class attribute that is overridden: if different overrides
-        # if the config changes between calls to this method, we make sure to
+        # dictionaries are used on the same physical repo, we make sure to restore the original
-        # restore the original config values (in case the new config no longer
+        # config values
-        # sets attributes that it used to)
         new_overridden_attrs = {}
         new_attrs_exclusively_from_config = set()
-        for key, val in new_cfg_settings.items():
+        for key, val in self.overrides.get(pkg_name, {}).items():
             if hasattr(cls, key):
                 new_overridden_attrs[key] = getattr(cls, key)
             else:
@@ -1291,13 +1310,13 @@ def partition_package_name(self, pkg_name: str) -> Tuple[str, str]:
 
         return namespace, pkg_name
 
-    def __str__(self):
+    def __str__(self) -> str:
-        return "[Repo '%s' at '%s']" % (self.namespace, self.root)
+        return f"Repo '{self.namespace}' at {self.root}"
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return self.__str__()
 
-    def __contains__(self, pkg_name):
+    def __contains__(self, pkg_name: str) -> bool:
         return self.exists(pkg_name)
 
 
@@ -1373,12 +1392,17 @@ def create_repo(root, namespace=None, subdir=packages_dir_name):
     return full_path, namespace
 
 
+def from_path(path: str) -> "Repo":
+    """Returns a repository from the path passed as input. Injects the global misc cache."""
+    return Repo(path, cache=spack.caches.MISC_CACHE)
+
+
 def create_or_construct(path, namespace=None):
     """Create a repository, or just return a Repo if it already exists."""
     if not os.path.exists(path):
         fs.mkdirp(path)
         create_repo(path, namespace)
-    return Repo(path)
+    return from_path(path)
 
 
 def _path(configuration=None):
@@ -1387,7 +1411,9 @@ def _path(configuration=None):
     return create(configuration=configuration)
 
 
-def create(configuration):
+def create(
+    configuration: Union["spack.config.Configuration", llnl.util.lang.Singleton]
+) -> RepoPath:
     """Create a RepoPath from a configuration object.
 
     Args:
@@ -1396,7 +1422,17 @@ def create(configuration):
     repo_dirs = configuration.get("repos")
     if not repo_dirs:
         raise NoRepoConfiguredError("Spack configuration contains no package repositories.")
-    return RepoPath(*repo_dirs)
+
+    overrides = {}
+    for pkg_name, data in configuration.get("packages").items():
+        if pkg_name == "all":
+            continue
+        value = data.get("package_attributes", {})
+        if not value:
+            continue
+        overrides[pkg_name] = value
+
+    return RepoPath(*repo_dirs, cache=spack.caches.MISC_CACHE, overrides=overrides)
 
 
 #: Singleton repo path instance
@@ -1413,20 +1449,20 @@ def all_package_names(include_virtuals=False):
 
 
 @contextlib.contextmanager
-def use_repositories(*paths_and_repos, **kwargs):
+def use_repositories(
+    *paths_and_repos: Union[str, Repo], override: bool = True
+) -> Generator[RepoPath, None, None]:
     """Use the repositories passed as arguments within the context manager.
 
     Args:
         *paths_and_repos: paths to the repositories to be used, or
             already constructed Repo objects
-        override (bool): if True use only the repositories passed as input,
+        override: if True use only the repositories passed as input,
             if False add them to the top of the list of current repositories.
     Returns:
         Corresponding RepoPath object
     """
     global PATH
-    # TODO (Python 2.7): remove this kwargs on deprecation of Python 2.7 support
-    override = kwargs.get("override", True)
     paths = [getattr(x, "root", x) for x in paths_and_repos]
     scope_name = "use-repo-{}".format(uuid.uuid4())
     repos_key = "repos:" if override else "repos"
@@ -1435,7 +1471,8 @@ def use_repositories(*paths_and_repos, **kwargs):
     )
     PATH, saved = create(configuration=spack.config.CONFIG), PATH
     try:
-        yield PATH
+        with REPOS_FINDER.switch_repo(PATH):  # type: ignore
+            yield PATH
     finally:
         spack.config.CONFIG.remove_scope(scope_name=scope_name)
         PATH = saved
@@ -1535,10 +1572,9 @@ class UnknownNamespaceError(UnknownEntityError):
     """Raised when we encounter an unknown namespace"""
 
     def __init__(self, namespace, name=None):
-        msg, long_msg = "Unknown namespace: {}".format(namespace), None
+        msg, long_msg = f"Unknown namespace: {namespace}", None
         if name == "yaml":
-            long_msg = "Did you mean to specify a filename with './{}.{}'?"
+            long_msg = f"Did you mean to specify a filename with './{namespace}.{name}'?"
-            long_msg = long_msg.format(namespace, name)
         super().__init__(msg, long_msg)
 
 

lib/spack/spack/solver/asp.py

@@ -23,6 +23,7 @@
 
 import llnl.util.lang
 import llnl.util.tty as tty
+from llnl.util.lang import elide_list
 
 import spack
 import spack.binary_distribution
@@ -116,6 +117,8 @@ class Provenance(enum.IntEnum):
     PACKAGE_PY = enum.auto()
     # An installed spec
     INSTALLED = enum.auto()
+    # lower provenance for installed git refs so concretizer prefers StandardVersion installs
+    INSTALLED_GIT_VERSION = enum.auto()
     # A runtime injected from another package (e.g. a compiler)
     RUNTIME = enum.auto()
 
@@ -619,8 +622,9 @@ def _external_config_with_implicit_externals(configuration):
 
 
 class ErrorHandler:
-    def __init__(self, model):
+    def __init__(self, model, input_specs: List[spack.spec.Spec]):
         self.model = model
+        self.input_specs = input_specs
         self.full_model = None
 
     def multiple_values_error(self, attribute, pkg):
@@ -707,12 +711,13 @@ def handle_error(self, msg, *args):
         return msg
 
     def message(self, errors) -> str:
-        messages = [
+        input_specs = ", ".join(elide_list([f"`{s}`" for s in self.input_specs], 5))
-            f"  {idx+1: 2}. {self.handle_error(msg, *args)}"
+        header = f"failed to concretize {input_specs} for the following reasons:"
+        messages = (
+            f"    {idx+1:2}. {self.handle_error(msg, *args)}"
             for idx, (_, msg, args) in enumerate(errors)
-        ]
+        )
-        header = "concretization failed for the following reasons:\n"
+        return "\n".join((header, *messages))
-        return "\n".join([header] + messages)
 
     def raise_if_errors(self):
         initial_error_args = extract_args(self.model, "error")
@@ -748,7 +753,7 @@ def on_model(model):
                 f"unexpected error during concretization [{str(e)}]. "
                 f"Please report a bug at https://github.com/spack/spack/issues"
             )
-            raise spack.error.SpackError(msg)
+            raise spack.error.SpackError(msg) from e
         raise UnsatisfiableSpecError(msg)
 
 
@@ -844,8 +849,6 @@ def solve(self, setup, specs, reuse=None, output=None, control=None, allow_depre
         parent_dir = os.path.dirname(__file__)
         self.control.load(os.path.join(parent_dir, "concretize.lp"))
         self.control.load(os.path.join(parent_dir, "heuristic.lp"))
-        if spack.config.CONFIG.get("concretizer:duplicates:strategy", "none") != "none":
-            self.control.load(os.path.join(parent_dir, "heuristic_separate.lp"))
         self.control.load(os.path.join(parent_dir, "display.lp"))
         if not setup.concretize_everything:
             self.control.load(os.path.join(parent_dir, "when_possible.lp"))
@@ -894,7 +897,7 @@ def on_model(model):
             min_cost, best_model = min(models)
 
             # first check for errors
-            error_handler = ErrorHandler(best_model)
+            error_handler = ErrorHandler(best_model, specs)
             error_handler.raise_if_errors()
 
             # build specs from spec attributes in the model
@@ -1435,16 +1438,14 @@ def condition(
             # caller, we won't emit partial facts.
 
             condition_id = next(self._id_counter)
-            self.gen.fact(fn.pkg_fact(required_spec.name, fn.condition(condition_id)))
-            self.gen.fact(fn.condition_reason(condition_id, msg))
-
             trigger_id = self._get_condition_id(
                 required_spec, cache=self._trigger_cache, body=True, transform=transform_required
             )
+            self.gen.fact(fn.pkg_fact(required_spec.name, fn.condition(condition_id)))
+            self.gen.fact(fn.condition_reason(condition_id, msg))
             self.gen.fact(
                 fn.pkg_fact(required_spec.name, fn.condition_trigger(condition_id, trigger_id))
             )
-
             if not imposed_spec:
                 return condition_id
 
@@ -1693,19 +1694,43 @@ def external_packages(self):
                 spack.spec.parse_with_version_concrete(x["spec"]) for x in externals
             ]
 
-            external_specs = []
+            selected_externals = set()
             if spec_filters:
                 for current_filter in spec_filters:
                     current_filter.factory = lambda: candidate_specs
-                    external_specs.extend(current_filter.selected_specs())
+                    selected_externals.update(current_filter.selected_specs())
-            else:
+
-                external_specs.extend(candidate_specs)
+            # Emit facts for externals specs. Note that "local_idx" is the index of the spec
+            # in packages:<pkg_name>:externals. This means:
+            #
+            # packages:<pkg_name>:externals[local_idx].spec == spec
+            external_versions = []
+            for local_idx, spec in enumerate(candidate_specs):
+                msg = f"{spec.name} available as external when satisfying {spec}"
+
+                if spec_filters and spec not in selected_externals:
+                    continue
+
+                if not spec.versions.concrete:
+                    warnings.warn(f"cannot use the external spec {spec}: needs a concrete version")
+                    continue
+
+                def external_imposition(input_spec, requirements):
+                    return requirements + [
+                        fn.attr("external_conditions_hold", input_spec.name, local_idx)
+                    ]
+
+                try:
+                    self.condition(spec, spec, msg=msg, transform_imposed=external_imposition)
+                except (spack.error.SpecError, RuntimeError) as e:
+                    warnings.warn(f"while setting up external spec {spec}: {e}")
+                    continue
+                external_versions.append((spec.version, local_idx))
+                self.possible_versions[spec.name].add(spec.version)
+                self.gen.newline()
 
             # Order the external versions to prefer more recent versions
             # even if specs in packages.yaml are not ordered that way
-            external_versions = [
-                (x.version, external_id) for external_id, x in enumerate(external_specs)
-            ]
             external_versions = [
                 (v, idx, external_id)
                 for idx, (v, external_id) in enumerate(sorted(external_versions, reverse=True))
@@ -1715,19 +1740,6 @@ def external_packages(self):
                     DeclaredVersion(version=version, idx=idx, origin=Provenance.EXTERNAL)
                 )
 
-            # Declare external conditions with a local index into packages.yaml
-            for local_idx, spec in enumerate(external_specs):
-                msg = "%s available as external when satisfying %s" % (spec.name, spec)
-
-                def external_imposition(input_spec, requirements):
-                    return requirements + [
-                        fn.attr("external_conditions_hold", input_spec.name, local_idx)
-                    ]
-
-                self.condition(spec, spec, msg=msg, transform_imposed=external_imposition)
-                self.possible_versions[spec.name].add(spec.version)
-                self.gen.newline()
-
             self.trigger_rules()
             self.effect_rules()
 
@@ -1839,6 +1851,8 @@ def _spec_clauses(
 
         if spec.name:
             clauses.append(f.node(spec.name) if not spec.virtual else f.virtual_node(spec.name))
+        if spec.namespace:
+            clauses.append(f.namespace(spec.name, spec.namespace))
 
         clauses.extend(self.spec_versions(spec))
 
@@ -1880,11 +1894,8 @@ def _spec_clauses(
                             )
 
                 clauses.append(f.variant_value(spec.name, vname, value))
-
                 if variant.propagate:
-                    clauses.append(
+                    clauses.append(f.propagate(spec.name, fn.variant_value(vname, value)))
-                        f.variant_propagation_candidate(spec.name, vname, value, spec.name)
-                    )
 
                 # Tell the concretizer that this is a possible value for the
                 # variant, to account for things like int/str values where we
@@ -1917,9 +1928,12 @@ def _spec_clauses(
         for flag_type, flags in spec.compiler_flags.items():
             for flag in flags:
                 clauses.append(f.node_flag(spec.name, flag_type, flag))
-                clauses.append(f.node_flag_source(spec.name, flag_type, spec.name))
                 if not spec.concrete and flag.propagate is True:
-                    clauses.append(f.node_flag_propagate(spec.name, flag_type))
+                    clauses.append(
+                        f.propagate(
+                            spec.name, fn.node_flag(flag_type, flag), fn.edge_types("link", "run")
+                        )
+                    )
 
         # dependencies
         if spec.concrete:
@@ -2072,7 +2086,7 @@ def define_ad_hoc_versions_from_specs(
             # best possible, so they're guaranteed to be used preferentially.
             version = s.versions.concrete
 
-            if version is None or any(v == version for v in self.possible_versions[s.name]):
+            if version is None or (any((v == version) for v in self.possible_versions[s.name])):
                 continue
 
             if require_checksum and not _is_checksummed_git_version(version):
@@ -2386,9 +2400,16 @@ def concrete_specs(self):
             # - Add OS to possible OS's
             for dep in spec.traverse():
                 self.possible_versions[dep.name].add(dep.version)
-                self.declared_versions[dep.name].append(
+                if isinstance(dep.version, vn.GitVersion):
-                    DeclaredVersion(version=dep.version, idx=0, origin=Provenance.INSTALLED)
+                    self.declared_versions[dep.name].append(
-                )
+                        DeclaredVersion(
+                            version=dep.version, idx=0, origin=Provenance.INSTALLED_GIT_VERSION
+                        )
+                    )
+                else:
+                    self.declared_versions[dep.name].append(
+                        DeclaredVersion(version=dep.version, idx=0, origin=Provenance.INSTALLED)
+                    )
                 self.possible_oses.add(dep.os)
 
     def define_concrete_input_specs(self, specs, possible):
@@ -2729,6 +2750,7 @@ class _Head:
     """ASP functions used to express spec clauses in the HEAD of a rule"""
 
     node = fn.attr("node")
+    namespace = fn.attr("namespace_set")
     virtual_node = fn.attr("virtual_node")
     node_platform = fn.attr("node_platform_set")
     node_os = fn.attr("node_os_set")
@@ -2737,15 +2759,14 @@ class _Head:
     node_compiler = fn.attr("node_compiler_set")
     node_compiler_version = fn.attr("node_compiler_version_set")
     node_flag = fn.attr("node_flag_set")
-    node_flag_source = fn.attr("node_flag_source")
+    propagate = fn.attr("propagate")
-    node_flag_propagate = fn.attr("node_flag_propagate")
-    variant_propagation_candidate = fn.attr("variant_propagation_candidate")
 
 
 class _Body:
     """ASP functions used to express spec clauses in the BODY of a rule"""
 
     node = fn.attr("node")
+    namespace = fn.attr("namespace")
     virtual_node = fn.attr("virtual_node")
     node_platform = fn.attr("node_platform")
     node_os = fn.attr("node_os")
@@ -2754,9 +2775,7 @@ class _Body:
     node_compiler = fn.attr("node_compiler")
     node_compiler_version = fn.attr("node_compiler_version")
     node_flag = fn.attr("node_flag")
-    node_flag_source = fn.attr("node_flag_source")
+    propagate = fn.attr("propagate")
-    node_flag_propagate = fn.attr("node_flag_propagate")
-    variant_propagation_candidate = fn.attr("variant_propagation_candidate")
 
 
 class ProblemInstanceBuilder:
@@ -3230,6 +3249,39 @@ def requires(self, impose: str, *, when: str):
         self.runtime_conditions.add((imposed_spec, when_spec))
         self.reset()
 
+    def propagate(self, constraint_str: str, *, when: str):
+        msg = "the 'propagate' method can be called only with pkg('*')"
+        assert self.current_package == "*", msg
+
+        when_spec = spack.spec.Spec(when)
+        assert when_spec.name is None, "only anonymous when specs are accepted"
+
+        placeholder = "XXX"
+        node_variable = "node(ID, Package)"
+        when_spec.name = placeholder
+
+        body_clauses = self._setup.spec_clauses(when_spec, body=True)
+        body_str = (
+            f"  {f',{os.linesep}  '.join(str(x) for x in body_clauses)},\n"
+            f"  not external({node_variable}),\n"
+            f"  not runtime(Package)"
+        ).replace(f'"{placeholder}"', f"{node_variable}")
+
+        constraint_spec = spack.spec.Spec(constraint_str)
+        assert constraint_spec.name is None, "only anonymous constraint specs are accepted"
+
+        constraint_spec.name = placeholder
+        constraint_clauses = self._setup.spec_clauses(constraint_spec, body=False)
+        for clause in constraint_clauses:
+            if clause.args[0] == "node_compiler_version_satisfies":
+                self._setup.compiler_version_constraints.add(constraint_spec.compiler)
+                args = f'"{constraint_spec.compiler.name}", "{constraint_spec.compiler.versions}"'
+                head_str = f"propagate({node_variable}, node_compiler_version_satisfies({args}))"
+                rule = f"{head_str} :-\n{body_str}.\n\n"
+                self.rules.append(rule)
+
+        self.reset()
+
     def consume_facts(self):
         """Consume the facts collected by this object, and emits rules and
         facts for the runtimes.
@@ -3309,6 +3361,8 @@ def hash(self, node, h):
     def node(self, node):
         if node not in self._specs:
             self._specs[node] = spack.spec.Spec(node.pkg)
+            for flag_type in spack.spec.FlagMap.valid_compiler_flags():
+                self._specs[node].compiler_flags[flag_type] = []
 
     def _arch(self, node):
         arch = self._specs[node].architecture
@@ -3361,9 +3415,6 @@ def node_flag(self, node, flag_type, flag):
     def node_flag_source(self, node, flag_type, source):
         self._flag_sources[(node, flag_type)].add(source)
 
-    def no_flags(self, node, flag_type):
-        self._specs[node].compiler_flags[flag_type] = []
-
     def external_spec_selected(self, node, idx):
         """This means that the external spec and index idx has been selected for this package."""
         packages_yaml = _external_config_with_implicit_externals(spack.config.CONFIG)
@@ -3456,7 +3507,7 @@ def reorder_flags(self):
                 ordered_compiler_flags = list(llnl.util.lang.dedupe(from_compiler + from_sources))
                 compiler_flags = spec.compiler_flags.get(flag_type, [])
 
-                msg = "%s does not equal %s" % (set(compiler_flags), set(ordered_compiler_flags))
+                msg = f"{set(compiler_flags)} does not equal {set(ordered_compiler_flags)}"
                 assert set(compiler_flags) == set(ordered_compiler_flags), msg
 
                 spec.compiler_flags.update({flag_type: ordered_compiler_flags})
@@ -3526,9 +3577,8 @@ def build_specs(self, function_tuples):
                 # do not bother calling actions on it except for node_flag_source,
                 # since node_flag_source is tracking information not in the spec itself
                 spec = self._specs.get(args[0])
-                if spec and spec.concrete:
+                if spec and spec.concrete and name != "node_flag_source":
-                    if name != "node_flag_source":
+                    continue
-                        continue
 
             action(*args)
 

lib/spack/spack/solver/concretize.lp

@@ -18,39 +18,79 @@
 { attr("virtual_node", node(0..X-1, Package)) } :- max_dupes(Package, X), virtual(Package).
 
 % Integrity constraints on DAG nodes
-:- attr("root", PackageNode), not attr("node", PackageNode).
+:- attr("root", PackageNode),
-:- attr("version", PackageNode, _), not attr("node", PackageNode), not attr("virtual_node", PackageNode).
+   not attr("node", PackageNode),
-:- attr("node_version_satisfies", PackageNode, _), not attr("node", PackageNode), not attr("virtual_node", PackageNode).
+   internal_error("Every root must be a node").
-:- attr("hash", PackageNode, _), not attr("node", PackageNode).
+:- attr("version", PackageNode, _),
-:- attr("node_platform", PackageNode, _), not attr("node", PackageNode).
+   not attr("node", PackageNode),
-:- attr("node_os", PackageNode, _), not attr("node", PackageNode).
+   not attr("virtual_node", PackageNode),
-:- attr("node_target", PackageNode, _), not attr("node", PackageNode).
+   internal_error("Only nodes and virtual_nodes can have versions").
-:- attr("node_compiler_version", PackageNode, _, _), not attr("node", PackageNode).
+:- attr("node_version_satisfies", PackageNode, _),
-:- attr("variant_value", PackageNode, _, _), not attr("node", PackageNode).
+   not attr("node", PackageNode),
-:- attr("node_flag_compiler_default", PackageNode), not attr("node", PackageNode).
+   not attr("virtual_node", PackageNode),
-:- attr("node_flag", PackageNode, _, _), not attr("node", PackageNode).
+   internal_error("Only nodes and virtual_nodes can have version satisfaction").
-:- attr("no_flags", PackageNode, _), not attr("node", PackageNode).
+:- attr("hash", PackageNode, _),
-:- attr("external_spec_selected", PackageNode, _), not attr("node", PackageNode).
+   not attr("node", PackageNode),
-:- attr("depends_on", ParentNode, _, _), not attr("node", ParentNode).
+   internal_error("Only nodes can have hashes").
-:- attr("depends_on", _, ChildNode, _), not attr("node", ChildNode).
+:- attr("node_platform", PackageNode, _),
-:- attr("node_flag_source", ParentNode, _, _), not attr("node", ParentNode).
+   not attr("node", PackageNode),
-:- attr("node_flag_source", _, _, ChildNode), not attr("node", ChildNode).
+   internal_error("Only nodes can have platforms").
-:- attr("virtual_node", VirtualNode), not provider(_, VirtualNode), internal_error("virtual node with no provider").
+:- attr("node_os", PackageNode, _), not attr("node", PackageNode),
-:- provider(_, VirtualNode), not attr("virtual_node", VirtualNode), internal_error("provider with no virtual node").
+   internal_error("Only nodes can have node_os").
-:- provider(PackageNode, _), not attr("node", PackageNode), internal_error("provider with no real node").
+:- attr("node_target", PackageNode, _), not attr("node", PackageNode),
+   internal_error("Only nodes can have node_target").
+:- attr("node_compiler_version", PackageNode, _, _), not attr("node", PackageNode),
+   internal_error("Only nodes can have node_compiler_version").
+:- attr("variant_value", PackageNode, _, _), not attr("node", PackageNode),
+   internal_error("variant_value true for a non-node").
+:- attr("node_flag_compiler_default", PackageNode), not attr("node", PackageNode),
+   internal_error("node_flag_compiler_default true for non-node").
+:- attr("node_flag", PackageNode, _, _), not attr("node", PackageNode),
+   internal_error("node_flag assigned for non-node").
+:- attr("external_spec_selected", PackageNode, _), not attr("node", PackageNode),
+   internal_error("external_spec_selected for non-node").
+:- attr("depends_on", ParentNode, _, _), not attr("node", ParentNode),
+   internal_error("non-node depends on something").
+:- attr("depends_on", _, ChildNode, _), not attr("node", ChildNode),
+   internal_error("something depends_on a non-node").
+:- attr("node_flag_source", Node, _, _), not attr("node", Node),
+   internal_error("node_flag_source assigned for a non-node").
+:- attr("node_flag_source", _, _, SourceNode), not attr("node", SourceNode),
+   internal_error("node_flag_source assigned with a non-node source").
+:- attr("virtual_node", VirtualNode), not provider(_, VirtualNode),
+   internal_error("virtual node with no provider").
+:- provider(_, VirtualNode), not attr("virtual_node", VirtualNode),
+   internal_error("provider with no virtual node").
+:- provider(PackageNode, _), not attr("node", PackageNode),
+   internal_error("provider with no real node").
 
-:- attr("root", node(ID, PackageNode)), ID > min_dupe_id, internal_error("root with a non-minimal duplicate ID").
+:- attr("root", node(ID, PackageNode)), ID > min_dupe_id,
+   internal_error("root with a non-minimal duplicate ID").
 
 % Nodes in the "root" unification set cannot depend on non-root nodes if the dependency is "link" or "run"
 :- attr("depends_on", node(min_dupe_id, Package), node(ID, _), "link"), ID != min_dupe_id, unification_set("root", node(min_dupe_id, Package)), internal_error("link dependency out of the root unification set").
 :- attr("depends_on", node(min_dupe_id, Package), node(ID, _), "run"),  ID != min_dupe_id, unification_set("root", node(min_dupe_id, Package)), internal_error("run dependency out of the root unification set").
 
-% Namespaces are statically assigned by a package fact
+% Namespaces are statically assigned by a package fact if not otherwise set
-attr("namespace", node(ID, Package), Namespace) :- attr("node", node(ID, Package)), pkg_fact(Package, namespace(Namespace)).
+error(100, "{0} does not have a namespace", Package) :- attr("node", node(ID, Package)),
+   not attr("namespace", node(ID, Package), _),
+   internal_error("A node must have a namespace").
+error(100, "{0} cannot come from both {1} and {2} namespaces", Package, NS1, NS2) :- attr("node", node(ID, Package)),
+   attr("namespace", node(ID, Package), NS1),
+   attr("namespace", node(ID, Package), NS2),
+   NS1 != NS2,
+   internal_error("A node cannot have two namespaces").
+
+attr("namespace", node(ID, Package), Namespace) :- attr("namespace_set", node(ID, Package), Namespace).
+attr("namespace", node(ID, Package), Namespace)
+  :- attr("node", node(ID, Package)),
+     not attr("namespace_set", node(ID, Package), _),
+     pkg_fact(Package, namespace(Namespace)).
 
 % Rules on "unification sets", i.e. on sets of nodes allowing a single configuration of any given package
 unify(SetID, PackageName) :- unification_set(SetID, node(_, PackageName)).
-:- 2 { unification_set(SetID, node(_, PackageName)) }, unify(SetID, PackageName).
+:- 2 { unification_set(SetID, node(_, PackageName)) }, unify(SetID, PackageName),
+   internal_error("Cannot have multiple unification sets IDs for one set").
 
 unification_set("root", PackageNode) :- attr("root", PackageNode).
 unification_set(SetID, ChildNode) :- attr("depends_on", ParentNode, ChildNode, Type), Type != "build", unification_set(SetID, ParentNode).
@@ -76,7 +116,8 @@ unification_set(SetID, VirtualNode)
 % as a build dependency.
 %
 % We'll need to relax the rule before we get to actual cross-compilation
-:- depends_on(ParentNode, node(X, Dependency)), depends_on(ParentNode, node(Y, Dependency)), X < Y.
+:- depends_on(ParentNode, node(X, Dependency)), depends_on(ParentNode, node(Y, Dependency)), X < Y,
+   internal_error("Cannot split link/build deptypes for a single edge (yet)").
 
 
 #defined multiple_unification_sets/1.
@@ -132,7 +173,8 @@ mentioned_in_literal(Root, Mentioned) :- mentioned_in_literal(TriggerID, Root, M
 condition_set(node(min_dupe_id, Root), node(min_dupe_id, Root)) :-  mentioned_in_literal(Root, Root).
 
 1 { condition_set(node(min_dupe_id, Root), node(0..Y-1, Mentioned)) : max_dupes(Mentioned, Y) } 1 :-
-  mentioned_in_literal(Root, Mentioned), Mentioned != Root.
+  mentioned_in_literal(Root, Mentioned), Mentioned != Root,
+  internal_error("must have exactly one condition_set for literals").
 
 % Discriminate between "roots" that have been explicitly requested, and roots that are deduced from "virtual roots"
 explicitly_requested_root(node(min_dupe_id, Package)) :-
@@ -152,7 +194,8 @@ associated_with_root(RootNode, ChildNode) :-
 :- attr("root", RootNode),
    condition_set(RootNode, node(X, Package)),
    not virtual(Package),
-   not associated_with_root(RootNode, node(X, Package)).
+   not associated_with_root(RootNode, node(X, Package)),
+   internal_error("nodes in root condition set must be associated with root").
 
 #defined concretize_everything/0.
 #defined literal/1.
@@ -256,6 +299,7 @@ possible_version_weight(node(ID, Package), Weight)
 :- attr("version", node(ID, Package), Version),
    version_weight(node(ID, Package), Weight),
    not pkg_fact(Package, version_declared(Version, Weight, "installed")),
+   not pkg_fact(Package, version_declared(Version, Weight, "installed_git_version")),
    not build(node(ID, Package)),
    internal_error("Build version weight used for reused package").
 
@@ -385,8 +429,10 @@ imposed_nodes(ConditionID, PackageNode, node(X, A1))
      condition_set(PackageNode, node(X, A1)),
      attr("hash", PackageNode, ConditionID).
 
-:- imposed_packages(ID, A1), impose(ID, PackageNode), not condition_set(PackageNode, node(_, A1)).
+:- imposed_packages(ID, A1), impose(ID, PackageNode), not condition_set(PackageNode, node(_, A1)),
-:- imposed_packages(ID, A1), impose(ID, PackageNode), not imposed_nodes(ID, PackageNode, node(_, A1)).
+   internal_error("Imposing constraint outside of condition set").
+:- imposed_packages(ID, A1), impose(ID, PackageNode), not imposed_nodes(ID, PackageNode, node(_, A1)),
+   internal_error("Imposing constraint outside of imposed_nodes").
 
 % Conditions that hold impose may impose constraints on other specs
 attr(Name, node(X, A1))             :- impose(ID, PackageNode), imposed_constraint(ID, Name, A1),             imposed_nodes(ID, PackageNode, node(X, A1)).
@@ -416,7 +462,8 @@ provider(ProviderNode, VirtualNode) :- attr("provider_set", ProviderNode, Virtua
 % satisfy the dependency.
 1 { attr("depends_on", node(X, A1), node(0..Y-1, A2), A3) : max_dupes(A2, Y) } 1
   :- impose(ID, node(X, A1)),
-     imposed_constraint(ID, "depends_on", A1, A2, A3).
+     imposed_constraint(ID, "depends_on", A1, A2, A3),
+     internal_error("Build deps must land in exactly one duplicate").
 
 % Reconstruct virtual dependencies for reused specs
 attr("virtual_on_edge", node(X, A1), node(Y, A2), Virtual)
@@ -611,25 +658,18 @@ do_not_impose(EffectID, node(X, Package))
 % Virtual dependency weights
 %-----------------------------------------------------------------------------
 
-% A provider may have different possible weights depending on whether it's an external
+% A provider has different possible weights depending on its preference. This rule ensures that
-% or not, or on preferences expressed in packages.yaml etc. This rule ensures that
 % we select the weight, among the possible ones, that minimizes the overall objective function.
 1 { provider_weight(DependencyNode, VirtualNode, Weight) :
     possible_provider_weight(DependencyNode, VirtualNode, Weight, _) } 1
  :- provider(DependencyNode, VirtualNode), internal_error("Package provider weights must be unique").
 
-% A provider that is an external can use a weight of 0
+% Any configured provider has a weight based on index in the preference list
-possible_provider_weight(DependencyNode, VirtualNode, 0, "external")
-  :- provider(DependencyNode, VirtualNode),
-     external(DependencyNode).
-
-% A provider mentioned in the default configuration can use a weight
-% according to its priority in the list of providers
 possible_provider_weight(node(ProviderID, Provider), node(VirtualID, Virtual), Weight, "default")
   :- provider(node(ProviderID, Provider), node(VirtualID, Virtual)),
      default_provider_preference(Virtual, Provider, Weight).
 
-% Any provider can use 100 as a weight, which is very high and discourage its use
+% Any non-configured provider has a default weight of 100
 possible_provider_weight(node(ProviderID, Provider), VirtualNode, 100, "fallback")
   :- provider(node(ProviderID, Provider), VirtualNode).
 
@@ -811,37 +851,6 @@ node_has_variant(node(ID, Package), Variant) :-
   pkg_fact(Package, variant(Variant)),
   attr("node", node(ID, Package)).
 
-% Variant propagation is forwarded to dependencies
-attr("variant_propagation_candidate", PackageNode, Variant, Value, Source) :-
-  attr("node", PackageNode),
-  depends_on(ParentNode, PackageNode),
-  attr("variant_value", node(_, Source), Variant, Value),
-  attr("variant_propagation_candidate", ParentNode, Variant, _, Source).
-
-% If the node is a candidate, and it has the variant and value,
-% then those variant and value should be propagated
-attr("variant_propagate", node(ID, Package), Variant, Value, Source) :-
-  attr("variant_propagation_candidate", node(ID, Package), Variant, Value, Source),
-  node_has_variant(node(ID, Package), Variant),
-  pkg_fact(Package, variant_possible_value(Variant, Value)),
-  not attr("variant_set", node(ID, Package), Variant).
-
-% Propagate the value, if there is the corresponding attribute
-attr("variant_value", PackageNode, Variant, Value) :- attr("variant_propagate", PackageNode, Variant, Value, _).
-
-% If a variant is propagated, we cannot have extraneous values (this is for multi valued variants)
-variant_is_propagated(PackageNode, Variant) :- attr("variant_propagate", PackageNode, Variant, _, _).
-:- variant_is_propagated(PackageNode, Variant),
-   attr("variant_value", PackageNode, Variant, Value),
-   not attr("variant_propagate", PackageNode, Variant, Value, _).
-
-% Cannot receive different values from different sources on the same variant
-error(100, "{0} and {1} cannot both propagate variant '{2}' to package {3} with values '{4}' and '{5}'", Source1, Source2, Variant, Package, Value1, Value2) :-
-  attr("variant_propagate", node(X, Package), Variant, Value1, Source1),
-  attr("variant_propagate", node(X, Package), Variant, Value2, Source2),
-  node_has_variant(node(X, Package), Variant),
-  Value1 < Value2, Source1 < Source2.
-
 % a variant cannot be set if it is not a variant on the package
 error(100, "Cannot set variant '{0}' for package '{1}' because the variant condition cannot be satisfied for the given spec", Variant, Package)
   :- attr("variant_set", node(X, Package), Variant),
@@ -919,7 +928,7 @@ variant_not_default(node(ID, Package), Variant, Value)
     % variants set explicitly on the CLI don't count as non-default
     not attr("variant_set", node(ID, Package), Variant, Value),
     % variant values forced by propagation don't count as non-default
-    not attr("variant_propagate", node(ID, Package), Variant, Value, _),
+    not propagate(node(ID, Package), variant_value(Variant, Value)),
     % variants set on externals that we could use don't count as non-default
     % this makes spack prefer to use an external over rebuilding with the
     % default configuration
@@ -932,7 +941,7 @@ variant_default_not_used(node(ID, Package), Variant, Value)
   :- variant_default_value(Package, Variant, Value),
      node_has_variant(node(ID, Package), Variant),
      not attr("variant_value", node(ID, Package), Variant, Value),
-     not attr("variant_propagate", node(ID, Package), Variant, _, _),
+     not propagate(node(ID, Package), variant_value(Variant, _)),
      attr("node", node(ID, Package)).
 
 % The variant is set in an external spec
@@ -989,6 +998,101 @@ pkg_fact(Package, variant_single_value("dev_path"))
 #defined variant_default_value/3.
 #defined variant_default_value_from_packages_yaml/3.
 
+%-----------------------------------------------------------------------------
+% Propagation semantics
+%-----------------------------------------------------------------------------
+
+% Propagation roots have a corresponding attr("propagate", ...)
+propagate(RootNode, PropagatedAttribute) :- attr("propagate", RootNode, PropagatedAttribute).
+propagate(RootNode, PropagatedAttribute, EdgeTypes) :- attr("propagate", RootNode, PropagatedAttribute, EdgeTypes).
+
+
+% Propagate an attribute along edges to child nodes
+propagate(ChildNode, PropagatedAttribute) :-
+  propagate(ParentNode, PropagatedAttribute),
+  depends_on(ParentNode, ChildNode).
+
+propagate(ChildNode, PropagatedAttribute, edge_types(DepType1, DepType2)) :-
+  propagate(ParentNode, PropagatedAttribute, edge_types(DepType1, DepType2)),
+  depends_on(ParentNode, ChildNode),
+  1 { attr("depends_on", ParentNode, ChildNode, DepType1);  attr("depends_on", ParentNode, ChildNode, DepType2) }.
+
+%-----------------------------------------------------------------------------
+% Activation of propagated values
+%-----------------------------------------------------------------------------
+
+%----
+% Variants
+%----
+
+% If a variant is propagated, and can be accepted, set its value
+attr("variant_value", node(ID, Package), Variant, Value) :-
+  propagate(node(ID, Package), variant_value(Variant, Value)),
+  node_has_variant(node(ID, Package), Variant),
+  pkg_fact(Package, variant_possible_value(Variant, Value)),
+  not attr("variant_set", node(ID, Package), Variant).
+
+% If a variant is propagated, we cannot have extraneous values
+variant_is_propagated(PackageNode, Variant) :-
+  attr("variant_value", PackageNode, Variant, Value),
+  propagate(PackageNode, variant_value(Variant, Value)),
+  not attr("variant_set", PackageNode, Variant).
+
+:- variant_is_propagated(PackageNode, Variant),
+   attr("variant_value", PackageNode, Variant, Value),
+   not propagate(PackageNode, variant_value(Variant, Value)).
+
+%----
+% Flags
+%----
+
+% A propagated flag implies:
+% 1. The same flag type is not set on this node
+% 2. This node has the same compiler as the propagation source
+
+propagated_flag(node(PackageID, Package), node_flag(FlagType, Flag), SourceNode) :-
+  propagate(node(PackageID, Package), node_flag(FlagType, Flag), _),
+  not attr("node_flag_set", node(PackageID, Package), FlagType, _),
+  % Same compiler as propagation source
+  node_compiler(node(PackageID, Package), CompilerID),
+  node_compiler(SourceNode, CompilerID),
+  attr("propagate", SourceNode, node_flag(FlagType, Flag), _),
+  node(PackageID, Package) != SourceNode,
+  not runtime(Package).
+
+attr("node_flag", PackageNode, FlagType, Flag) :- propagated_flag(PackageNode, node_flag(FlagType, Flag), _).
+attr("node_flag_source", PackageNode, FlagType, SourceNode) :- propagated_flag(PackageNode, node_flag(FlagType, _), SourceNode).
+
+% Cannot propagate the same flag from two distinct sources
+error(100, "{0} and {1} cannot both propagate compiler flags '{2}' to {3}", Source1, Source2, Package, FlagType) :-
+  propagated_flag(node(ID, Package), node_flag(FlagType, _), node(_, Source1)),
+  propagated_flag(node(ID, Package), node_flag(FlagType, _), node(_, Source2)),
+  Source1 < Source2.
+
+%----
+% Compiler constraints
+%----
+
+attr("node_compiler_version_satisfies", node(ID, Package), Compiler, Version) :-
+  propagate(node(ID, Package), node_compiler_version_satisfies(Compiler, Version)),
+  node_compiler(node(ID, Package), CompilerID),
+  compiler_name(CompilerID, Compiler),
+  not runtime(Package),
+  not external(Package).
+
+%-----------------------------------------------------------------------------
+% Runtimes
+%-----------------------------------------------------------------------------
+
+% Check whether the DAG has any built package
+has_built_packages() :- build(X), not external(X).
+
+% If we build packages, the runtime nodes must use an available compiler
+1 { node_compiler(PackageNode, CompilerID) : build(PackageNode), not external(PackageNode) } :-
+  has_built_packages(),
+  runtime(RuntimePackage),
+  node_compiler(node(_, RuntimePackage), CompilerID).
+
 %-----------------------------------------------------------------------------
 % Platform semantics
 %-----------------------------------------------------------------------------
@@ -1090,10 +1194,18 @@ attr("node_target", PackageNode, Target)
  :- attr("node", PackageNode), attr("node_target_set", PackageNode, Target).
 
 % each node has the weight of its assigned target
-node_target_weight(node(ID, Package), Weight)
+target_weight(Target, 0)
- :- attr("node", node(ID, Package)),
+ :- attr("node", PackageNode),
-    attr("node_target", node(ID, Package), Target),
+    attr("node_target", PackageNode, Target),
-    target_weight(Target, Weight).
+    attr("node_target_set", PackageNode, Target).
+
+node_target_weight(PackageNode, MinWeight)
+ :- attr("node", PackageNode),
+    attr("node_target", PackageNode, Target),
+    target(Target),
+    MinWeight = #min { Weight : target_weight(Target, Weight) }.
+
+:- attr("node_target", PackageNode, Target), not node_target_weight(PackageNode, _).
 
 % compatibility rules for targets among nodes
 node_target_match(ParentNode, DependencyNode)
@@ -1155,12 +1267,12 @@ error(10, "No valid compiler for {0} satisfies '%{1}'", Package, Compiler)
 
 % If the compiler of a node must satisfy a constraint, then its version
 % must be chosen among the ones that satisfy said constraint
-error(100, "No valid version for '{0}' compiler '{1}' satisfies '@{2}'", Package, Compiler, Constraint)
+error(100, "Package {0} cannot satisfy '%{1}@{2}'", Package, Compiler, Constraint)
   :- attr("node", node(X, Package)),
      attr("node_compiler_version_satisfies", node(X, Package), Compiler, Constraint),
-		 not compiler_version_satisfies(Compiler, Constraint, _).
+	 not compiler_version_satisfies(Compiler, Constraint, _).
 
-error(100, "No valid version for '{0}' compiler '{1}' satisfies '@{2}'", Package, Compiler, Constraint)
+error(100, "Package {0} cannot satisfy '%{1}@{2}'", Package, Compiler, Constraint)
   :- attr("node", node(X, Package)),
      attr("node_compiler_version_satisfies", node(X, Package), Compiler, Constraint),
      not compiler_version_satisfies(Compiler, Constraint, ID),
@@ -1241,45 +1353,9 @@ error(100, "Compiler {1}@{2} requested for {0} cannot be found. Set install_miss
 % Compiler flags
 %-----------------------------------------------------------------------------
 
-% propagate flags when compiler match
-can_inherit_flags(PackageNode, DependencyNode, FlagType)
-  :- same_compiler(PackageNode, DependencyNode),
-     not attr("node_flag_set", DependencyNode, FlagType, _),
-     flag_type(FlagType).
-
-same_compiler(PackageNode, DependencyNode)
-  :- depends_on(PackageNode, DependencyNode),
-     node_compiler(PackageNode, CompilerID),
-     node_compiler(DependencyNode, CompilerID),
-     compiler_id(CompilerID).
-
-node_flag_inherited(DependencyNode, FlagType, Flag)
-  :- attr("node_flag_set", PackageNode, FlagType, Flag),
-     can_inherit_flags(PackageNode, DependencyNode, FlagType),
-     attr("node_flag_propagate", PackageNode, FlagType).
-
-% Ensure propagation
-:- node_flag_inherited(PackageNode, FlagType, Flag),
-   can_inherit_flags(PackageNode, DependencyNode, FlagType),
-   attr("node_flag_propagate", PackageNode, FlagType).
-
-error(100, "{0} and {1} cannot both propagate compiler flags '{2}' to {3}", Source1, Source2, Package, FlagType) :-
-  depends_on(Source1, Package),
-  depends_on(Source2, Package),
-  attr("node_flag_propagate", Source1, FlagType),
-  attr("node_flag_propagate", Source2, FlagType),
-  can_inherit_flags(Source1, Package, FlagType),
-  can_inherit_flags(Source2, Package, FlagType),
-  Source1 < Source2.
-
 % remember where flags came from
-attr("node_flag_source", PackageNode, FlagType, PackageNode)
+attr("node_flag_source", PackageNode, FlagType, PackageNode) :- attr("node_flag_set", PackageNode, FlagType, _).
-  :- attr("node_flag_set", PackageNode, FlagType, _).
+attr("node_flag_source", PackageNode, FlagType, PackageNode) :- attr("node_flag", PackageNode, FlagType, _), attr("hash", PackageNode, _).
-
-attr("node_flag_source", DependencyNode, FlagType, Q)
-  :- attr("node_flag_source", PackageNode, FlagType, Q),
-     node_flag_inherited(DependencyNode, FlagType, _),
-     attr("node_flag_propagate", PackageNode, FlagType).
 
 % compiler flags from compilers.yaml are put on nodes if compiler matches
 attr("node_flag", PackageNode, FlagType, Flag)
@@ -1299,15 +1375,8 @@ attr("node_flag_compiler_default", PackageNode)
      compiler_name(CompilerID, CompilerName),
      compiler_version(CompilerID, Version).
 
-% if a flag is set to something or inherited, it's included
+% Flag set to something
 attr("node_flag", PackageNode, FlagType, Flag) :- attr("node_flag_set", PackageNode, FlagType, Flag).
-attr("node_flag", PackageNode, FlagType, Flag) :- node_flag_inherited(PackageNode, FlagType, Flag).
-
-% if no node flags are set for a type, there are no flags.
-attr("no_flags", PackageNode, FlagType)
-  :- not attr("node_flag", PackageNode, FlagType, _),
-     attr("node", PackageNode),
-     flag_type(FlagType).
 
 #defined compiler_flag/3.
 
@@ -1496,7 +1565,7 @@ opt_criterion(45, "preferred providers (non-roots)").
 }.
 
 % Try to minimize the number of compiler mismatches in the DAG.
-opt_criterion(40, "compiler mismatches that are not from CLI").
+opt_criterion(40, "compiler mismatches that are not required").
 #minimize{ 0@240: #true }.
 #minimize{ 0@40: #true }.
 #minimize{
@@ -1506,7 +1575,7 @@ opt_criterion(40, "compiler mismatches that are not from CLI").
       not runtime(Dependency)
 }.
 
-opt_criterion(39, "compiler mismatches that are not from CLI").
+opt_criterion(39, "compiler mismatches that are required").
 #minimize{ 0@239: #true }.
 #minimize{ 0@39: #true }.
 #minimize{

lib/spack/spack/solver/heuristic.lp

@@ -4,21 +4,35 @@
 % SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 %=============================================================================
-% Heuristic to speed-up solves (node with ID 0)
+% Heuristic to speed-up solves
 %=============================================================================
 
+% No duplicates by default (most of them will be true)
+#heuristic attr("node",         node(PackageID, Package)). [100, init]
+#heuristic attr("node",         node(PackageID, Package)). [  2, factor]
+#heuristic attr("virtual_node", node(VirtualID, Virtual)). [100, init]
+#heuristic attr("node",         node(1..X-1, Package)) : max_dupes(Package, X), not virtual(Package), X > 1. [-1, sign]
+#heuristic attr("virtual_node", node(1..X-1, Package)) : max_dupes(Package, X), virtual(Package)    , X > 1. [-1, sign]
 
-%-----------------
+% Pick preferred version
-% Domain heuristic
+#heuristic attr("version", node(PackageID, Package), Version) : pkg_fact(Package, version_declared(Version, Weight)), attr("node", node(PackageID, Package)).             [40, init]
-%-----------------
+#heuristic version_weight(node(PackageID, Package), 0)        : pkg_fact(Package, version_declared(Version, 0     )), attr("node", node(PackageID, Package)).             [ 1, sign]
+#heuristic attr("version", node(PackageID, Package), Version) : pkg_fact(Package, version_declared(Version, 0     )), attr("node", node(PackageID, Package)).             [ 1, sign]
+#heuristic attr("version", node(PackageID, Package), Version) : pkg_fact(Package, version_declared(Version, Weight)), attr("node", node(PackageID, Package)), Weight > 0. [-1, sign]
 
-% Root node
+% Use default variants
-#heuristic attr("version", node(0, Package), Version) : pkg_fact(Package, version_declared(Version, 0)), attr("root", node(0, Package)). [35, true]
+#heuristic attr("variant_value", node(PackageID, Package), Variant, Value) : variant_default_value(Package, Variant, Value),     attr("node", node(PackageID, Package)). [40, true]
-#heuristic version_weight(node(0, Package), 0) : pkg_fact(Package, version_declared(Version, 0)), attr("root", node(0, Package)). [35, true]
+#heuristic attr("variant_value", node(PackageID, Package), Variant, Value) : not variant_default_value(Package, Variant, Value), attr("node", node(PackageID, Package)). [40, false]
-#heuristic attr("variant_value", node(0, Package), Variant, Value) : variant_default_value(Package, Variant, Value), attr("root", node(0, Package)). [35, true]
-#heuristic attr("node_target", node(0, Package), Target) : target_weight(Target, 0), attr("root", node(0, Package)). [35, true]
-#heuristic node_target_weight(node(0, Package), 0) : attr("root", node(0, Package)). [35, true]
-#heuristic node_compiler(node(0, Package), CompilerID) : compiler_weight(ID, 0), compiler_id(ID), attr("root", node(0, Package)). [35, true]
 
-% Providers
+% Use default operating system and platform
-#heuristic attr("node", node(0, Package)) : default_provider_preference(Virtual, Package, 0), possible_in_link_run(Package). [30, true]
+#heuristic attr("node_os", node(PackageID, Package), OS) : os(OS, 0), attr("root", node(PackageID, Package)). [40, true]
+#heuristic attr("node_platform", node(PackageID, Package), Platform) : allowed_platform(Platform), attr("root", node(PackageID, Package)). [40, true]
+
+% Use default targets
+#heuristic attr("node_target", node(PackageID, Package), Target) : target_weight(Target, Weight), attr("node", node(PackageID, Package)). [30, init]
+#heuristic attr("node_target", node(PackageID, Package), Target) : target_weight(Target, Weight), attr("node", node(PackageID, Package)). [ 2, factor]
+#heuristic attr("node_target", node(PackageID, Package), Target) : target_weight(Target,      0), attr("node", node(PackageID, Package)). [ 1, sign]
+#heuristic attr("node_target", node(PackageID, Package), Target) : target_weight(Target, Weight), attr("node", node(PackageID, Package)), Weight > 0. [-1, sign]
+
+% Use the default compilers
+#heuristic node_compiler(node(PackageID, Package), ID) : compiler_weight(ID, 0), compiler_id(ID), attr("node", node(PackageID, Package)). [30, init]

lib/spack/spack/solver/heuristic_separate.lp

@@ -1,24 +0,0 @@
-% Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
-% Spack Project Developers. See the top-level COPYRIGHT file for details.
-%
-% SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-%=============================================================================
-% Heuristic to speed-up solves (node with ID > 0)
-%=============================================================================
-
-% node(ID, _)
-#heuristic attr("version", node(ID, Package), Version) : pkg_fact(Package, version_declared(Version, 0)), attr("node", node(ID, Package)), ID > 0. [25-5*ID, true]
-#heuristic version_weight(node(ID, Package), 0) : pkg_fact(Package, version_declared(Version, 0)), attr("node", node(ID, Package)), ID > 0. [25-5*ID, true]
-#heuristic attr("variant_value", node(ID, Package), Variant, Value) : variant_default_value(Package, Variant, Value), attr("node", node(ID, Package)), ID > 0. [25-5*ID, true]
-#heuristic attr("node_target", node(ID, Package), Target) : pkg_fact(Package, target_weight(Target, 0)), attr("node", node(ID, Package)), ID > 0. [25-5*ID, true]
-#heuristic node_target_weight(node(ID, Package), 0) : attr("node", node(ID, Package)), ID > 0. [25-5*ID, true]
-#heuristic node_compiler(node(ID, Package), CompilerID) : compiler_weight(CompilerID, 0), compiler_id(CompilerID), attr("node", node(ID, Package)), ID > 0. [25-5*ID, true]
-
-% node(ID, _), split build dependencies
-#heuristic attr("version", node(ID, Package), Version) : pkg_fact(Package, version_declared(Version, 0)), attr("node", node(ID, Package)), multiple_unification_sets(Package), ID > 0. [25, true]
-#heuristic version_weight(node(ID, Package), 0) : pkg_fact(Package, version_declared(Version, 0)), attr("node", node(ID, Package)), multiple_unification_sets(Package), ID > 0. [25, true]
-#heuristic attr("variant_value", node(ID, Package), Variant, Value) : variant_default_value(Package, Variant, Value), attr("node", node(ID, Package)), multiple_unification_sets(Package), ID > 0. [25, true]
-#heuristic attr("node_target", node(ID, Package), Target) : pkg_fact(Package, target_weight(Target, 0)), attr("node", node(ID, Package)), multiple_unification_sets(Package), ID > 0. [25, true]
-#heuristic node_target_weight(node(ID, Package), 0) : attr("node", node(ID, Package)), multiple_unification_sets(Package), ID > 0. [25, true]
-#heuristic node_compiler(node(ID, Package), CompilerID) : compiler_weight(CompilerID, 0), compiler_id(CompilerID), attr("node", node(ID, Package)), multiple_unification_sets(Package), ID > 0. [25, true]

lib/spack/spack/solver/libc_compatibility.lp

@@ -18,9 +18,6 @@ error(100, "Cannot reuse {0} since we cannot determine libc compatibility", Reus
      ReusedPackage != LibcPackage,
      not attr("compatible_libc", node(R, ReusedPackage), LibcPackage, LibcVersion).
 
-% Check whether the DAG has any built package
-has_built_packages() :- build(X), not external(X).
-
 % A libc is needed in the DAG
 :- has_built_packages(), not provider(_, node(0, "libc")).
 

lib/spack/spack/solver/os_compatibility.lp

@@ -12,6 +12,7 @@
 %=============================================================================
 
 % macOS
+os_compatible("sequoia", "sonoma").
 os_compatible("sonoma", "ventura").
 os_compatible("ventura", "monterey").
 os_compatible("monterey", "bigsur").

lib/spack/spack/spec.py

@@ -99,7 +99,7 @@
     "CompilerSpec",
     "Spec",
     "SpecParseError",
-    "ArchitecturePropagationError",
+    "UnsupportedPropagationError",
     "DuplicateDependencyError",
     "DuplicateCompilerSpecError",
     "UnsupportedCompilerError",
@@ -129,7 +129,7 @@
     r"|"  # or
     # OPTION 2: an actual format string
     r"{"  # non-escaped open brace {
-    r"([%@/]|arch=)?"  # optional sigil (to print sigil in color)
+    r"([%@/]|[\w ][\w -]*=)?"  # optional sigil (or identifier or space) to print sigil in color
     r"(?:\^([^}\.]+)\.)?"  # optional ^depname. (to get attr from dependency)
     # after the sigil or depname, we can have a hash expression or another attribute
     r"(?:"  # one of
@@ -163,14 +163,14 @@
 DEFAULT_FORMAT = (
     "{name}{@versions}"
     "{%compiler.name}{@compiler.versions}{compiler_flags}"
-    "{variants}{arch=architecture}{/abstract_hash}"
+    "{variants}{ namespace=namespace_if_anonymous}{ arch=architecture}{/abstract_hash}"
 )
 
 #: Display format, which eliminates extra `@=` in the output, for readability.
 DISPLAY_FORMAT = (
     "{name}{@version}"
     "{%compiler.name}{@compiler.version}{compiler_flags}"
-    "{variants}{arch=architecture}{/abstract_hash}"
+    "{variants}{ namespace=namespace_if_anonymous}{ arch=architecture}{/abstract_hash}"
 )
 
 #: Regular expression to pull spec contents out of clearsigned signature
@@ -1287,6 +1287,104 @@ def copy(self, *args, **kwargs):
         return self.wrapped_obj.copy(*args, **kwargs)
 
 
+def tree(
+    specs: List["spack.spec.Spec"],
+    *,
+    color: Optional[bool] = None,
+    depth: bool = False,
+    hashes: bool = False,
+    hashlen: Optional[int] = None,
+    cover: str = "nodes",
+    indent: int = 0,
+    format: str = DEFAULT_FORMAT,
+    deptypes: Union[Tuple[str, ...], str] = "all",
+    show_types: bool = False,
+    depth_first: bool = False,
+    recurse_dependencies: bool = True,
+    status_fn: Optional[Callable[["Spec"], InstallStatus]] = None,
+    prefix: Optional[Callable[["Spec"], str]] = None,
+    key=id,
+) -> str:
+    """Prints out specs and their dependencies, tree-formatted with indentation.
+
+    Status function may either output a boolean or an InstallStatus
+
+    Args:
+        color: if True, always colorize the tree. If False, don't colorize the tree. If None,
+            use the default from llnl.tty.color
+        depth: print the depth from the root
+        hashes: if True, print the hash of each node
+        hashlen: length of the hash to be printed
+        cover: either "nodes" or "edges"
+        indent: extra indentation for the tree being printed
+        format: format to be used to print each node
+        deptypes: dependency types to be represented in the tree
+        show_types: if True, show the (merged) dependency type of a node
+        depth_first: if True, traverse the DAG depth first when representing it as a tree
+        recurse_dependencies: if True, recurse on dependencies
+        status_fn: optional callable that takes a node as an argument and return its
+            installation status
+        prefix: optional callable that takes a node as an argument and return its
+            installation prefix
+    """
+    out = ""
+
+    if color is None:
+        color = clr.get_color_when()
+
+    # reduce deptypes over all in-edges when covering nodes
+    if show_types and cover == "nodes":
+        deptype_lookup: Dict[str, dt.DepFlag] = collections.defaultdict(dt.DepFlag)
+        for edge in traverse.traverse_edges(specs, cover="edges", deptype=deptypes, root=False):
+            deptype_lookup[edge.spec.dag_hash()] |= edge.depflag
+
+    for d, dep_spec in traverse.traverse_tree(
+        sorted(specs), cover=cover, deptype=deptypes, depth_first=depth_first, key=key
+    ):
+        node = dep_spec.spec
+
+        if prefix is not None:
+            out += prefix(node)
+        out += " " * indent
+
+        if depth:
+            out += "%-4d" % d
+
+        if status_fn:
+            status = status_fn(node)
+            if status in list(InstallStatus):
+                out += clr.colorize(status.value, color=color)
+            elif status:
+                out += clr.colorize("@g{[+]}  ", color=color)
+            else:
+                out += clr.colorize("@r{[-]}  ", color=color)
+
+        if hashes:
+            out += clr.colorize("@K{%s}  ", color=color) % node.dag_hash(hashlen)
+
+        if show_types:
+            if cover == "nodes":
+                depflag = deptype_lookup[dep_spec.spec.dag_hash()]
+            else:
+                # when covering edges or paths, we show dependency
+                # types only for the edge through which we visited
+                depflag = dep_spec.depflag
+
+            type_chars = dt.flag_to_chars(depflag)
+            out += "[%s]  " % type_chars
+
+        out += "    " * d
+        if d > 0:
+            out += "^"
+        out += node.format(format, color=color) + "\n"
+
+        # Check if we wanted just the first line
+        if not recurse_dependencies:
+            break
+
+    return out
+
+
 @lang.lazy_lexicographic_ordering(set_hash=False)
 class Spec:
     #: Cache for spec's prefix, computed lazily in the corresponding property
@@ -1542,19 +1640,9 @@ def _add_flag(self, name, value, propagate):
         Known flags currently include "arch"
         """
 
-        # If the == syntax is used to propagate the spec architecture
+        if propagate and name in spack.directives.reserved_names:
-        # This is an error
+            raise UnsupportedPropagationError(
-        architecture_names = [
+                f"Propagation with '==' is not supported for '{name}'."
-            "arch",
-            "architecture",
-            "platform",
-            "os",
-            "operating_system",
-            "target",
-        ]
-        if propagate and name in architecture_names:
-            raise ArchitecturePropagationError(
-                "Unable to propagate the architecture failed." " Use a '=' instead."
             )
 
         valid_flags = FlagMap.valid_compiler_flags()
@@ -1568,6 +1656,8 @@ def _add_flag(self, name, value, propagate):
             self._set_architecture(os=value)
         elif name == "target":
             self._set_architecture(target=value)
+        elif name == "namespace":
+            self.namespace = value
         elif name in valid_flags:
             assert self.compiler_flags is not None
             flags_and_propagation = spack.compiler.tokenize_flags(value, propagate)
@@ -1587,9 +1677,7 @@ def _set_architecture(self, **kwargs):
         """Called by the parser to set the architecture."""
         arch_attrs = ["platform", "os", "target"]
         if self.architecture and self.architecture.concrete:
-            raise DuplicateArchitectureError(
+            raise DuplicateArchitectureError("Spec cannot have two architectures.")
-                "Spec for '%s' cannot have two architectures." % self.name
-            )
 
         if not self.architecture:
             new_vals = tuple(kwargs.get(arg, None) for arg in arch_attrs)
@@ -1598,10 +1686,7 @@ def _set_architecture(self, **kwargs):
             new_attrvals = [(a, v) for a, v in kwargs.items() if a in arch_attrs]
             for new_attr, new_value in new_attrvals:
                 if getattr(self.architecture, new_attr):
-                    raise DuplicateArchitectureError(
+                    raise DuplicateArchitectureError(f"Cannot specify '{new_attr}' twice")
-                        "Spec for '%s' cannot have two '%s' specified "
-                        "for its architecture" % (self.name, new_attr)
-                    )
                 else:
                     setattr(self.architecture, new_attr, new_value)
 
@@ -1796,14 +1881,14 @@ def short_spec(self):
         """Returns a version of the spec with the dependencies hashed
         instead of completely enumerated."""
         spec_format = "{name}{@version}{%compiler.name}{@compiler.version}"
-        spec_format += "{variants}{arch=architecture}{/hash:7}"
+        spec_format += "{variants}{ arch=architecture}{/hash:7}"
         return self.format(spec_format)
 
     @property
     def cshort_spec(self):
         """Returns an auto-colorized version of ``self.short_spec``."""
         spec_format = "{name}{@version}{%compiler.name}{@compiler.version}"
-        spec_format += "{variants}{arch=architecture}{/hash:7}"
+        spec_format += "{variants}{ arch=architecture}{/hash:7}"
         return self.cformat(spec_format)
 
     @property
@@ -4162,29 +4247,21 @@ def __getitem__(self, name: str):
             csv = query_parameters.pop().strip()
             query_parameters = re.split(r"\s*,\s*", csv)
 
-        # In some cases a package appears multiple times in the same DAG for *distinct*
-        # specs. For example, a build-type dependency may itself depend on a package
-        # the current spec depends on, but their specs may differ. Therefore we iterate
-        # in an order here that prioritizes the build, test and runtime dependencies;
-        # only when we don't find the package do we consider the full DAG.
         order = lambda: itertools.chain(
-            self.traverse(deptype="link"),
+            self.traverse_edges(deptype=dt.LINK, order="breadth", cover="edges"),
-            self.dependencies(deptype=dt.BUILD | dt.RUN | dt.TEST),
+            self.edges_to_dependencies(depflag=dt.BUILD | dt.RUN | dt.TEST),
-            self.traverse(),  # fall back to a full search
+            self.traverse_edges(deptype=dt.ALL, order="breadth", cover="edges"),
         )
 
+        # Consider runtime dependencies and direct build/test deps before transitive dependencies,
+        # and prefer matches closest to the root.
         try:
             child: Spec = next(
-                itertools.chain(
+                e.spec
-                    # Regular specs
+                for e in itertools.chain(
-                    (x for x in order() if x.name == name),
+                    (e for e in order() if e.spec.name == name or name in e.virtuals),
-                    (
+                    # for historical reasons
-                        x
+                    (e for e in order() if e.spec.concrete and e.spec.package.provides(name)),
-                        for x in order()
-                        if (not x.virtual)
-                        and any(name in edge.virtuals for edge in x.edges_from_dependents())
-                    ),
-                    (x for x in order() if (not x.virtual) and x.package.provides(name)),
                 )
             )
         except StopIteration:
@@ -4296,14 +4373,19 @@ def deps():
 
         yield deps
 
-    def format(self, format_string: str = DEFAULT_FORMAT, color: Optional[bool] = False) -> str:
+    @property
-        r"""Prints out particular pieces of a spec, depending on what is
+    def namespace_if_anonymous(self):
-        in the format string.
+        return self.namespace if not self.name else None
 
-        Using the ``{attribute}`` syntax, any field of the spec can be
+    def format(self, format_string: str = DEFAULT_FORMAT, color: Optional[bool] = False) -> str:
-        selected.  Those attributes can be recursive. For example,
+        r"""Prints out attributes of a spec according to a format string.
-        ``s.format({compiler.version})`` will print the version of the
+
-        compiler.
+        Using an ``{attribute}`` format specifier, any field of the spec can be
+        selected. Those attributes can be recursive. For example,
+        ``s.format({compiler.version})`` will print the version of the compiler.
+
+        If the attribute in a format specifier evaluates to ``None``, then the format
+        specifier will evaluate to the empty string, ``""``.
 
         Commonly used attributes of the Spec for format strings include::
 
@@ -4319,6 +4401,7 @@ def format(self, format_string: str = DEFAULT_FORMAT, color: Optional[bool] = Fa
             architecture.os
             architecture.target
             prefix
+            namespace
 
         Some additional special-case properties can be added::
 
@@ -4327,40 +4410,51 @@ def format(self, format_string: str = DEFAULT_FORMAT, color: Optional[bool] = Fa
             spack_install The spack install directory
 
         The ``^`` sigil can be used to access dependencies by name.
-        ``s.format({^mpi.name})`` will print the name of the MPI
+        ``s.format({^mpi.name})`` will print the name of the MPI implementation in the
-        implementation in the spec.
+        spec.
 
-        The ``@``, ``%``, ``arch=``, and ``/`` sigils
+        The ``@``, ``%``, and ``/`` sigils can be used to include the sigil with the
-        can be used to include the sigil with the printed
+        printed string. These sigils may only be used with the appropriate attributes,
-        string. These sigils may only be used with the appropriate
+        listed below::
-        attributes, listed below::
 
             @        ``{@version}``, ``{@compiler.version}``
             %        ``{%compiler}``, ``{%compiler.name}``
-            arch=    ``{arch=architecture}``
             /        ``{/hash}``, ``{/hash:7}``, etc
 
-        The ``@`` sigil may also be used for any other property named
+        The ``@`` sigil may also be used for any other property named ``version``.
-        ``version``. Sigils printed with the attribute string are only
+        Sigils printed with the attribute string are only printed if the attribute
-        printed if the attribute string is non-empty, and are colored
+        string is non-empty, and are colored according to the color of the attribute.
-        according to the color of the attribute.
 
-        Sigils are not used for printing variants. Variants listed by
+        Variants listed by name naturally print with their sigil. For example,
-        name naturally print with their sigil. For example,
+        ``spec.format('{variants.debug}')`` prints either ``+debug`` or ``~debug``
-        ``spec.format('{variants.debug}')`` would print either
+        depending on the name of the variant. Non-boolean variants print as
-        ``+debug`` or ``~debug`` depending on the name of the
+        ``name=value``. To print variant names or values independently, use
-        variant. Non-boolean variants print as ``name=value``. To
-        print variant names or values independently, use
         ``spec.format('{variants.<name>.name}')`` or
         ``spec.format('{variants.<name>.value}')``.
 
-        Spec format strings use ``\`` as the escape character. Use
+        There are a few attributes on specs that can be specified as key-value pairs
-        ``\{`` and ``\}`` for literal braces, and ``\\`` for the
+        that are *not* variants, e.g.: ``os``, ``arch``, ``architecture``, ``target``,
-        literal ``\`` character.
+        ``namespace``, etc. You can format these with an optional ``key=`` prefix, e.g.
+        ``{namespace=namespace}`` or ``{arch=architecture}``, etc. The ``key=`` prefix
+        will be colorized along with the value.
+
+        When formatting specs, key-value pairs are separated from preceding parts of the
+        spec by whitespace. To avoid printing extra whitespace when the formatted
+        attribute is not set, you can add whitespace to the key *inside* the braces of
+        the format string, e.g.:
+
+            { namespace=namespace}
+
+        This evaluates to `` namespace=builtin`` if ``namespace`` is set to ``builtin``,
+        and to ``""`` if ``namespace`` is ``None``.
+
+        Spec format strings use ``\`` as the escape character. Use ``\{`` and ``\}`` for
+        literal braces, and ``\\`` for the literal ``\`` character.
 
         Args:
             format_string: string containing the format to be expanded
             color: True for colorized result; False for no color; None for auto color.
+
         """
         ensure_modern_format_string(format_string)
 
@@ -4414,10 +4508,6 @@ def format_attribute(match_object: Match) -> str:
                 raise SpecFormatSigilError(sig, "compilers", attribute)
             elif sig == "/" and attribute != "abstract_hash":
                 raise SpecFormatSigilError(sig, "DAG hashes", attribute)
-            elif sig == "arch=":
-                if attribute not in ("architecture", "arch"):
-                    raise SpecFormatSigilError(sig, "the architecture", attribute)
-                sig = " arch="  # include space as separator
 
             # Iterate over components using getattr to get next element
             for idx, part in enumerate(parts):
@@ -4462,15 +4552,19 @@ def format_attribute(match_object: Match) -> str:
 
             # Set color codes for various attributes
             color = None
-            if "variants" in parts:
+            if "architecture" in parts:
-                color = VARIANT_COLOR
-            elif "architecture" in parts:
                 color = ARCHITECTURE_COLOR
+            elif "variants" in parts or sig.endswith("="):
+                color = VARIANT_COLOR
             elif "compiler" in parts or "compiler_flags" in parts:
                 color = COMPILER_COLOR
             elif "version" in parts or "versions" in parts:
                 color = VERSION_COLOR
 
+            # return empty string if the value of the attribute is None.
+            if current is None:
+                return ""
+
             # return colored output
             return safe_color(sig, str(current), color)
 
@@ -4560,7 +4654,7 @@ def colored_str(self):
         spec_str = " ^".join(root_str + sorted_dependencies)
         return spec_str.strip()
 
-    def install_status(self):
+    def install_status(self) -> InstallStatus:
         """Helper for tree to print DB install status."""
         if not self.concrete:
             return InstallStatus.absent
@@ -4604,13 +4698,14 @@ def tree(
         recurse_dependencies: bool = True,
         status_fn: Optional[Callable[["Spec"], InstallStatus]] = None,
         prefix: Optional[Callable[["Spec"], str]] = None,
+        key=id,
     ) -> str:
-        """Prints out this spec and its dependencies, tree-formatted
+        """Prints out this spec and its dependencies, tree-formatted with indentation.
-        with indentation.
 
-        Status function may either output a boolean or an InstallStatus
+        See multi-spec ``spack.spec.tree()`` function for details.
 
         Args:
+            specs: List of specs to format.
             color: if True, always colorize the tree. If False, don't colorize the tree. If None,
                 use the default from llnl.tty.color
             depth: print the depth from the root
@@ -4628,60 +4723,23 @@ def tree(
             prefix: optional callable that takes a node as an argument and return its
                 installation prefix
         """
-        out = ""
+        return tree(
-
+            [self],
-        if color is None:
+            color=color,
-            color = clr.get_color_when()
+            depth=depth,
-
+            hashes=hashes,
-        for d, dep_spec in traverse.traverse_tree(
+            hashlen=hashlen,
-            [self], cover=cover, deptype=deptypes, depth_first=depth_first
+            cover=cover,
-        ):
+            indent=indent,
-            node = dep_spec.spec
+            format=format,
-
+            deptypes=deptypes,
-            if prefix is not None:
+            show_types=show_types,
-                out += prefix(node)
+            depth_first=depth_first,
-            out += " " * indent
+            recurse_dependencies=recurse_dependencies,
-
+            status_fn=status_fn,
-            if depth:
+            prefix=prefix,
-                out += "%-4d" % d
+            key=key,
-
+        )
-            if status_fn:
-                status = status_fn(node)
-                if status in list(InstallStatus):
-                    out += clr.colorize(status.value, color=color)
-                elif status:
-                    out += clr.colorize("@g{[+]}  ", color=color)
-                else:
-                    out += clr.colorize("@r{[-]}  ", color=color)
-
-            if hashes:
-                out += clr.colorize("@K{%s}  ", color=color) % node.dag_hash(hashlen)
-
-            if show_types:
-                if cover == "nodes":
-                    # when only covering nodes, we merge dependency types
-                    # from all dependents before showing them.
-                    depflag = 0
-                    for ds in node.edges_from_dependents():
-                        depflag |= ds.depflag
-                else:
-                    # when covering edges or paths, we show dependency
-                    # types only for the edge through which we visited
-                    depflag = dep_spec.depflag
-
-                type_chars = dt.flag_to_chars(depflag)
-                out += "[%s]  " % type_chars
-
-            out += "    " * d
-            if d > 0:
-                out += "^"
-            out += node.format(format, color=color) + "\n"
-
-            # Check if we wanted just the first line
-            if not recurse_dependencies:
-                break
-
-        return out
 
     def __repr__(self):
         return str(self)
@@ -5336,10 +5394,8 @@ def long_message(self):
         )
 
 
-class ArchitecturePropagationError(spack.error.SpecError):
+class UnsupportedPropagationError(spack.error.SpecError):
-    """Raised when the double equal symbols are used to assign
+    """Raised when propagation (==) is used with reserved variant names."""
-    the spec's architecture.
-    """
 
 
 class DuplicateDependencyError(spack.error.SpecError):
@@ -5469,7 +5525,7 @@ def __init__(self, spec):
 class AmbiguousHashError(spack.error.SpecError):
     def __init__(self, msg, *specs):
         spec_fmt = "{namespace}.{name}{@version}{%compiler}{compiler_flags}"
-        spec_fmt += "{variants}{arch=architecture}{/hash:7}"
+        spec_fmt += "{variants}{ arch=architecture}{/hash:7}"
         specs_str = "\n  " + "\n  ".join(spec.format(spec_fmt) for spec in specs)
         super().__init__(msg + specs_str)
 

lib/spack/spack/spec_list.py

@@ -212,10 +212,7 @@ def _expand_matrix_constraints(matrix_config):
     results = []
     for combo in itertools.product(*expanded_rows):
         # Construct a combined spec to test against excludes
-        flat_combo = [constraint for constraint_list in combo for constraint in constraint_list]
+        flat_combo = [Spec(constraint) for constraints in combo for constraint in constraints]
-
-        # Resolve abstract hashes so we can exclude by their concrete properties
-        flat_combo = [Spec(x).lookup_hash() for x in flat_combo]
 
         test_spec = flat_combo[0].copy()
         for constraint in flat_combo[1:]:
@@ -231,7 +228,9 @@ def _expand_matrix_constraints(matrix_config):
             spack.variant.substitute_abstract_variants(test_spec)
         except spack.variant.UnknownVariantError:
             pass
-        if any(test_spec.satisfies(x) for x in excludes):
+
+        # Resolve abstract hashes for exclusion criteria
+        if any(test_spec.lookup_hash().satisfies(x) for x in excludes):
             continue
 
         if sigil:

lib/spack/spack/stage.py

@@ -346,8 +346,6 @@ class Stage(LockableStagingDir):
     similar, and are intended to persist for only one run of spack.
     """
 
-    #: Most staging is managed by Spack. DIYStage is one exception.
-    needs_fetching = True
     requires_patch_success = True
 
     def __init__(
@@ -772,8 +770,6 @@ def __init__(self):
                 "cache_mirror",
                 "steal_source",
                 "disable_mirrors",
-                "needs_fetching",
-                "requires_patch_success",
             ]
         )
 
@@ -812,6 +808,10 @@ def path(self):
     def archive_file(self):
         return self[0].archive_file
 
+    @property
+    def requires_patch_success(self):
+        return self[0].requires_patch_success
+
     @property
     def keep(self):
         return self[0].keep
@@ -822,64 +822,7 @@ def keep(self, value):
             item.keep = value
 
 
-class DIYStage:
-    """
-    Simple class that allows any directory to be a spack stage.  Consequently,
-    it does not expect or require that the source path adhere to the standard
-    directory naming convention.
-    """
-
-    needs_fetching = False
-    requires_patch_success = False
-
-    def __init__(self, path):
-        if path is None:
-            raise ValueError("Cannot construct DIYStage without a path.")
-        elif not os.path.isdir(path):
-            raise StagePathError("The stage path directory does not exist:", path)
-
-        self.archive_file = None
-        self.path = path
-        self.source_path = path
-        self.created = True
-
-    # DIY stages do nothing as context managers.
-    def __enter__(self):
-        pass
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        pass
-
-    def fetch(self, *args, **kwargs):
-        tty.debug("No need to fetch for DIY.")
-
-    def check(self):
-        tty.debug("No checksum needed for DIY.")
-
-    def expand_archive(self):
-        tty.debug("Using source directory: {0}".format(self.source_path))
-
-    @property
-    def expanded(self):
-        """Returns True since the source_path must exist."""
-        return True
-
-    def restage(self):
-        raise RestageError("Cannot restage a DIY stage.")
-
-    def create(self):
-        self.created = True
-
-    def destroy(self):
-        # No need to destroy DIY stage.
-        pass
-
-    def cache_local(self):
-        tty.debug("Sources for DIY stages are not cached")
-
-
 class DevelopStage(LockableStagingDir):
-    needs_fetching = False
     requires_patch_success = False
 
     def __init__(self, name, dev_path, reference_link):

lib/spack/spack/store.py

@@ -371,7 +371,6 @@ def use_store(
         data.update(extra_data)
 
     # Swap the store with the one just constructed and return it
-    ensure_singleton_created()
     spack.config.CONFIG.push_scope(
         spack.config.InternalConfigScope(name=scope_name, data={"config": {"install_tree": data}})
     )

lib/spack/spack/subprocess_context.py

@@ -79,9 +79,11 @@ def restore(self):
         self.test_state.restore()
         spack.main.spack_working_dir = self.spack_working_dir
         env = pickle.load(self.serialized_env) if _SERIALIZE else self.env
-        pkg = pickle.load(self.serialized_pkg) if _SERIALIZE else self.pkg
         if env:
             spack.environment.activate(env)
+        # Order of operation is important, since the package might be retrieved
+        # from a repo defined within the environment configuration
+        pkg = pickle.load(self.serialized_pkg) if _SERIALIZE else self.pkg
         return pkg
 
 

lib/spack/spack/test/architecture.py

@@ -213,7 +213,9 @@ def test_satisfy_strict_constraint_when_not_concrete(architecture_tuple, constra
     str(archspec.cpu.host().family) != "x86_64", reason="tests are for x86_64 uarch ranges"
 )
 def test_concretize_target_ranges(root_target_range, dep_target_range, result, monkeypatch):
-    spec = Spec(f"a %gcc@10 foobar=bar target={root_target_range} ^b target={dep_target_range}")
+    spec = Spec(
+        f"pkg-a %gcc@10 foobar=bar target={root_target_range} ^pkg-b target={dep_target_range}"
+    )
     with spack.concretize.disable_compiler_existence_check():
         spec.concretize()
-    assert spec.target == spec["b"].target == result
+    assert spec.target == spec["pkg-b"].target == result