fix

We've seen `getfqdn()` cause slowdowns on macOS in CI when added elsewhere. It's also
called by database.py every time we write the DB file.

- [x] replace the call with a memoized version so that it is only called once per process.

Signed-off-by: Todd Gamblin <tgamblin@llnl.gov>

.devcontainer/ubuntu20.04/devcontainer.json

@@ -1,4 +1,5 @@
 {
+  "name": "Ubuntu 20.04",
   "image": "ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01",
   "postCreateCommand": "./.devcontainer/postCreateCommand.sh"
 }

.devcontainer/ubuntu22.04/devcontainer.json

@@ -0,0 +1,5 @@
+{
+  "name": "Ubuntu 22.04",
+  "image": "ghcr.io/spack/ubuntu-22.04:v2024-05-07",
+  "postCreateCommand": "./.devcontainer/postCreateCommand.sh"
+}

.github/workflows/audit.yaml

@@ -29,7 +29,7 @@ jobs:
         shell: ${{ matrix.system.shell }}
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: ${{inputs.python_version}}
     - name: Install Python packages
@@ -40,6 +40,8 @@ jobs:
       run: |
         python -m pip install --upgrade pywin32
     - name: Package audits (with coverage)
+      env:
+          COVERAGE_FILE: coverage/.coverage-audits-${{ matrix.system.os }}
       if: ${{ inputs.with_coverage == 'true' && runner.os != 'Windows' }}
       run: |
           . share/spack/setup-env.sh
@@ -47,27 +49,26 @@ jobs:
           coverage run $(which spack) audit configs
           coverage run $(which spack) -d audit externals
           coverage combine
-          coverage xml
     - name: Package audits (without coverage)
       if: ${{ inputs.with_coverage == 'false' && runner.os != 'Windows' }}
       run: |
-          . share/spack/setup-env.sh          
+          . share/spack/setup-env.sh
           spack -d audit packages
           spack -d audit configs
           spack -d audit externals
     - name: Package audits (without coverage)
       if: ${{ runner.os == 'Windows' }}
       run: |
-          . share/spack/setup-env.sh          
+          . share/spack/setup-env.sh
           spack -d audit packages
           ./share/spack/qa/validate_last_exit.ps1
           spack -d audit configs
           ./share/spack/qa/validate_last_exit.ps1
           spack -d audit externals
           ./share/spack/qa/validate_last_exit.ps1
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
-      if: ${{ inputs.with_coverage == 'true' }}
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
+      if: ${{ inputs.with_coverage == 'true' && runner.os != 'Windows' }}
       with:
-        flags: unittests,audits
-        token: ${{ secrets.CODECOV_TOKEN }}
-        verbose: true
+        name: coverage-audits-${{ matrix.system.os }}
+        path: coverage
+        include-hidden-files: true

.github/workflows/bootstrap.yml

@@ -63,7 +63,7 @@ jobs:
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
         with:
           python-version: "3.12"
       - name: Bootstrap clingo
@@ -112,10 +112,10 @@ jobs:
     runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        runner: ['macos-13', 'macos-14', "ubuntu-latest"]
+        runner: ['macos-13', 'macos-14', "ubuntu-latest", "windows-latest"]
     steps:
       - name: Setup macOS
-        if: ${{ matrix.runner != 'ubuntu-latest' }}
+        if: ${{ matrix.runner != 'ubuntu-latest' && matrix.runner != 'windows-latest'}}
         run: |
           brew install tree
           # Remove GnuPG since we want to bootstrap it
@@ -124,11 +124,16 @@ jobs:
         if: ${{ matrix.runner == 'ubuntu-latest' }}
         run: |
           sudo rm -rf $(which gpg) $(which gpg2) $(which patchelf)
+      - name: Setup Windows
+        if: ${{ matrix.runner == 'windows-latest' }}
+        run: |
+          Remove-Item -Path (Get-Command gpg).Path
+          Remove-Item -Path (Get-Command file).Path
       - name: Checkout
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
         with:
           python-version: |
             3.8
@@ -137,11 +142,20 @@ jobs:
             3.11
             3.12
       - name: Set bootstrap sources
+        env:
+          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
+          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
+        run: |
+          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
+          spack bootstrap disable github-actions-v0.4
+      - name: Disable from source bootstrap
+        if: ${{ matrix.runner != 'windows-latest' }}
         run: |
           source share/spack/setup-env.sh
-          spack bootstrap disable github-actions-v0.4
           spack bootstrap disable spack-install
       - name: Bootstrap clingo
+        # No binary clingo on Windows yet
+        if: ${{ matrix.runner != 'windows-latest' }}
         run: |
           set -e
           for ver in '3.8' '3.9' '3.10' '3.11' '3.12' ; do
@@ -164,7 +178,24 @@ jobs:
             fi
           done
       - name: Bootstrap GnuPG
+        env:
+          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
+          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
+          USER_SCOPE_PARENT_DIR: ${{ matrix.runner == 'windows-latest' && '$env:userprofile' || '$HOME' }}
+          VALIDATE_LAST_EXIT: ${{ matrix.runner == 'windows-latest' && './share/spack/qa/validate_last_exit.ps1' || '' }}
         run: |
-          source share/spack/setup-env.sh
+          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
           spack -d gpg list
-          tree ~/.spack/bootstrap/store/
+          ${{ env.VALIDATE_LAST_EXIT }}
+          tree ${{ env.USER_SCOPE_PARENT_DIR }}/.spack/bootstrap/store/
+      - name: Bootstrap File
+        env:
+          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
+          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
+          USER_SCOPE_PARENT_DIR: ${{ matrix.runner == 'windows-latest' && '$env:userprofile' || '$HOME' }}
+          VALIDATE_LAST_EXIT: ${{ matrix.runner == 'windows-latest' && './share/spack/qa/validate_last_exit.ps1' || '' }}
+        run: |
+          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
+          spack -d python share/spack/qa/bootstrap-file.py
+          ${{ env.VALIDATE_LAST_EXIT }}
+          tree ${{ env.USER_SCOPE_PARENT_DIR }}/.spack/bootstrap/store/

.github/workflows/build-containers.yml

@@ -87,7 +87,7 @@ jobs:
           fi
 
       - name: Upload Dockerfile
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
         with:
           name: dockerfiles_${{ matrix.dockerfile[0] }}
           path: dockerfiles
@@ -126,7 +126,7 @@ jobs:
     needs: deploy-images
     steps:
       - name: Merge Artifacts
-        uses: actions/upload-artifact/merge@834a144ee995460fba8ed112a2fc961b36a5ec5a
+        uses: actions/upload-artifact/merge@50769540e7f4bd5e21e526ee35c689e35e0d6874
         with:
           name: dockerfiles
           pattern: dockerfiles_*

.github/workflows/ci.yaml

@@ -84,8 +84,30 @@ jobs:
     needs: [ prechecks, changes ]
     uses: ./.github/workflows/unit_tests.yaml
     secrets: inherit
+  upload-coverage:
+    needs: [ unit-tests, prechecks ]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      with:
+        fetch-depth: 0
+    - name: Download coverage files
+      uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
+      with:
+        pattern: coverage-*
+        path: coverage
+        merge-multiple: true
+    - run: pip install --upgrade coverage
+    - run: ls -la coverage
+    - run: coverage combine -a coverage/.coverage*
+    - run: coverage xml
+    - name: "Upload coverage"
+      uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        verbose: true
   all:
-    needs: [ unit-tests, bootstrap ]
+    needs: [ upload-coverage, bootstrap ]
     runs-on: ubuntu-latest
     steps:
     - name: Success

.github/workflows/nightly-win-builds.yml

@@ -17,7 +17,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: 3.9
     - name: Install Python packages

.github/workflows/unit_tests.yaml

@@ -43,7 +43,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install System packages
@@ -76,14 +76,15 @@ jobs:
           SPACK_PYTHON: python
           SPACK_TEST_PARALLEL: 2
           COVERAGE: true
+          COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }}
           UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }}
       run: |
           share/spack/qa/run-unit-tests
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
       with:
-        flags: unittests,linux,${{ matrix.concretizer }}
-        token: ${{ secrets.CODECOV_TOKEN }}
-        verbose: true
+        name: coverage-${{ matrix.os }}-python${{ matrix.python-version }}
+        path: coverage
+        include-hidden-files: true
   # Test shell integration
   shell:
     runs-on: ubuntu-latest
@@ -91,7 +92,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -112,11 +113,11 @@ jobs:
           COVERAGE: true
       run: |
           share/spack/qa/run-shell-tests
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
       with:
-        flags: shelltests,linux
-        token: ${{ secrets.CODECOV_TOKEN }}
-        verbose: true
+        name: coverage-shell
+        path: coverage
+        include-hidden-files: true
 
   # Test RHEL8 UBI with platform Python. This job is run
   # only on PRs modifying core Spack
@@ -151,7 +152,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -170,13 +171,14 @@ jobs:
     - name: Run unit tests (full suite with coverage)
       env:
           COVERAGE: true
+          COVERAGE_FILE: coverage/.coverage-clingo-cffi
       run: |
           share/spack/qa/run-unit-tests
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
       with:
-        flags: unittests,linux,clingo
-        token: ${{ secrets.CODECOV_TOKEN }}
-        verbose: true
+        name: coverage-clingo-cffi
+        path: coverage
+        include-hidden-files: true
   # Run unit tests on MacOS
   macos:
     runs-on: ${{ matrix.os }}
@@ -188,7 +190,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Python packages
@@ -201,6 +203,7 @@ jobs:
     - name: Run unit tests
       env:
         SPACK_TEST_PARALLEL: 4
+        COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }}
       run: |
         git --version
         . .github/workflows/bin/setup_git.sh
@@ -209,11 +212,11 @@ jobs:
         $(which spack) solve zlib
         common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
         $(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
       with:
-        flags: unittests,macos
-        token: ${{ secrets.CODECOV_TOKEN }}
-        verbose: true
+        name: coverage-${{ matrix.os }}-python${{ matrix.python-version }}
+        path: coverage
+        include-hidden-files: true
   # Run unit tests on Windows
   windows:
     defaults:
@@ -225,7 +228,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: 3.9
     - name: Install Python packages
@@ -235,13 +238,13 @@ jobs:
       run: |
         ./.github/workflows/bin/setup_git.ps1
     - name: Unit Test
+      env:
+        COVERAGE_FILE: coverage/.coverage-windows
       run: |
         spack unit-test -x --verbose --cov --cov-config=pyproject.toml
         ./share/spack/qa/validate_last_exit.ps1
-        coverage combine -a
-        coverage xml
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
       with:
-        flags: unittests,windows
-        token: ${{ secrets.CODECOV_TOKEN }}
-        verbose: true
+        name: coverage-windows
+        path: coverage
+        include-hidden-files: true

.github/workflows/valid-style.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: '3.11'
         cache: 'pip'
@@ -38,7 +38,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: '3.11'
         cache: 'pip'
@@ -87,3 +87,62 @@ jobs:
           spack -d bootstrap now --dev
           spack style -t black
           spack unit-test -V
+  import-check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: julia-actions/setup-julia@v2
+      with:
+        version: '1.10'
+    - uses: julia-actions/cache@v2
+
+    # PR: use the base of the PR as the old commit
+    - name: Checkout PR base commit
+      if: github.event_name == 'pull_request'
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      with:
+        ref: ${{ github.event.pull_request.base.sha }}
+        path: old
+    # not a PR: use the previous commit as the old commit
+    - name: Checkout previous commit
+      if: github.event_name != 'pull_request'
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      with:
+        fetch-depth: 2
+        path: old
+    - name: Checkout previous commit
+      if: github.event_name != 'pull_request'
+      run: git -C old reset --hard HEAD^
+
+    - name: Checkout new commit
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      with:
+        path: new
+    - name: Install circular import checker
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      with:
+        repository: haampie/circular-import-fighter
+        ref: 555519c6fd5564fd2eb844e7b87e84f4d12602e2
+        path: circular-import-fighter
+    - name: Install dependencies
+      working-directory: circular-import-fighter
+      run: make -j dependencies
+    - name: Import cycles before
+      working-directory: circular-import-fighter
+      run: make SPACK_ROOT=../old && cp solution solution.old
+    - name: Import cycles after
+      working-directory: circular-import-fighter
+      run: make clean-graph && make SPACK_ROOT=../new && cp solution solution.new
+    - name: Compare import cycles
+      working-directory: circular-import-fighter
+      run: |
+        edges_before="$(grep -oP 'edges to delete: \K\d+' solution.old)"
+        edges_after="$(grep -oP 'edges to delete: \K\d+' solution.new)"
+        if [ "$edges_after" -gt "$edges_before" ]; then
+          printf '\033[1;31mImport check failed: %s imports need to be deleted, ' "$edges_after"
+          printf 'previously this was %s\033[0m\n'  "$edges_before"
+          printf 'Compare \033[1;97m"Import cycles before"\033[0m and '
+          printf '\033[1;97m"Import cycles after"\033[0m to see problematic imports.\n'
+          exit 1
+        else
+          printf '\033[1;32mImport check passed: %s <= %s\033[0m\n' "$edges_after" "$edges_before"
+        fi

etc/spack/defaults/config.yaml

@@ -115,12 +115,6 @@ config:
   suppress_gpg_warnings: false
 
 
-  # If set to true, Spack will attempt to build any compiler on the spec
-  # that is not already available. If set to False, Spack will only use
-  # compilers already configured in compilers.yaml
-  install_missing_compilers: false
-
-
   # If set to true, Spack will always check checksums after downloading
   # archives. If false, Spack skips the checksum step.
   checksum: true

etc/spack/defaults/packages.yaml

@@ -72,3 +72,13 @@ packages:
     permissions:
       read: world
       write: user
+  cray-mpich:
+    buildable: false
+  cray-mvapich2:
+    buildable: false
+  fujitsu-mpi:
+    buildable: false
+  hpcx-mpi:
+    buildable: false
+  spectrum-mpi:
+    buildable: false

lib/spack/docs/basic_usage.rst

@@ -1175,6 +1175,17 @@ unspecified version, but packages can depend on other packages with
 could depend on ``mpich@1.2:`` if it can only build with version
 ``1.2`` or higher of ``mpich``.
 
+.. note:: Windows Spec Syntax Caveats
+   Windows has a few idiosyncrasies when it comes to the Spack spec syntax and the use of certain shells
+   Spack's spec dependency syntax uses the carat (``^``) character, however this is an escape string in CMD
+   so it must be escaped with an additional carat (i.e. ``^^``).
+   CMD also will attempt to interpret strings with ``=`` characters in them. Any spec including this symbol
+   must double quote the string.
+
+   Note: All of these issues are unique to CMD, they can be avoided by using Powershell.
+
+   For more context on these caveats see the related issues: `carat <https://github.com/spack/spack/issues/42833>`_ and `equals <https://github.com/spack/spack/issues/43348>`_
+
 Below are more details about the specifiers that you can add to specs.
 
 .. _version-specifier:

lib/spack/docs/chain.rst

@@ -5,9 +5,9 @@
 
 .. chain:
 
-============================
-Chaining Spack Installations
-============================
+=============================================
+Chaining Spack Installations (upstreams.yaml)
+=============================================
 
 You can point your Spack installation to another installation to use any
 packages that are installed there. To register the other Spack instance,

lib/spack/docs/conf.py

@@ -218,6 +218,8 @@ def setup(sphinx):
     ("py:class", "spack.spec.SpecfileReaderBase"),
     ("py:class", "spack.install_test.Pb"),
     ("py:class", "spack.filesystem_view.SimpleFilesystemView"),
+    ("py:class", "spack.traverse.EdgeAndDepth"),
+    ("py:class", "archspec.cpu.microarchitecture.Microarchitecture"),
 ]
 
 # The reST default role (used for this markup: `text`) to use for all documents.

lib/spack/docs/developer_guide.rst

@@ -181,10 +181,6 @@ Spec-related modules
 :mod:`spack.parser`
   Contains :class:`~spack.parser.SpecParser` and functions related to parsing specs.
 
-:mod:`spack.concretize`
-  Contains :class:`~spack.concretize.Concretizer` implementation,
-  which allows site administrators to change Spack's :ref:`concretization-policies`.
-
 :mod:`spack.version`
   Implements a simple :class:`~spack.version.Version` class with simple
   comparison semantics.  Also implements :class:`~spack.version.VersionRange`

lib/spack/docs/environments.rst

@@ -863,7 +863,7 @@ named list ``compilers`` is ``['%gcc', '%clang', '%intel']`` on
    spack:
      definitions:
        - compilers: ['%gcc', '%clang']
-       - when: arch.satisfies('x86_64:')
+       - when: arch.satisfies('target=x86_64:')
          compilers: ['%intel']
 
 .. note::

lib/spack/docs/getting_started.rst

@@ -1475,16 +1475,14 @@ in a Windows CMD prompt.
 Step 3: Run and configure Spack
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-To use Spack, run ``bin\spack_cmd.bat`` (you may need to Run as Administrator) from the top-level spack
-directory. This will provide a Windows command prompt with an environment properly set up with Spack
-and its prerequisites. If you receive a warning message that Python is not in your ``PATH``
+On Windows, Spack supports both primary native shells, Powershell and the traditional command prompt.
+To use Spack, pick your favorite shell, and run ``bin\spack_cmd.bat`` or ``share/spack/setup-env.ps1``
+(you may need to Run as Administrator) from the top-level spack
+directory. This will provide a Spack enabled shell. If you receive a warning message that Python is not in your ``PATH``
 (which may happen if you installed Python from the website and not the Windows Store) add the location
 of the Python executable to your ``PATH`` now. You can permanently add Python to your ``PATH`` variable
 by using the ``Edit the system environment variables`` utility in Windows Control Panel.
 
-.. note::
-   Alternatively, Powershell can be used in place of CMD
-
 To configure Spack, first run the following command inside the Spack console:
 
 .. code-block:: console
@@ -1549,7 +1547,7 @@ and not tabs, so ensure that this is the case when editing one directly.
 
 .. note:: Cygwin
    The use of Cygwin is not officially supported by Spack and is not tested.
-   However Spack will not throw an error, so use if choosing to use Spack
+   However Spack will not prevent this, so use if choosing to use Spack
    with Cygwin, know that no functionality is garunteed.
 
 ^^^^^^^^^^^^^^^^^
@@ -1563,21 +1561,12 @@ Spack console via:
 
    spack install cpuinfo
 
-If in the previous step, you did not have CMake or Ninja installed, running the command above should bootstrap both packages
+If in the previous step, you did not have CMake or Ninja installed, running the command above should install both packages
 
-"""""""""""""""""""""""""""
-Windows Compatible Packages
-"""""""""""""""""""""""""""
+.. note:: Spec Syntax Caveats
+   Windows has a few idiosyncrasies when it comes to the Spack spec syntax and the use of certain shells
+   See the Spack spec syntax doc for more information
 
-Not all spack packages currently have Windows support. Some are inherently incompatible with the
-platform, and others simply have yet to be ported. To view the current set of packages with Windows
-support, the list command should be used via `spack list -t windows`. If there's a package you'd like
-to install on Windows but is not in that list, feel free to reach out to request the port or contribute
-the port yourself.
-
-.. note::
-   This is by no means a comprehensive list, some packages may have ports that were not tagged
-   while others may just work out of the box on Windows and have not been tagged as such.
 
 ^^^^^^^^^^^^^^
 For developers
@@ -1588,5 +1577,3 @@ Python, Git, and Spack, instead of requiring the user to do so manually.
 Instructions for creating the installer are at
 https://github.com/spack/spack/blob/develop/lib/spack/spack/cmd/installer/README.md
 
-Alternatively a pre-built copy of the Windows installer is available as an artifact of Spack's Windows CI
-available at each run of the CI on develop or any PR.

lib/spack/docs/packaging_guide.rst

@@ -1263,6 +1263,11 @@ Git fetching supports the following parameters to ``version``:
   option ``--depth 1`` will be used if the version of git and the specified
   transport protocol support it, and ``--single-branch`` will be used if the
   version of git supports it.
+* ``git_sparse_paths``: Use ``sparse-checkout`` to only clone these relative paths.
+  This feature requires ``git`` to be version ``2.25.0`` or later but is useful for
+  large repositories that have separate portions that can be built independently.
+  If paths provided are directories then all the subdirectories and associated files
+  will also be cloned. 
 
 Only one of ``tag``, ``branch``, or ``commit`` can be used at a time.
 
@@ -1361,6 +1366,41 @@ Submodules
   For more information about git submodules see the manpage of git: ``man
   git-submodule``.
 
+Sparse-Checkout
+  You can supply ``git_sparse_paths`` at the package or version level to utilize git's 
+  sparse-checkout feature. This will only clone the paths that are specified in the 
+  ``git_sparse_paths`` attribute for the package along with the files in the top level directory.
+  This feature allows you to only clone what you need from a large repository.
+  Note that this is a newer feature in git and requries git ``2.25.0`` or greater.
+  If ``git_sparse_paths`` is supplied and the git version is too old
+  then a warning will be issued and that package will use the standard cloning operations instead.
+  ``git_sparse_paths`` should be supplied as a list of paths, a callable function for versions,
+  or a more complex package attribute using the ``@property`` decorator. The return value should be
+  a list for a callable implementation of ``git_sparse_paths``.
+
+  .. code-block:: python
+
+    def sparse_path_function(package)
+        """a callable function that can be used in side a version"""
+        # paths can be directories or functions, all subdirectories and files are included
+        paths = ["doe", "rae", "me/file.cpp"]
+        if package.spec.version >  Version("1.2.0"):
+            paths.extend(["fae"])
+        return paths
+
+    class MyPackage(package):
+        # can also be a package attribute that will be used if not specified in versions
+        git_sparse_paths = ["doe", "rae"]
+
+        # use the package attribute
+        version("1.0.0")
+        version("1.1.0")
+        # use the function
+        version("1.1.5", git_sparse_paths=sparse_path_func)
+        version("1.2.0", git_sparse_paths=sparse_path_func)
+        version("1.2.5", git_sparse_paths=sparse_path_func)
+        version("1.1.5", git_sparse_paths=sparse_path_func)
+
 .. _github-fetch:
 
 ^^^^^^

lib/spack/docs/pipelines.rst

@@ -663,11 +663,7 @@ build the package.
 
 When including a bootstrapping phase as in the example above, the result is that
 the bootstrapped compiler packages will be pushed to the binary mirror (and the
-local artifacts mirror) before the actual release specs are built. In this case,
-the jobs corresponding to subsequent release specs are configured to
-``install_missing_compilers``, so that if spack is asked to install a package
-with a compiler it doesn't know about, it can be quickly installed from the
-binary mirror first.
+local artifacts mirror) before the actual release specs are built.
 
 Since bootstrapping compilers is optional, those items can be left out of the
 environment/stack file, and in that case no bootstrapping will be done (only the

lib/spack/docs/requirements.txt

@@ -5,8 +5,8 @@ sphinx-rtd-theme==2.0.0
 python-levenshtein==0.25.1
 docutils==0.20.1
 pygments==2.18.0
-urllib3==2.2.2
-pytest==8.3.2
+urllib3==2.2.3
+pytest==8.3.3
 isort==5.13.2
 black==24.8.0
 flake8==7.1.1

lib/spack/external/__init__.py

@@ -18,7 +18,7 @@
 
 * Homepage: https://pypi.python.org/pypi/archspec
 * Usage: Labeling, comparison and detection of microarchitectures
-* Version: 0.2.4 (commit 48b92512b9ce203ded0ebd1ac41b42593e931f7c)
+* Version: 0.2.5-dev (commit bceb39528ac49dd0c876b2e9bf3e7482e9c2be4a)
 
 astunparse
 ----------------

lib/spack/external/_vendoring/distro/distro.py

@@ -1265,27 +1265,29 @@ def _distro_release_info(self) -> Dict[str, str]:
             match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename)
         else:
             try:
-                basenames = [
-                    basename
-                    for basename in os.listdir(self.etc_dir)
-                    if basename not in _DISTRO_RELEASE_IGNORE_BASENAMES
-                    and os.path.isfile(os.path.join(self.etc_dir, basename))
-                ]
+                with os.scandir(self.etc_dir) as it:
+                    etc_files = [
+                        p.path for p in it
+                        if p.is_file() and p.name not in _DISTRO_RELEASE_IGNORE_BASENAMES
+                    ]
                 # We sort for repeatability in cases where there are multiple
                 # distro specific files; e.g. CentOS, Oracle, Enterprise all
                 # containing `redhat-release` on top of their own.
-                basenames.sort()
+                etc_files.sort()
             except OSError:
                 # This may occur when /etc is not readable but we can't be
                 # sure about the *-release files. Check common entries of
                 # /etc for information. If they turn out to not be there the
                 # error is handled in `_parse_distro_release_file()`.
-                basenames = _DISTRO_RELEASE_BASENAMES
-            for basename in basenames:
-                match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename)
+                etc_files = [
+                    os.path.join(self.etc_dir, basename)
+                    for basename in _DISTRO_RELEASE_BASENAMES
+                ]
+
+            for filepath in etc_files:
+                match = _DISTRO_RELEASE_BASENAME_PATTERN.match(os.path.basename(filepath))
                 if match is None:
                     continue
-                filepath = os.path.join(self.etc_dir, basename)
                 distro_info = self._parse_distro_release_file(filepath)
                 # The name is always present if the pattern matches.
                 if "name" not in distro_info:

lib/spack/external/_vendoring/jsonschema/_format.py

@@ -231,96 +231,6 @@ def is_host_name(instance):
     return True
 
 
-try:
-    # The built-in `idna` codec only implements RFC 3890, so we go elsewhere.
-    import idna
-except ImportError:
-    pass
-else:
-    @_checks_drafts(draft7="idn-hostname", raises=idna.IDNAError)
-    def is_idn_host_name(instance):
-        if not isinstance(instance, str_types):
-            return True
-        idna.encode(instance)
-        return True
-
-
-try:
-    import rfc3987
-except ImportError:
-    try:
-        from rfc3986_validator import validate_rfc3986
-    except ImportError:
-        pass
-    else:
-        @_checks_drafts(name="uri")
-        def is_uri(instance):
-            if not isinstance(instance, str_types):
-                return True
-            return validate_rfc3986(instance, rule="URI")
-
-        @_checks_drafts(
-            draft6="uri-reference",
-            draft7="uri-reference",
-            raises=ValueError,
-        )
-        def is_uri_reference(instance):
-            if not isinstance(instance, str_types):
-                return True
-            return validate_rfc3986(instance, rule="URI_reference")
-
-else:
-    @_checks_drafts(draft7="iri", raises=ValueError)
-    def is_iri(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return rfc3987.parse(instance, rule="IRI")
-
-    @_checks_drafts(draft7="iri-reference", raises=ValueError)
-    def is_iri_reference(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return rfc3987.parse(instance, rule="IRI_reference")
-
-    @_checks_drafts(name="uri", raises=ValueError)
-    def is_uri(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return rfc3987.parse(instance, rule="URI")
-
-    @_checks_drafts(
-        draft6="uri-reference",
-        draft7="uri-reference",
-        raises=ValueError,
-    )
-    def is_uri_reference(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return rfc3987.parse(instance, rule="URI_reference")
-
-
-try:
-    from strict_rfc3339 import validate_rfc3339
-except ImportError:
-    try:
-        from rfc3339_validator import validate_rfc3339
-    except ImportError:
-        validate_rfc3339 = None
-
-if validate_rfc3339:
-    @_checks_drafts(name="date-time")
-    def is_datetime(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return validate_rfc3339(instance)
-
-    @_checks_drafts(draft7="time")
-    def is_time(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return is_datetime("1970-01-01T" + instance)
-
-
 @_checks_drafts(name="regex", raises=re.error)
 def is_regex(instance):
     if not isinstance(instance, str_types):
@@ -340,86 +250,3 @@ def is_draft3_time(instance):
     if not isinstance(instance, str_types):
         return True
     return datetime.datetime.strptime(instance, "%H:%M:%S")
-
-
-try:
-    import webcolors
-except ImportError:
-    pass
-else:
-    def is_css_color_code(instance):
-        return webcolors.normalize_hex(instance)
-
-    @_checks_drafts(draft3="color", raises=(ValueError, TypeError))
-    def is_css21_color(instance):
-        if (
-            not isinstance(instance, str_types) or
-            instance.lower() in webcolors.css21_names_to_hex
-        ):
-            return True
-        return is_css_color_code(instance)
-
-    def is_css3_color(instance):
-        if instance.lower() in webcolors.css3_names_to_hex:
-            return True
-        return is_css_color_code(instance)
-
-
-try:
-    import jsonpointer
-except ImportError:
-    pass
-else:
-    @_checks_drafts(
-        draft6="json-pointer",
-        draft7="json-pointer",
-        raises=jsonpointer.JsonPointerException,
-    )
-    def is_json_pointer(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return jsonpointer.JsonPointer(instance)
-
-    # TODO: I don't want to maintain this, so it
-    #       needs to go either into jsonpointer (pending
-    #       https://github.com/stefankoegl/python-json-pointer/issues/34) or
-    #       into a new external library.
-    @_checks_drafts(
-        draft7="relative-json-pointer",
-        raises=jsonpointer.JsonPointerException,
-    )
-    def is_relative_json_pointer(instance):
-        # Definition taken from:
-        # https://tools.ietf.org/html/draft-handrews-relative-json-pointer-01#section-3
-        if not isinstance(instance, str_types):
-            return True
-        non_negative_integer, rest = [], ""
-        for i, character in enumerate(instance):
-            if character.isdigit():
-                non_negative_integer.append(character)
-                continue
-
-            if not non_negative_integer:
-                return False
-
-            rest = instance[i:]
-            break
-        return (rest == "#") or jsonpointer.JsonPointer(rest)
-
-
-try:
-    import uritemplate.exceptions
-except ImportError:
-    pass
-else:
-    @_checks_drafts(
-        draft6="uri-template",
-        draft7="uri-template",
-        raises=uritemplate.exceptions.InvalidTemplate,
-    )
-    def is_uri_template(
-        instance,
-        template_validator=uritemplate.Validator().force_balanced_braces(),
-    ):
-        template = uritemplate.URITemplate(instance)
-        return template_validator.validate(template)

lib/spack/external/archspec/cpu/detect.py

@@ -47,7 +47,11 @@ def decorator(factory):
 
 
 def partial_uarch(
-    name: str = "", vendor: str = "", features: Optional[Set[str]] = None, generation: int = 0
+    name: str = "",
+    vendor: str = "",
+    features: Optional[Set[str]] = None,
+    generation: int = 0,
+    cpu_part: str = "",
 ) -> Microarchitecture:
     """Construct a partial microarchitecture, from information gathered during system scan."""
     return Microarchitecture(
@@ -57,6 +61,7 @@ def partial_uarch(
         features=features or set(),
         compilers={},
         generation=generation,
+        cpu_part=cpu_part,
     )
 
 
@@ -90,6 +95,7 @@ def proc_cpuinfo() -> Microarchitecture:
         return partial_uarch(
             vendor=_canonicalize_aarch64_vendor(data),
             features=_feature_set(data, key="Features"),
+            cpu_part=data.get("CPU part", ""),
         )
 
     if architecture in (PPC64LE, PPC64):
@@ -345,6 +351,10 @@ def sorting_fn(item):
     generic_candidates = [c for c in candidates if c.vendor == "generic"]
     best_generic = max(generic_candidates, key=sorting_fn)
 
+    # Relevant for AArch64. Filter on "cpu_part" if we have any match
+    if info.cpu_part != "" and any(c for c in candidates if info.cpu_part == c.cpu_part):
+        candidates = [c for c in candidates if info.cpu_part == c.cpu_part]
+
     # Filter the candidates to be descendant of the best generic candidate.
     # This is to avoid that the lack of a niche feature that can be disabled
     # from e.g. BIOS prevents detection of a reasonably performant architecture

lib/spack/external/archspec/cpu/microarchitecture.py

@@ -2,9 +2,7 @@
 # Archspec Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-"""Types and functions to manage information
-on CPU microarchitectures.
-"""
+"""Types and functions to manage information on CPU microarchitectures."""
 import functools
 import platform
 import re
@@ -65,21 +63,24 @@ class Microarchitecture:
                 passed in as argument above.
             * versions: versions that support this micro-architecture.
 
-        generation (int): generation of the micro-architecture, if
-            relevant.
+        generation (int): generation of the micro-architecture, if relevant.
+        cpu_part (str): cpu part of the architecture, if relevant.
     """
 
-    # pylint: disable=too-many-arguments
+    # pylint: disable=too-many-arguments,too-many-instance-attributes
     #: Aliases for micro-architecture's features
     feature_aliases = FEATURE_ALIASES
 
-    def __init__(self, name, parents, vendor, features, compilers, generation=0):
+    def __init__(self, name, parents, vendor, features, compilers, generation=0, cpu_part=""):
         self.name = name
         self.parents = parents
         self.vendor = vendor
         self.features = features
         self.compilers = compilers
+        # Only relevant for PowerPC
         self.generation = generation
+        # Only relevant for AArch64
+        self.cpu_part = cpu_part
         # Cache the ancestor computation
         self._ancestors = None
 
@@ -111,8 +112,12 @@ def __eq__(self, other):
             and self.parents == other.parents  # avoid ancestors here
             and self.compilers == other.compilers
             and self.generation == other.generation
+            and self.cpu_part == other.cpu_part
         )
 
+    def __hash__(self):
+        return hash(self.name)
+
     @coerce_target_names
     def __ne__(self, other):
         return not self == other
@@ -143,7 +148,8 @@ def __repr__(self):
         cls_name = self.__class__.__name__
         fmt = (
             cls_name + "({0.name!r}, {0.parents!r}, {0.vendor!r}, "
-            "{0.features!r}, {0.compilers!r}, {0.generation!r})"
+            "{0.features!r}, {0.compilers!r}, generation={0.generation!r}, "
+            "cpu_part={0.cpu_part!r})"
         )
         return fmt.format(self)
 
@@ -190,6 +196,7 @@ def to_dict(self):
             "generation": self.generation,
             "parents": [str(x) for x in self.parents],
             "compilers": self.compilers,
+            "cpupart": self.cpu_part,
         }
 
     @staticmethod
@@ -202,6 +209,7 @@ def from_dict(data) -> "Microarchitecture":
             features=set(data["features"]),
             compilers=data.get("compilers", {}),
             generation=data.get("generation", 0),
+            cpu_part=data.get("cpupart", ""),
         )
 
     def optimization_flags(self, compiler, version):
@@ -360,8 +368,11 @@ def fill_target_from_dict(name, data, targets):
         features = set(values["features"])
         compilers = values.get("compilers", {})
         generation = values.get("generation", 0)
+        cpu_part = values.get("cpupart", "")
 
-        targets[name] = Microarchitecture(name, parents, vendor, features, compilers, generation)
+        targets[name] = Microarchitecture(
+            name, parents, vendor, features, compilers, generation=generation, cpu_part=cpu_part
+        )
 
     known_targets = {}
     data = archspec.cpu.schema.TARGETS_JSON["microarchitectures"]

lib/spack/external/archspec/json/cpu/microarchitectures.json

@@ -2225,10 +2225,14 @@
         ],
         "nvhpc": [
           {
-            "versions": "21.11:",
+            "versions": "21.11:23.8",
 	    "name": "zen3",
             "flags": "-tp {name}",
-	    "warnings": "zen4 is not fully supported by nvhpc yet, falling back to zen3"
+	    "warnings": "zen4 is not fully supported by nvhpc versions < 23.9, falling back to zen3"
+          },
+          {
+            "versions": "23.9:",
+            "flags": "-tp {name}"
           }
 	]
       }
@@ -2711,7 +2715,8 @@
             "flags": "-mcpu=thunderx2t99"
           }
         ]
-      }
+      },
+      "cpupart": "0x0af"
     },
     "a64fx": {
       "from": ["armv8.2a"],
@@ -2779,7 +2784,8 @@
             "flags": "-march=armv8.2-a+crc+crypto+fp16+sve"
           }
         ]
-      }
+      },
+      "cpupart": "0x001"
     },
     "cortex_a72": {
       "from": ["aarch64"],
@@ -2816,7 +2822,8 @@
                   "flags" : "-mcpu=cortex-a72"
               }
           ]
-      }
+      },
+      "cpupart": "0xd08"
     },
     "neoverse_n1": {
       "from": ["cortex_a72", "armv8.2a"],
@@ -2837,8 +2844,7 @@
           "asimdrdm",
           "lrcpc",
           "dcpop",
-          "asimddp",
-          "ssbs"
+          "asimddp"
       ],
       "compilers" : {
           "gcc": [
@@ -2902,7 +2908,8 @@
                   "flags": "-tp {name}"
               }
           ]
-      }
+      },
+      "cpupart": "0xd0c"
     },
     "neoverse_v1": {
       "from": ["neoverse_n1", "armv8.4a"],
@@ -2926,8 +2933,6 @@
           "lrcpc",
           "dcpop",
           "sha3",
-          "sm3",
-          "sm4",
           "asimddp",
           "sha512",
           "sve",
@@ -2936,7 +2941,6 @@
           "uscat",
           "ilrcpc",
           "flagm",
-          "ssbs",
           "dcpodp",
           "svei8mm",
           "svebf16",
@@ -3004,7 +3008,7 @@
               },
               {
                   "versions": "11:",
-                  "flags" : "-march=armv8.4-a+sve+ssbs+fp16+bf16+crypto+i8mm+rng"
+                  "flags" : "-march=armv8.4-a+sve+fp16+bf16+crypto+i8mm+rng"
               },
               {
                   "versions": "12:",
@@ -3028,7 +3032,8 @@
                   "flags": "-tp {name}"
               }
           ]
-      }
+      },
+      "cpupart": "0xd40"
     },
     "neoverse_v2": {
       "from": ["neoverse_n1", "armv9.0a"],
@@ -3052,32 +3057,22 @@
 	  "lrcpc",
 	  "dcpop",
 	  "sha3",
-	  "sm3",
-	  "sm4",
 	  "asimddp",
 	  "sha512",
 	  "sve",
 	  "asimdfhm",
-	  "dit",
 	  "uscat",
 	  "ilrcpc",
 	  "flagm",
-	  "ssbs",
 	  "sb",
 	  "dcpodp",
 	  "sve2",
-	  "sveaes",
-	  "svepmull",
-	  "svebitperm",
-	  "svesha3",
-	  "svesm4",
 	  "flagm2",
 	  "frint",
 	  "svei8mm",
 	  "svebf16",
 	  "i8mm",
-	  "bf16",
-	  "dgh"
+	  "bf16"
       ],
       "compilers" : {
           "gcc": [
@@ -3102,15 +3097,19 @@
                   "flags" : "-march=armv8.5-a+sve -mtune=cortex-a76"
               },
               {
-                  "versions": "10.0:11.99",
+                  "versions": "10.0:11.3.99",
                   "flags" : "-march=armv8.5-a+sve+sve2+i8mm+bf16 -mtune=cortex-a77"
               },
+	      {
+                  "versions": "11.4:11.99",
+                  "flags" : "-mcpu=neoverse-v2"
+              },
               {
-                  "versions": "12.0:12.99",
+                  "versions": "12.0:12.2.99",
                   "flags" : "-march=armv9-a+i8mm+bf16 -mtune=cortex-a710"
               },
 	      {
-                  "versions": "13.0:",
+                  "versions": "12.3:",
                   "flags" : "-mcpu=neoverse-v2"
               }
           ],
@@ -3145,7 +3144,112 @@
                   "flags": "-tp {name}"
               }
           ]
-      }
+      },
+      "cpupart": "0xd4f"
+    },
+    "neoverse_n2": {
+      "from": ["neoverse_n1", "armv9.0a"],
+      "vendor": "ARM",
+      "features": [
+          "fp",
+	  "asimd",
+	  "evtstrm",
+	  "aes",
+	  "pmull",
+	  "sha1",
+	  "sha2",
+	  "crc32",
+	  "atomics",
+	  "fphp",
+	  "asimdhp",
+	  "cpuid",
+	  "asimdrdm",
+	  "jscvt",
+	  "fcma",
+	  "lrcpc",
+	  "dcpop",
+	  "sha3",
+	  "asimddp",
+	  "sha512",
+	  "sve",
+	  "asimdfhm",
+	  "uscat",
+	  "ilrcpc",
+	  "flagm",
+	  "sb",
+	  "dcpodp",
+	  "sve2",
+	  "flagm2",
+	  "frint",
+	  "svei8mm",
+	  "svebf16",
+	  "i8mm",
+	  "bf16"
+      ],
+      "compilers" : {
+          "gcc": [
+              {
+                  "versions": "4.8:5.99",
+                  "flags": "-march=armv8-a"
+              },
+              {
+                  "versions": "6:6.99",
+                  "flags" : "-march=armv8.1-a"
+              },
+              {
+                  "versions": "7.0:7.99",
+                  "flags" : "-march=armv8.2-a -mtune=cortex-a72"
+              },
+              {
+                  "versions": "8.0:8.99",
+                  "flags" : "-march=armv8.4-a+sve -mtune=cortex-a72"
+              },
+              {
+                  "versions": "9.0:9.99",
+                  "flags" : "-march=armv8.5-a+sve -mtune=cortex-a76"
+              },
+              {
+                  "versions": "10.0:10.99",
+                  "flags" : "-march=armv8.5-a+sve+sve2+i8mm+bf16 -mtune=cortex-a77"
+              },
+              {
+                  "versions": "11.0:",
+                  "flags" : "-mcpu=neoverse-n2"
+              }
+          ],
+          "clang" : [
+              {
+                  "versions": "9.0:10.99",
+                  "flags" : "-march=armv8.5-a+sve"
+              },
+              {
+                  "versions": "11.0:13.99",
+                  "flags" : "-march=armv8.5-a+sve+sve2+i8mm+bf16"
+              },
+              {
+                  "versions": "14.0:15.99",
+                  "flags" : "-march=armv9-a+i8mm+bf16"
+              },
+              {
+                  "versions": "16.0:",
+                  "flags" : "-mcpu=neoverse-n2"
+              }
+          ],
+          "arm" : [
+              {
+                  "versions": "23.04.0:",
+                  "flags" : "-mcpu=neoverse-n2"
+              }
+          ],
+          "nvhpc" : [
+              {
+                  "versions": "23.3:",
+                  "name": "neoverse-n1",
+                  "flags": "-tp {name}"
+              }
+          ]
+      },
+      "cpupart": "0xd49"
     },
     "m1": {
       "from": ["armv8.4a"],
@@ -3211,7 +3315,8 @@
             "flags" : "-mcpu=apple-m1"
           }
         ]
-      }
+      },
+      "cpupart": "0x022"
     },
     "m2": {
       "from": ["m1", "armv8.5a"],
@@ -3289,7 +3394,8 @@
             "flags" : "-mcpu=apple-m2"
           }
         ]
-      }
+      },
+      "cpupart": "0x032"
     },
     "arm": {
       "from": [],

lib/spack/external/archspec/json/cpu/microarchitectures_schema.json

@@ -52,6 +52,9 @@
                   }
                 }
               }
+            },
+            "cpupart": {
+              "type": "string"
             }
           },
           "required": [
@@ -107,4 +110,4 @@
       "additionalProperties": false
     }
   }
-}
+}

lib/spack/external/patches/distro.patch

@@ -0,0 +1,45 @@
+diff --git a/lib/spack/external/_vendoring/distro/distro.py b/lib/spack/external/_vendoring/distro/distro.py
+index 89e1868047..50c3b18d4d 100644
+--- a/lib/spack/external/_vendoring/distro/distro.py
++++ b/lib/spack/external/_vendoring/distro/distro.py
+@@ -1265,27 +1265,29 @@ def _distro_release_info(self) -> Dict[str, str]:
+             match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename)
+         else:
+             try:
+-                basenames = [
+-                    basename
+-                    for basename in os.listdir(self.etc_dir)
+-                    if basename not in _DISTRO_RELEASE_IGNORE_BASENAMES
+-                    and os.path.isfile(os.path.join(self.etc_dir, basename))
+-                ]
++                with os.scandir(self.etc_dir) as it:
++                    etc_files = [
++                        p.path for p in it
++                        if p.is_file() and p.name not in _DISTRO_RELEASE_IGNORE_BASENAMES
++                    ]
+                 # We sort for repeatability in cases where there are multiple
+                 # distro specific files; e.g. CentOS, Oracle, Enterprise all
+                 # containing `redhat-release` on top of their own.
+-                basenames.sort()
++                etc_files.sort()
+             except OSError:
+                 # This may occur when /etc is not readable but we can't be
+                 # sure about the *-release files. Check common entries of
+                 # /etc for information. If they turn out to not be there the
+                 # error is handled in `_parse_distro_release_file()`.
+-                basenames = _DISTRO_RELEASE_BASENAMES
+-            for basename in basenames:
+-                match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename)
++                etc_files = [
++                    os.path.join(self.etc_dir, basename)
++                    for basename in _DISTRO_RELEASE_BASENAMES
++                ]
++
++            for filepath in etc_files:
++                match = _DISTRO_RELEASE_BASENAME_PATTERN.match(os.path.basename(filepath))
+                 if match is None:
+                     continue
+-                filepath = os.path.join(self.etc_dir, basename)
+                 distro_info = self._parse_distro_release_file(filepath)
+                 # The name is always present if the pattern matches.
+                 if "name" not in distro_info:

lib/spack/external/patches/jsonschema.patch

@@ -13,3 +13,191 @@ index 6b630cdfbb..1791fe7fbf 100644
 -__version__ = metadata.version("jsonschema")
 +
 +__version__ = "3.2.0"
+diff --git a/lib/spack/external/_vendoring/jsonschema/_format.py b/lib/spack/external/_vendoring/jsonschema/_format.py
+index 281a7cfcff..29061e3661 100644
+--- a/lib/spack/external/_vendoring/jsonschema/_format.py
++++ b/lib/spack/external/_vendoring/jsonschema/_format.py
+@@ -231,96 +231,6 @@ def is_host_name(instance):
+     return True
+ 
+ 
+-try:
+-    # The built-in `idna` codec only implements RFC 3890, so we go elsewhere.
+-    import idna
+-except ImportError:
+-    pass
+-else:
+-    @_checks_drafts(draft7="idn-hostname", raises=idna.IDNAError)
+-    def is_idn_host_name(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        idna.encode(instance)
+-        return True
+-
+-
+-try:
+-    import rfc3987
+-except ImportError:
+-    try:
+-        from rfc3986_validator import validate_rfc3986
+-    except ImportError:
+-        pass
+-    else:
+-        @_checks_drafts(name="uri")
+-        def is_uri(instance):
+-            if not isinstance(instance, str_types):
+-                return True
+-            return validate_rfc3986(instance, rule="URI")
+-
+-        @_checks_drafts(
+-            draft6="uri-reference",
+-            draft7="uri-reference",
+-            raises=ValueError,
+-        )
+-        def is_uri_reference(instance):
+-            if not isinstance(instance, str_types):
+-                return True
+-            return validate_rfc3986(instance, rule="URI_reference")
+-
+-else:
+-    @_checks_drafts(draft7="iri", raises=ValueError)
+-    def is_iri(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return rfc3987.parse(instance, rule="IRI")
+-
+-    @_checks_drafts(draft7="iri-reference", raises=ValueError)
+-    def is_iri_reference(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return rfc3987.parse(instance, rule="IRI_reference")
+-
+-    @_checks_drafts(name="uri", raises=ValueError)
+-    def is_uri(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return rfc3987.parse(instance, rule="URI")
+-
+-    @_checks_drafts(
+-        draft6="uri-reference",
+-        draft7="uri-reference",
+-        raises=ValueError,
+-    )
+-    def is_uri_reference(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return rfc3987.parse(instance, rule="URI_reference")
+-
+-
+-try:
+-    from strict_rfc3339 import validate_rfc3339
+-except ImportError:
+-    try:
+-        from rfc3339_validator import validate_rfc3339
+-    except ImportError:
+-        validate_rfc3339 = None
+-
+-if validate_rfc3339:
+-    @_checks_drafts(name="date-time")
+-    def is_datetime(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return validate_rfc3339(instance)
+-
+-    @_checks_drafts(draft7="time")
+-    def is_time(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return is_datetime("1970-01-01T" + instance)
+-
+-
+ @_checks_drafts(name="regex", raises=re.error)
+ def is_regex(instance):
+     if not isinstance(instance, str_types):
+@@ -340,86 +250,3 @@ def is_draft3_time(instance):
+     if not isinstance(instance, str_types):
+         return True
+     return datetime.datetime.strptime(instance, "%H:%M:%S")
+-
+-
+-try:
+-    import webcolors
+-except ImportError:
+-    pass
+-else:
+-    def is_css_color_code(instance):
+-        return webcolors.normalize_hex(instance)
+-
+-    @_checks_drafts(draft3="color", raises=(ValueError, TypeError))
+-    def is_css21_color(instance):
+-        if (
+-            not isinstance(instance, str_types) or
+-            instance.lower() in webcolors.css21_names_to_hex
+-        ):
+-            return True
+-        return is_css_color_code(instance)
+-
+-    def is_css3_color(instance):
+-        if instance.lower() in webcolors.css3_names_to_hex:
+-            return True
+-        return is_css_color_code(instance)
+-
+-
+-try:
+-    import jsonpointer
+-except ImportError:
+-    pass
+-else:
+-    @_checks_drafts(
+-        draft6="json-pointer",
+-        draft7="json-pointer",
+-        raises=jsonpointer.JsonPointerException,
+-    )
+-    def is_json_pointer(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return jsonpointer.JsonPointer(instance)
+-
+-    # TODO: I don't want to maintain this, so it
+-    #       needs to go either into jsonpointer (pending
+-    #       https://github.com/stefankoegl/python-json-pointer/issues/34) or
+-    #       into a new external library.
+-    @_checks_drafts(
+-        draft7="relative-json-pointer",
+-        raises=jsonpointer.JsonPointerException,
+-    )
+-    def is_relative_json_pointer(instance):
+-        # Definition taken from:
+-        # https://tools.ietf.org/html/draft-handrews-relative-json-pointer-01#section-3
+-        if not isinstance(instance, str_types):
+-            return True
+-        non_negative_integer, rest = [], ""
+-        for i, character in enumerate(instance):
+-            if character.isdigit():
+-                non_negative_integer.append(character)
+-                continue
+-
+-            if not non_negative_integer:
+-                return False
+-
+-            rest = instance[i:]
+-            break
+-        return (rest == "#") or jsonpointer.JsonPointer(rest)
+-
+-
+-try:
+-    import uritemplate.exceptions
+-except ImportError:
+-    pass
+-else:
+-    @_checks_drafts(
+-        draft6="uri-template",
+-        draft7="uri-template",
+-        raises=uritemplate.exceptions.InvalidTemplate,
+-    )
+-    def is_uri_template(
+-        instance,
+-        template_validator=uritemplate.Validator().force_balanced_braces(),
+-    ):
+-        template = uritemplate.URITemplate(instance)
+-        return template_validator.validate(template)

lib/spack/llnl/util/filesystem.py

@@ -27,8 +27,6 @@
 from llnl.util.lang import dedupe, memoized
 from llnl.util.symlink import islink, readlink, resolve_link_target_relative_to_the_link, symlink
 
-from spack.util.executable import Executable, which
-
 from ..path import path_to_os_path, system_path_filter
 
 if sys.platform != "win32":
@@ -53,7 +51,6 @@
     "find_all_headers",
     "find_libraries",
     "find_system_libraries",
-    "fix_darwin_install_name",
     "force_remove",
     "force_symlink",
     "getuid",
@@ -248,42 +245,6 @@ def path_contains_subdirectory(path, root):
     return norm_path.startswith(norm_root)
 
 
-@memoized
-def file_command(*args):
-    """Creates entry point to `file` system command with provided arguments"""
-    file_cmd = which("file", required=True)
-    for arg in args:
-        file_cmd.add_default_arg(arg)
-    return file_cmd
-
-
-@memoized
-def _get_mime_type():
-    """Generate method to call `file` system command to aquire mime type
-    for a specified path
-    """
-    if sys.platform == "win32":
-        # -h option (no-dereference) does not exist in Windows
-        return file_command("-b", "--mime-type")
-    else:
-        return file_command("-b", "-h", "--mime-type")
-
-
-def mime_type(filename):
-    """Returns the mime type and subtype of a file.
-
-    Args:
-        filename: file to be analyzed
-
-    Returns:
-        Tuple containing the MIME type and subtype
-    """
-    output = _get_mime_type()(filename, output=str, error=str).strip()
-    tty.debug("==> " + output)
-    type, _, subtype = output.partition("/")
-    return type, subtype
-
-
 #: This generates the library filenames that may appear on any OS.
 library_extensions = ["a", "la", "so", "tbd", "dylib"]
 
@@ -1624,6 +1585,12 @@ def remove_linked_tree(path):
             shutil.rmtree(os.path.realpath(path), **kwargs)
             os.unlink(path)
         else:
+            if sys.platform == "win32":
+                # Adding this prefix allows shutil to remove long paths on windows
+                # https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry
+                long_path_pfx = "\\\\?\\"
+                if not path.startswith(long_path_pfx):
+                    path = long_path_pfx + path
             shutil.rmtree(path, **kwargs)
 
 
@@ -1673,41 +1640,6 @@ def safe_remove(*files_or_dirs):
         raise
 
 
-@system_path_filter
-def fix_darwin_install_name(path):
-    """Fix install name of dynamic libraries on Darwin to have full path.
-
-    There are two parts of this task:
-
-    1. Use ``install_name('-id', ...)`` to change install name of a single lib
-    2. Use ``install_name('-change', ...)`` to change the cross linking between
-       libs. The function assumes that all libraries are in one folder and
-       currently won't follow subfolders.
-
-    Parameters:
-        path (str): directory in which .dylib files are located
-    """
-    libs = glob.glob(join_path(path, "*.dylib"))
-    for lib in libs:
-        # fix install name first:
-        install_name_tool = Executable("install_name_tool")
-        install_name_tool("-id", lib, lib)
-        otool = Executable("otool")
-        long_deps = otool("-L", lib, output=str).split("\n")
-        deps = [dep.partition(" ")[0][1::] for dep in long_deps[2:-1]]
-        # fix all dependencies:
-        for dep in deps:
-            for loc in libs:
-                # We really want to check for either
-                #     dep == os.path.basename(loc)   or
-                #     dep == join_path(builddir, os.path.basename(loc)),
-                # but we don't know builddir (nor how symbolic links look
-                # in builddir). We thus only compare the basenames.
-                if os.path.basename(dep) == os.path.basename(loc):
-                    install_name_tool("-change", dep, loc, lib)
-                    break
-
-
 def find_first(root: str, files: Union[Iterable[str], str], bfs_depth: int = 2) -> Optional[str]:
     """Find the first file matching a pattern.
 

lib/spack/llnl/util/lang.py

@@ -6,7 +6,6 @@
 import collections.abc
 import contextlib
 import functools
-import inspect
 import itertools
 import os
 import re
@@ -16,7 +15,7 @@
 from typing import Any, Callable, Iterable, List, Tuple
 
 # Ignore emacs backups when listing modules
-ignore_modules = [r"^\.#", "~$"]
+ignore_modules = r"^\.#|~$"
 
 
 def index_by(objects, *funcs):
@@ -84,20 +83,6 @@ def index_by(objects, *funcs):
     return result
 
 
-def caller_locals():
-    """This will return the locals of the *parent* of the caller.
-    This allows a function to insert variables into its caller's
-    scope.  Yes, this is some black magic, and yes it's useful
-    for implementing things like depends_on and provides.
-    """
-    # Passing zero here skips line context for speed.
-    stack = inspect.stack(0)
-    try:
-        return stack[2][0].f_locals
-    finally:
-        del stack
-
-
 def attr_setdefault(obj, name, value):
     """Like dict.setdefault, but for objects."""
     if not hasattr(obj, name):
@@ -105,15 +90,6 @@ def attr_setdefault(obj, name, value):
     return getattr(obj, name)
 
 
-def has_method(cls, name):
-    for base in inspect.getmro(cls):
-        if base is object:
-            continue
-        if name in base.__dict__:
-            return True
-    return False
-
-
 def union_dicts(*dicts):
     """Use update() to combine all dicts into one.
 
@@ -178,19 +154,22 @@ def list_modules(directory, **kwargs):
     order."""
     list_directories = kwargs.setdefault("directories", True)
 
-    for name in os.listdir(directory):
-        if name == "__init__.py":
-            continue
+    ignore = re.compile(ignore_modules)
 
-        path = os.path.join(directory, name)
-        if list_directories and os.path.isdir(path):
-            init_py = os.path.join(path, "__init__.py")
-            if os.path.isfile(init_py):
-                yield name
+    with os.scandir(directory) as it:
+        for entry in it:
+            if entry.name == "__init__.py" or entry.name == "__pycache__":
+                continue
 
-        elif name.endswith(".py"):
-            if not any(re.search(pattern, name) for pattern in ignore_modules):
-                yield re.sub(".py$", "", name)
+            if (
+                list_directories
+                and entry.is_dir()
+                and os.path.isfile(os.path.join(entry.path, "__init__.py"))
+            ):
+                yield entry.name
+
+            elif entry.name.endswith(".py") and entry.is_file() and not ignore.search(entry.name):
+                yield entry.name[:-3]  # strip .py
 
 
 def decorator_with_or_without_args(decorator):
@@ -237,8 +216,8 @@ def setter(name, value):
         value.__name__ = name
         setattr(cls, name, value)
 
-    if not has_method(cls, "_cmp_key"):
-        raise TypeError("'%s' doesn't define _cmp_key()." % cls.__name__)
+    if not hasattr(cls, "_cmp_key"):
+        raise TypeError(f"'{cls.__name__}' doesn't define _cmp_key().")
 
     setter("__eq__", lambda s, o: (s is o) or (o is not None and s._cmp_key() == o._cmp_key()))
     setter("__lt__", lambda s, o: o is not None and s._cmp_key() < o._cmp_key())
@@ -388,8 +367,8 @@ def cd_fun():
         TypeError: If the class does not have a ``_cmp_iter`` method
 
     """
-    if not has_method(cls, "_cmp_iter"):
-        raise TypeError("'%s' doesn't define _cmp_iter()." % cls.__name__)
+    if not hasattr(cls, "_cmp_iter"):
+        raise TypeError(f"'{cls.__name__}' doesn't define _cmp_iter().")
 
     # comparison operators are implemented in terms of lazy_eq and lazy_lt
     def eq(self, other):
@@ -864,20 +843,19 @@ def uniq(sequence):
     return uniq_list
 
 
-def elide_list(line_list, max_num=10):
+def elide_list(line_list: List[str], max_num: int = 10) -> List[str]:
     """Takes a long list and limits it to a smaller number of elements,
     replacing intervening elements with '...'.  For example::
 
-        elide_list([1,2,3,4,5,6], 4)
+        elide_list(["1", "2", "3", "4", "5", "6"], 4)
 
     gives::
 
-        [1, 2, 3, '...', 6]
+        ["1", "2", "3", "...", "6"]
     """
     if len(line_list) > max_num:
-        return line_list[: max_num - 1] + ["..."] + line_list[-1:]
-    else:
-        return line_list
+        return [*line_list[: max_num - 1], "...", line_list[-1]]
+    return line_list
 
 
 @contextlib.contextmanager

lib/spack/llnl/util/tty/log.py

@@ -10,6 +10,7 @@
 import errno
 import io
 import multiprocessing
+import multiprocessing.connection
 import os
 import re
 import select

lib/spack/spack/__init__.py

@@ -3,6 +3,13 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import os
+import re
+from typing import Optional
+
+import spack.paths
+import spack.util.git
+
 #: PEP440 canonical <major>.<minor>.<micro>.<devN> string
 __version__ = "0.23.0.dev0"
 spack_version = __version__
@@ -19,4 +26,47 @@ def __try_int(v):
 spack_version_info = tuple([__try_int(v) for v in __version__.split(".")])
 
 
-__all__ = ["spack_version_info", "spack_version"]
+def get_spack_commit() -> Optional[str]:
+    """Get the Spack git commit sha.
+
+    Returns:
+        (str or None) the commit sha if available, otherwise None
+    """
+    git_path = os.path.join(spack.paths.prefix, ".git")
+    if not os.path.exists(git_path):
+        return None
+
+    git = spack.util.git.git()
+    if not git:
+        return None
+
+    rev = git(
+        "-C",
+        spack.paths.prefix,
+        "rev-parse",
+        "HEAD",
+        output=str,
+        error=os.devnull,
+        fail_on_error=False,
+    )
+    if git.returncode != 0:
+        return None
+
+    match = re.match(r"[a-f\d]{7,}$", rev)
+    return match.group(0) if match else None
+
+
+def get_version() -> str:
+    """Get a descriptive version of this instance of Spack.
+
+    Outputs '<PEP440 version> (<git commit sha>)'.
+
+    The commit sha is only added when available.
+    """
+    commit = get_spack_commit()
+    if commit:
+        return f"{spack_version} ({commit})"
+    return spack_version
+
+
+__all__ = ["spack_version_info", "spack_version", "get_version", "get_spack_commit"]

lib/spack/spack/abi.py

@@ -1,131 +0,0 @@
-# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-import os
-
-from llnl.util.lang import memoized
-
-import spack.spec
-import spack.version
-from spack.compilers.clang import Clang
-from spack.util.executable import Executable, ProcessError
-
-
-class ABI:
-    """This class provides methods to test ABI compatibility between specs.
-    The current implementation is rather rough and could be improved."""
-
-    def architecture_compatible(
-        self, target: spack.spec.Spec, constraint: spack.spec.Spec
-    ) -> bool:
-        """Return true if architecture of target spec is ABI compatible
-        to the architecture of constraint spec. If either the target
-        or constraint specs have no architecture, target is also defined
-        as architecture ABI compatible to constraint."""
-        return (
-            not target.architecture
-            or not constraint.architecture
-            or target.architecture.intersects(constraint.architecture)
-        )
-
-    @memoized
-    def _gcc_get_libstdcxx_version(self, version):
-        """Returns gcc ABI compatibility info by getting the library version of
-        a compiler's libstdc++ or libgcc_s"""
-        from spack.build_environment import dso_suffix
-
-        spec = spack.spec.CompilerSpec("gcc", version)
-        compilers = spack.compilers.compilers_for_spec(spec)
-        if not compilers:
-            return None
-        compiler = compilers[0]
-        rungcc = None
-        libname = None
-        output = None
-        if compiler.cxx:
-            rungcc = Executable(compiler.cxx)
-            libname = "libstdc++." + dso_suffix
-        elif compiler.cc:
-            rungcc = Executable(compiler.cc)
-            libname = "libgcc_s." + dso_suffix
-        else:
-            return None
-        try:
-            # Some gcc's are actually clang and don't respond properly to
-            # --print-file-name (they just print the filename, not the
-            # full path).  Ignore these and expect them to be handled as clang.
-            if Clang.default_version(rungcc.exe[0]) != "unknown":
-                return None
-
-            output = rungcc("--print-file-name=%s" % libname, output=str)
-        except ProcessError:
-            return None
-        if not output:
-            return None
-        libpath = os.path.realpath(output.strip())
-        if not libpath:
-            return None
-        return os.path.basename(libpath)
-
-    @memoized
-    def _gcc_compiler_compare(self, pversion, cversion):
-        """Returns true iff the gcc version pversion and cversion
-        are ABI compatible."""
-        plib = self._gcc_get_libstdcxx_version(pversion)
-        clib = self._gcc_get_libstdcxx_version(cversion)
-        if not plib or not clib:
-            return False
-        return plib == clib
-
-    def _intel_compiler_compare(
-        self, pversion: spack.version.ClosedOpenRange, cversion: spack.version.ClosedOpenRange
-    ) -> bool:
-        """Returns true iff the intel version pversion and cversion
-        are ABI compatible"""
-
-        # Test major and minor versions.  Ignore build version.
-        pv = pversion.lo
-        cv = cversion.lo
-        return pv.up_to(2) == cv.up_to(2)
-
-    def compiler_compatible(
-        self, parent: spack.spec.Spec, child: spack.spec.Spec, loose: bool = False
-    ) -> bool:
-        """Return true if compilers for parent and child are ABI compatible."""
-        if not parent.compiler or not child.compiler:
-            return True
-
-        if parent.compiler.name != child.compiler.name:
-            # Different compiler families are assumed ABI incompatible
-            return False
-
-        if loose:
-            return True
-
-        # TODO: Can we move the specialized ABI matching stuff
-        # TODO: into compiler classes?
-        for pversion in parent.compiler.versions:
-            for cversion in child.compiler.versions:
-                # For a few compilers use specialized comparisons.
-                # Otherwise match on version match.
-                if pversion.intersects(cversion):
-                    return True
-                elif parent.compiler.name == "gcc" and self._gcc_compiler_compare(
-                    pversion, cversion
-                ):
-                    return True
-                elif parent.compiler.name == "intel" and self._intel_compiler_compare(
-                    pversion, cversion
-                ):
-                    return True
-        return False
-
-    def compatible(
-        self, target: spack.spec.Spec, constraint: spack.spec.Spec, loose: bool = False
-    ) -> bool:
-        """Returns true if target spec is ABI compatible to constraint spec"""
-        return self.architecture_compatible(target, constraint) and self.compiler_compatible(
-            target, constraint, loose=loose
-        )

lib/spack/spack/audit.py

@@ -39,18 +39,21 @@ def _search_duplicate_compilers(error_cls):
 import collections
 import collections.abc
 import glob
-import inspect
 import io
 import itertools
+import os
 import pathlib
 import pickle
 import re
 import warnings
+from typing import Iterable, List, Set, Tuple
 from urllib.request import urlopen
 
 import llnl.util.lang
 
+import spack.builder
 import spack.config
+import spack.fetch_strategy
 import spack.patch
 import spack.repo
 import spack.spec
@@ -73,7 +76,9 @@ def __init__(self, summary, details):
         self.details = tuple(details)
 
     def __str__(self):
-        return self.summary + "\n" + "\n".join(["    " + detail for detail in self.details])
+        if self.details:
+            return f"{self.summary}\n" + "\n".join(f"    {detail}" for detail in self.details)
+        return self.summary
 
     def __eq__(self, other):
         if self.summary != other.summary or self.details != other.details:
@@ -210,6 +215,11 @@ def _search_duplicate_compilers(error_cls):
     group="configs", tag="CFG-PACKAGES", description="Sanity checks on packages.yaml", kwargs=()
 )
 
+#: Sanity checks on packages.yaml
+config_repos = AuditClass(
+    group="configs", tag="CFG-REPOS", description="Sanity checks on repositories", kwargs=()
+)
+
 
 @config_packages
 def _search_duplicate_specs_in_externals(error_cls):
@@ -252,40 +262,6 @@ def _search_duplicate_specs_in_externals(error_cls):
     return errors
 
 
-@config_packages
-def _deprecated_preferences(error_cls):
-    """Search package preferences deprecated in v0.21 (and slated for removal in v0.23)"""
-    # TODO (v0.23): remove this audit as the attributes will not be allowed in config
-    errors = []
-    packages_yaml = spack.config.CONFIG.get_config("packages")
-
-    def make_error(attribute_name, config_data, summary):
-        s = io.StringIO()
-        s.write("Occurring in the following file:\n")
-        dict_view = syaml.syaml_dict((k, v) for k, v in config_data.items() if k == attribute_name)
-        syaml.dump_config(dict_view, stream=s, blame=True)
-        return error_cls(summary=summary, details=[s.getvalue()])
-
-    if "all" in packages_yaml and "version" in packages_yaml["all"]:
-        summary = "Using the deprecated 'version' attribute under 'packages:all'"
-        errors.append(make_error("version", packages_yaml["all"], summary))
-
-    for package_name in packages_yaml:
-        if package_name == "all":
-            continue
-
-        package_conf = packages_yaml[package_name]
-        for attribute in ("compiler", "providers", "target"):
-            if attribute not in package_conf:
-                continue
-            summary = (
-                f"Using the deprecated '{attribute}' attribute " f"under 'packages:{package_name}'"
-            )
-            errors.append(make_error(attribute, package_conf, summary))
-
-    return errors
-
-
 @config_packages
 def _avoid_mismatched_variants(error_cls):
     """Warns if variant preferences have mismatched types or names."""
@@ -306,7 +282,7 @@ def _avoid_mismatched_variants(error_cls):
             pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
             for variant in current_spec.variants.values():
                 # Variant does not exist at all
-                if variant.name not in pkg_cls.variants:
+                if variant.name not in pkg_cls.variant_names():
                     summary = (
                         f"Setting a preference for the '{pkg_name}' package to the "
                         f"non-existing variant '{variant.name}'"
@@ -315,9 +291,8 @@ def _avoid_mismatched_variants(error_cls):
                     continue
 
                 # Variant cannot accept this value
-                s = spack.spec.Spec(pkg_name)
                 try:
-                    s.update_variant_validate(variant.name, variant.value)
+                    spack.variant.prevalidate_variant_value(pkg_cls, variant, strict=True)
                 except Exception:
                     summary = (
                         f"Setting the variant '{variant.name}' of the '{pkg_name}' package "
@@ -367,6 +342,27 @@ def _ensure_all_virtual_packages_have_default_providers(error_cls):
     ]
 
 
+@config_repos
+def _ensure_no_folders_without_package_py(error_cls):
+    """Check that we don't leave any folder without a package.py in repos"""
+    errors = []
+    for repository in spack.repo.PATH.repos:
+        missing = []
+        for entry in os.scandir(repository.packages_path):
+            if not entry.is_dir():
+                continue
+            package_py = pathlib.Path(entry.path) / spack.repo.package_file_name
+            if not package_py.exists():
+                missing.append(entry.path)
+        if missing:
+            summary = (
+                f"The '{repository.namespace}' repository misses a package.py file"
+                f" in the following folders"
+            )
+            errors.append(error_cls(summary=summary, details=[f"{x}" for x in missing]))
+    return errors
+
+
 def _make_config_error(config_data, summary, error_cls):
     s = io.StringIO()
     s.write("Occurring in the following file:\n")
@@ -498,7 +494,7 @@ def _search_for_reserved_attributes_names_in_packages(pkgs, error_cls):
         name_definitions = collections.defaultdict(list)
         pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
 
-        for cls_item in inspect.getmro(pkg_cls):
+        for cls_item in pkg_cls.__mro__:
             for name in RESERVED_NAMES:
                 current_value = cls_item.__dict__.get(name)
                 if current_value is None:
@@ -527,7 +523,7 @@ def _ensure_all_package_names_are_lowercase(pkgs, error_cls):
     badname_regex, errors = re.compile(r"[_A-Z]"), []
     for pkg_name in pkgs:
         if badname_regex.search(pkg_name):
-            error_msg = "Package name '{}' is either lowercase or conatine '_'".format(pkg_name)
+            error_msg = f"Package name '{pkg_name}' should be lowercase and must not contain '_'"
             errors.append(error_cls(error_msg, []))
     return errors
 
@@ -666,9 +662,15 @@ def _ensure_env_methods_are_ported_to_builders(pkgs, error_cls):
     errors = []
     for pkg_name in pkgs:
         pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
-        buildsystem_variant, _ = pkg_cls.variants["build_system"]
-        buildsystem_names = [getattr(x, "value", x) for x in buildsystem_variant.values]
-        builder_cls_names = [spack.builder.BUILDER_CLS[x].__name__ for x in buildsystem_names]
+
+        # values are either Value objects (for conditional values) or the values themselves
+        build_system_names = set(
+            v.value if isinstance(v, spack.variant.Value) else v
+            for _, variant in pkg_cls.variant_definitions("build_system")
+            for v in variant.values
+        )
+        builder_cls_names = [spack.builder.BUILDER_CLS[x].__name__ for x in build_system_names]
+
         module = pkg_cls.module
         has_builders_in_package_py = any(
             getattr(module, name, False) for name in builder_cls_names
@@ -687,6 +689,88 @@ def _ensure_env_methods_are_ported_to_builders(pkgs, error_cls):
     return errors
 
 
+class DeprecatedMagicGlobals(ast.NodeVisitor):
+    def __init__(self, magic_globals: Iterable[str]):
+        super().__init__()
+
+        self.magic_globals: Set[str] = set(magic_globals)
+
+        # State to track whether we're in a class function
+        self.depth: int = 0
+        self.in_function: bool = False
+        self.path = (ast.Module, ast.ClassDef, ast.FunctionDef)
+
+        # Defined locals in the current function (heuristically at least)
+        self.locals: Set[str] = set()
+
+        # List of (name, lineno) tuples for references to magic globals
+        self.references_to_globals: List[Tuple[str, int]] = []
+
+    def descend_in_function_def(self, node: ast.AST) -> None:
+        if not isinstance(node, self.path[self.depth]):
+            return
+        self.depth += 1
+        if self.depth == len(self.path):
+            self.in_function = True
+        super().generic_visit(node)
+        if self.depth == len(self.path):
+            self.in_function = False
+            self.locals.clear()
+        self.depth -= 1
+
+    def generic_visit(self, node: ast.AST) -> None:
+        # Recurse into function definitions
+        if self.depth < len(self.path):
+            return self.descend_in_function_def(node)
+        elif not self.in_function:
+            return
+        elif isinstance(node, ast.Global):
+            for name in node.names:
+                if name in self.magic_globals:
+                    self.references_to_globals.append((name, node.lineno))
+        elif isinstance(node, ast.Assign):
+            # visit the rhs before lhs
+            super().visit(node.value)
+            for target in node.targets:
+                super().visit(target)
+        elif isinstance(node, ast.Name) and node.id in self.magic_globals:
+            if isinstance(node.ctx, ast.Load) and node.id not in self.locals:
+                self.references_to_globals.append((node.id, node.lineno))
+            elif isinstance(node.ctx, ast.Store):
+                self.locals.add(node.id)
+        else:
+            super().generic_visit(node)
+
+
+@package_properties
+def _uses_deprecated_globals(pkgs, error_cls):
+    """Ensure that packages do not use deprecated globals"""
+    errors = []
+
+    for pkg_name in pkgs:
+        # some packages scheduled to be removed in v0.23 are not worth fixing.
+        pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
+        if all(v.get("deprecated", False) for v in pkg_cls.versions.values()):
+            continue
+
+        file = spack.repo.PATH.filename_for_package_name(pkg_name)
+        tree = ast.parse(open(file).read())
+        visitor = DeprecatedMagicGlobals(("std_cmake_args",))
+        visitor.visit(tree)
+        if visitor.references_to_globals:
+            errors.append(
+                error_cls(
+                    f"Package '{pkg_name}' uses deprecated globals",
+                    [
+                        f"{file}:{line} references '{name}'"
+                        for name, line in visitor.references_to_globals
+                    ],
+                )
+            )
+
+    return errors
+
+
 @package_https_directives
 def _linting_package_file(pkgs, error_cls):
     """Check for correctness of links"""
@@ -853,20 +937,22 @@ def check_virtual_with_variants(spec, msg):
 
                 # check variants
                 dependency_variants = dep.spec.variants
-                for name, value in dependency_variants.items():
+                for name, variant in dependency_variants.items():
                     try:
-                        v, _ = dependency_pkg_cls.variants[name]
-                        v.validate_or_raise(value, pkg_cls=dependency_pkg_cls)
+                        spack.variant.prevalidate_variant_value(
+                            dependency_pkg_cls, variant, dep.spec, strict=True
+                        )
                     except Exception as e:
                         summary = (
                             f"{pkg_name}: wrong variant used for dependency in 'depends_on()'"
                         )
 
+                        error_msg = str(e)
                         if isinstance(e, KeyError):
                             error_msg = (
                                 f"variant {str(e).strip()} does not exist in package {dep_name}"
+                                f" in package '{dep_name}'"
                             )
-                        error_msg += f" in package '{dep_name}'"
 
                         errors.append(
                             error_cls(summary=summary, details=[error_msg, f"in {filename}"])
@@ -878,39 +964,38 @@ def check_virtual_with_variants(spec, msg):
 @package_directives
 def _ensure_variant_defaults_are_parsable(pkgs, error_cls):
     """Ensures that variant defaults are present and parsable from cli"""
+
+    def check_variant(pkg_cls, variant, vname):
+        # bool is a subclass of int in python. Permitting a default that is an instance
+        # of 'int' means both foo=false and foo=0 are accepted. Other falsish values are
+        # not allowed, since they can't be parsed from CLI ('foo=')
+        default_is_parsable = isinstance(variant.default, int) or variant.default
+
+        if not default_is_parsable:
+            msg = f"Variant '{vname}' of package '{pkg_cls.name}' has an unparsable default value"
+            return [error_cls(msg, [])]
+
+        try:
+            vspec = variant.make_default()
+        except spack.variant.MultipleValuesInExclusiveVariantError:
+            msg = f"Can't create default value for variant '{vname}' in package '{pkg_cls.name}'"
+            return [error_cls(msg, [])]
+
+        try:
+            variant.validate_or_raise(vspec, pkg_cls.name)
+        except spack.variant.InvalidVariantValueError:
+            msg = "Default value of variant '{vname}' in package '{pkg.name}' is invalid"
+            question = "Is it among the allowed values?"
+            return [error_cls(msg, [question])]
+
+        return []
+
     errors = []
     for pkg_name in pkgs:
         pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
-        for variant_name, entry in pkg_cls.variants.items():
-            variant, _ = entry
-            default_is_parsable = (
-                # Permitting a default that is an instance on 'int' permits
-                # to have foo=false or foo=0. Other falsish values are
-                # not allowed, since they can't be parsed from cli ('foo=')
-                isinstance(variant.default, int)
-                or variant.default
-            )
-            if not default_is_parsable:
-                error_msg = "Variant '{}' of package '{}' has a bad default value"
-                errors.append(error_cls(error_msg.format(variant_name, pkg_name), []))
-                continue
-
-            try:
-                vspec = variant.make_default()
-            except spack.variant.MultipleValuesInExclusiveVariantError:
-                error_msg = "Cannot create a default value for the variant '{}' in package '{}'"
-                errors.append(error_cls(error_msg.format(variant_name, pkg_name), []))
-                continue
-
-            try:
-                variant.validate_or_raise(vspec, pkg_cls=pkg_cls)
-            except spack.variant.InvalidVariantValueError:
-                error_msg = (
-                    "The default value of the variant '{}' in package '{}' failed validation"
-                )
-                question = "Is it among the allowed values?"
-                errors.append(error_cls(error_msg.format(variant_name, pkg_name), [question]))
-
+        for vname in pkg_cls.variant_names():
+            for _, variant_def in pkg_cls.variant_definitions(vname):
+                errors.extend(check_variant(pkg_cls, variant_def, vname))
     return errors
 
 
@@ -920,11 +1005,11 @@ def _ensure_variants_have_descriptions(pkgs, error_cls):
     errors = []
     for pkg_name in pkgs:
         pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
-        for variant_name, entry in pkg_cls.variants.items():
-            variant, _ = entry
-            if not variant.description:
-                error_msg = "Variant '{}' in package '{}' is missing a description"
-                errors.append(error_cls(error_msg.format(variant_name, pkg_name), []))
+        for name in pkg_cls.variant_names():
+            for when, variant in pkg_cls.variant_definitions(name):
+                if not variant.description:
+                    msg = f"Variant '{name}' in package '{pkg_name}' is missing a description"
+                    errors.append(error_cls(msg, []))
 
     return errors
 
@@ -981,29 +1066,26 @@ def _version_constraints_are_satisfiable_by_some_version_in_repo(pkgs, error_cls
 
 
 def _analyze_variants_in_directive(pkg, constraint, directive, error_cls):
-    variant_exceptions = (
-        spack.variant.InconsistentValidationError,
-        spack.variant.MultipleValuesInExclusiveVariantError,
-        spack.variant.InvalidVariantValueError,
-        KeyError,
-    )
     errors = []
+    variant_names = pkg.variant_names()
+    summary = f"{pkg.name}: wrong variant in '{directive}' directive"
+    filename = spack.repo.PATH.filename_for_package_name(pkg.name)
+
     for name, v in constraint.variants.items():
+        if name not in variant_names:
+            msg = f"variant {name} does not exist in {pkg.name}"
+            errors.append(error_cls(summary=summary, details=[msg, f"in {filename}"]))
+            continue
+
         try:
-            variant, _ = pkg.variants[name]
-            variant.validate_or_raise(v, pkg_cls=pkg)
-        except variant_exceptions as e:
-            summary = pkg.name + ': wrong variant in "{0}" directive'
-            summary = summary.format(directive)
-            filename = spack.repo.PATH.filename_for_package_name(pkg.name)
-
-            error_msg = str(e).strip()
-            if isinstance(e, KeyError):
-                error_msg = "the variant {0} does not exist".format(error_msg)
-
-            err = error_cls(summary=summary, details=[error_msg, "in " + filename])
-
-            errors.append(err)
+            spack.variant.prevalidate_variant_value(pkg, v, constraint, strict=True)
+        except (
+            spack.variant.InconsistentValidationError,
+            spack.variant.MultipleValuesInExclusiveVariantError,
+            spack.variant.InvalidVariantValueError,
+        ) as e:
+            msg = str(e).strip()
+            errors.append(error_cls(summary=summary, details=[msg, f"in {filename}"]))
 
     return errors
 
@@ -1041,9 +1123,10 @@ def _extracts_errors(triggers, summary):
                 for dname in dnames
             )
 
-        for vname, (variant, triggers) in pkg_cls.variants.items():
-            summary = f"{pkg_name}: wrong 'when=' condition for the '{vname}' variant"
-            errors.extend(_extracts_errors(triggers, summary))
+        for when, variants_by_name in pkg_cls.variants.items():
+            for vname, variant in variants_by_name.items():
+                summary = f"{pkg_name}: wrong 'when=' condition for the '{vname}' variant"
+                errors.extend(_extracts_errors([when], summary))
 
         for when, providers, details in _error_items(pkg_cls.provided):
             errors.extend(

lib/spack/spack/binary_distribution.py

@@ -6,7 +6,6 @@
 import codecs
 import collections
 import concurrent.futures
-import contextlib
 import copy
 import hashlib
 import io
@@ -25,7 +24,7 @@
 import urllib.request
 import warnings
 from contextlib import closing
-from typing import Dict, Generator, Iterable, List, NamedTuple, Optional, Set, Tuple, Union
+from typing import Dict, Iterable, List, NamedTuple, Optional, Set, Tuple, Union
 
 import llnl.util.filesystem as fsys
 import llnl.util.lang
@@ -34,7 +33,6 @@
 from llnl.util.symlink import readlink
 
 import spack.caches
-import spack.cmd
 import spack.config as config
 import spack.database as spack_db
 import spack.error
@@ -45,9 +43,9 @@
 import spack.oci.image
 import spack.oci.oci
 import spack.oci.opener
+import spack.paths
 import spack.platforms
 import spack.relocate as relocate
-import spack.repo
 import spack.spec
 import spack.stage
 import spack.store
@@ -55,6 +53,7 @@
 import spack.util.archive
 import spack.util.crypto
 import spack.util.file_cache as file_cache
+import spack.util.filesystem as ssys
 import spack.util.gpg
 import spack.util.parallel
 import spack.util.path
@@ -106,7 +105,7 @@ class BuildCacheDatabase(spack_db.Database):
     record_fields = ("spec", "ref_count", "in_buildcache")
 
     def __init__(self, root):
-        super().__init__(root, lock_cfg=spack_db.NO_LOCK)
+        super().__init__(root, lock_cfg=spack_db.NO_LOCK, layout=None)
         self._write_transaction_impl = llnl.util.lang.nullcontext
         self._read_transaction_impl = llnl.util.lang.nullcontext
 
@@ -688,7 +687,7 @@ def get_buildfile_manifest(spec):
     # Non-symlinks.
     for rel_path in visitor.files:
         abs_path = os.path.join(root, rel_path)
-        m_type, m_subtype = fsys.mime_type(abs_path)
+        m_type, m_subtype = ssys.mime_type(abs_path)
 
         if relocate.needs_binary_relocation(m_type, m_subtype):
             # Why is this branch not part of needs_binary_relocation? :(
@@ -789,7 +788,9 @@ def sign_specfile(key: str, specfile_path: str) -> str:
     return signed_specfile_path
 
 
-def _read_specs_and_push_index(file_list, read_method, cache_prefix, db, temp_dir, concurrency):
+def _read_specs_and_push_index(
+    file_list, read_method, cache_prefix, db: BuildCacheDatabase, temp_dir, concurrency
+):
     """Read all the specs listed in the provided list, using thread given thread parallelism,
         generate the index, and push it to the mirror.
 
@@ -813,7 +814,7 @@ def _read_specs_and_push_index(file_list, read_method, cache_prefix, db, temp_di
         else:
             continue
 
-        db.add(fetched_spec, None)
+        db.add(fetched_spec)
         db.mark(fetched_spec, "in_buildcache", True)
 
     # Now generate the index, compute its hash, and push the two files to
@@ -958,7 +959,7 @@ def _spec_files_from_cache(url: str):
     raise ListMirrorSpecsError("Failed to get list of specs from {0}".format(url))
 
 
-def generate_package_index(url: str, tmpdir: str, concurrency: int = 32):
+def _url_generate_package_index(url: str, tmpdir: str, concurrency: int = 32):
     """Create or replace the build cache index on the given mirror.  The
     buildcache index contains an entry for each binary package under the
     cache_prefix.
@@ -1119,7 +1120,7 @@ def _exists_in_buildcache(spec: Spec, tmpdir: str, out_url: str) -> ExistsInBuil
     return ExistsInBuildcache(signed, unsigned, tarball)
 
 
-def _upload_tarball_and_specfile(
+def _url_upload_tarball_and_specfile(
     spec: Spec, tmpdir: str, out_url: str, exists: ExistsInBuildcache, signing_key: Optional[str]
 ):
     files = BuildcacheFiles(spec, tmpdir, out_url)
@@ -1154,49 +1155,146 @@ def _upload_tarball_and_specfile(
     )
 
 
+class Uploader:
+    def __init__(self, mirror: spack.mirror.Mirror, force: bool, update_index: bool):
+        self.mirror = mirror
+        self.force = force
+        self.update_index = update_index
+
+        self.tmpdir: str
+        self.executor: concurrent.futures.Executor
+
+    def __enter__(self):
+        self._tmpdir = tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root())
+        self._executor = spack.util.parallel.make_concurrent_executor()
+
+        self.tmpdir = self._tmpdir.__enter__()
+        self.executor = self.executor = self._executor.__enter__()
+
+        return self
+
+    def __exit__(self, *args):
+        self._executor.__exit__(*args)
+        self._tmpdir.__exit__(*args)
+
+    def push_or_raise(self, specs: List[spack.spec.Spec]) -> List[spack.spec.Spec]:
+        skipped, errors = self.push(specs)
+        if errors:
+            raise PushToBuildCacheError(
+                f"Failed to push {len(errors)} specs to {self.mirror.push_url}:\n"
+                + "\n".join(
+                    f"Failed to push {_format_spec(spec)}: {error}" for spec, error in errors
+                )
+            )
+        return skipped
+
+    def push(
+        self, specs: List[spack.spec.Spec]
+    ) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]:
+        raise NotImplementedError
+
+    def tag(self, tag: str, roots: List[spack.spec.Spec]):
+        """Make a list of selected specs together available under the given tag"""
+        pass
+
+
+class OCIUploader(Uploader):
+    def __init__(
+        self,
+        mirror: spack.mirror.Mirror,
+        force: bool,
+        update_index: bool,
+        base_image: Optional[str],
+    ) -> None:
+        super().__init__(mirror, force, update_index)
+        self.target_image = spack.oci.oci.image_from_mirror(mirror)
+        self.base_image = ImageReference.from_string(base_image) if base_image else None
+
+    def push(
+        self, specs: List[spack.spec.Spec]
+    ) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]:
+        skipped, base_images, checksums, upload_errors = _oci_push(
+            target_image=self.target_image,
+            base_image=self.base_image,
+            installed_specs_with_deps=specs,
+            force=self.force,
+            tmpdir=self.tmpdir,
+            executor=self.executor,
+        )
+
+        self._base_images = base_images
+        self._checksums = checksums
+
+        # only update index if any binaries were uploaded
+        if self.update_index and len(skipped) + len(upload_errors) < len(specs):
+            _oci_update_index(self.target_image, self.tmpdir, self.executor)
+
+        return skipped, upload_errors
+
+    def tag(self, tag: str, roots: List[spack.spec.Spec]):
+        tagged_image = self.target_image.with_tag(tag)
+
+        # _push_oci may not populate self._base_images if binaries were already in the registry
+        for spec in roots:
+            _oci_update_base_images(
+                base_image=self.base_image,
+                target_image=self.target_image,
+                spec=spec,
+                base_image_cache=self._base_images,
+            )
+        _oci_put_manifest(
+            self._base_images, self._checksums, tagged_image, self.tmpdir, None, None, *roots
+        )
+
+
+class URLUploader(Uploader):
+    def __init__(
+        self,
+        mirror: spack.mirror.Mirror,
+        force: bool,
+        update_index: bool,
+        signing_key: Optional[str],
+    ) -> None:
+        super().__init__(mirror, force, update_index)
+        self.url = mirror.push_url
+        self.signing_key = signing_key
+
+    def push(
+        self, specs: List[spack.spec.Spec]
+    ) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]:
+        return _url_push(
+            specs,
+            out_url=self.url,
+            force=self.force,
+            update_index=self.update_index,
+            signing_key=self.signing_key,
+            tmpdir=self.tmpdir,
+            executor=self.executor,
+        )
+
+
+def make_uploader(
+    mirror: spack.mirror.Mirror,
+    force: bool = False,
+    update_index: bool = False,
+    signing_key: Optional[str] = None,
+    base_image: Optional[str] = None,
+) -> Uploader:
+    """Builder for the appropriate uploader based on the mirror type"""
+    if mirror.push_url.startswith("oci://"):
+        return OCIUploader(
+            mirror=mirror, force=force, update_index=update_index, base_image=base_image
+        )
+    else:
+        return URLUploader(
+            mirror=mirror, force=force, update_index=update_index, signing_key=signing_key
+        )
+
+
 def _format_spec(spec: Spec) -> str:
     return spec.cformat("{name}{@version}{/hash:7}")
 
 
-@contextlib.contextmanager
-def default_push_context() -> Generator[Tuple[str, concurrent.futures.Executor], None, None]:
-    with tempfile.TemporaryDirectory(
-        dir=spack.stage.get_stage_root()
-    ) as tmpdir, spack.util.parallel.make_concurrent_executor() as executor:
-        yield tmpdir, executor
-
-
-def push_or_raise(
-    specs: List[Spec],
-    out_url: str,
-    signing_key: Optional[str],
-    force: bool = False,
-    update_index: bool = False,
-) -> List[Spec]:
-    """Same as push, but raises an exception on error. Returns a list of skipped specs already
-    present in the build cache when force=False."""
-    skipped, errors = push(specs, out_url, signing_key, force, update_index)
-    if errors:
-        raise PushToBuildCacheError(
-            f"Failed to push {len(errors)} specs to {out_url}:\n"
-            + "\n".join(f"Failed to push {_format_spec(spec)}: {error}" for spec, error in errors)
-        )
-    return skipped
-
-
-def push(
-    specs: List[Spec],
-    out_url: str,
-    signing_key: Optional[str],
-    force: bool = False,
-    update_index: bool = False,
-) -> Tuple[List[Spec], List[Tuple[Spec, BaseException]]]:
-    """Pushes to the provided build cache, and returns a list of skipped specs that were already
-    present (when force=False). Does not raise on error."""
-    with default_push_context() as (tmpdir, executor):
-        return _push(specs, out_url, signing_key, force, update_index, tmpdir, executor)
-
-
 class FancyProgress:
     def __init__(self, total: int):
         self.n = 0
@@ -1234,7 +1332,7 @@ def fail(self) -> None:
         tty.info(f"{self.pre}Failed to push {self.pretty_spec}")
 
 
-def _push(
+def _url_push(
     specs: List[Spec],
     out_url: str,
     signing_key: Optional[str],
@@ -1279,7 +1377,7 @@ def _push(
 
     upload_futures = [
         executor.submit(
-            _upload_tarball_and_specfile,
+            _url_upload_tarball_and_specfile,
             spec,
             tmpdir,
             out_url,
@@ -1309,12 +1407,12 @@ def _push(
     if signing_key:
         keys_tmpdir = os.path.join(tmpdir, "keys")
         os.mkdir(keys_tmpdir)
-        push_keys(out_url, keys=[signing_key], update_index=update_index, tmpdir=keys_tmpdir)
+        _url_push_keys(out_url, keys=[signing_key], update_index=update_index, tmpdir=keys_tmpdir)
 
     if update_index:
         index_tmpdir = os.path.join(tmpdir, "index")
         os.mkdir(index_tmpdir)
-        generate_package_index(out_url, index_tmpdir)
+        _url_generate_package_index(out_url, index_tmpdir)
 
     return skipped, errors
 
@@ -1348,7 +1446,9 @@ def _oci_push_pkg_blob(
     filename = os.path.join(tmpdir, f"{spec.dag_hash()}.tar.gz")
 
     # Create an oci.image.layer aka tarball of the package
-    compressed_tarfile_checksum, tarfile_checksum = spack.oci.oci.create_tarball(spec, filename)
+    compressed_tarfile_checksum, tarfile_checksum = _do_create_tarball(
+        filename, spec.prefix, get_buildinfo_dict(spec)
+    )
 
     blob = spack.oci.oci.Blob(
         Digest.from_sha256(compressed_tarfile_checksum),
@@ -1431,12 +1531,9 @@ def _oci_put_manifest(
     for s in expected_blobs:
         # If a layer for a dependency has gone missing (due to removed manifest in the registry, a
         # failed push, or a local forced uninstall), we cannot create a runnable container image.
-        # If an OCI registry is only used for storage, this is not a hard error, but for now we
-        # raise an exception unconditionally, until someone requests a more lenient behavior.
         checksum = checksums.get(s.dag_hash())
-        if not checksum:
-            raise MissingLayerError(f"missing layer for {_format_spec(s)}")
-        config["rootfs"]["diff_ids"].append(str(checksum.uncompressed_digest))
+        if checksum:
+            config["rootfs"]["diff_ids"].append(str(checksum.uncompressed_digest))
 
     # Set the environment variables
     config["config"]["Env"] = [f"{k}={v}" for k, v in env.items()]
@@ -1481,6 +1578,7 @@ def _oci_put_manifest(
                     "size": checksums[s.dag_hash()].size,
                 }
                 for s in expected_blobs
+                if s.dag_hash() in checksums
             ),
         ],
     }
@@ -1519,7 +1617,7 @@ def _oci_update_base_images(
         )
 
 
-def _push_oci(
+def _oci_push(
     *,
     target_image: ImageReference,
     base_image: Optional[ImageReference],
@@ -1671,7 +1769,7 @@ def _oci_update_index(
 
     for spec_dict in spec_dicts:
         spec = Spec.from_dict(spec_dict)
-        db.add(spec, directory_layout=None)
+        db.add(spec)
         db.mark(spec, "in_buildcache", True)
 
     # Create the index.json file
@@ -2467,9 +2565,8 @@ def install_root_node(spec, unsigned=False, force=False, sha256=None):
     with spack.util.path.filter_padding():
         tty.msg('Installing "{0}" from a buildcache'.format(spec.format()))
         extract_tarball(spec, download_result, force)
-        spec.package.windows_establish_runtime_linkage()
         spack.hooks.post_install(spec, False)
-        spack.store.STORE.db.add(spec, spack.store.STORE.layout)
+        spack.store.STORE.db.add(spec)
 
 
 def install_single_spec(spec, unsigned=False, force=False):
@@ -2601,6 +2698,9 @@ def get_keys(install=False, trust=False, force=False, mirrors=None):
 
     for mirror in mirror_collection.values():
         fetch_url = mirror.fetch_url
+        # TODO: oci:// does not support signing.
+        if fetch_url.startswith("oci://"):
+            continue
         keys_url = url_util.join(
             fetch_url, BUILD_CACHE_RELATIVE_PATH, BUILD_CACHE_KEYS_RELATIVE_PATH
         )
@@ -2645,7 +2745,7 @@ def get_keys(install=False, trust=False, force=False, mirrors=None):
                     )
 
 
-def push_keys(
+def _url_push_keys(
     *mirrors: Union[spack.mirror.Mirror, str],
     keys: List[str],
     tmpdir: str,
@@ -3096,7 +3196,3 @@ class CannotListKeys(GenerateIndexError):
 
 class PushToBuildCacheError(spack.error.SpackError):
     """Raised when unable to push objects to binary mirror"""
-
-
-class MissingLayerError(spack.error.SpackError):
-    """Raised when a required layer for a dependency is missing in an OCI registry."""

lib/spack/spack/bootstrap/__init__.py

@@ -9,6 +9,7 @@
     all_core_root_specs,
     ensure_clingo_importable_or_raise,
     ensure_core_dependencies,
+    ensure_file_in_path_or_raise,
     ensure_gpg_in_path_or_raise,
     ensure_patchelf_in_path_or_raise,
 )
@@ -19,6 +20,7 @@
     "is_bootstrapping",
     "ensure_bootstrap_configuration",
     "ensure_core_dependencies",
+    "ensure_file_in_path_or_raise",
     "ensure_gpg_in_path_or_raise",
     "ensure_clingo_importable_or_raise",
     "ensure_patchelf_in_path_or_raise",

lib/spack/spack/bootstrap/_common.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Common basic functions used through the spack.bootstrap package"""
 import fnmatch
+import importlib
 import os.path
 import re
 import sys
@@ -28,7 +29,7 @@
 
 def _python_import(module: str) -> bool:
     try:
-        __import__(module)
+        importlib.import_module(module)
     except ImportError:
         return False
     return True

lib/spack/spack/bootstrap/config.py

@@ -14,6 +14,7 @@
 import spack.compilers
 import spack.config
 import spack.environment
+import spack.modules
 import spack.paths
 import spack.platforms
 import spack.repo
@@ -143,11 +144,7 @@ def _bootstrap_config_scopes() -> Sequence["spack.config.ConfigScope"]:
 def _add_compilers_if_missing() -> None:
     arch = spack.spec.ArchSpec.frontend_arch()
     if not spack.compilers.compilers_for_arch(arch):
-        new_compilers = spack.compilers.find_new_compilers(
-            mixed_toolchain=sys.platform == "darwin"
-        )
-        if new_compilers:
-            spack.compilers.add_compilers_to_config(new_compilers)
+        spack.compilers.find_compilers()
 
 
 @contextlib.contextmanager
@@ -156,7 +153,7 @@ def _ensure_bootstrap_configuration() -> Generator:
     bootstrap_store_path = store_path()
     user_configuration = _read_and_sanitize_configuration()
     with spack.environment.no_active_environment():
-        with spack.platforms.prevent_cray_detection(), spack.platforms.use_platform(
+        with spack.platforms.use_platform(
             spack.platforms.real_host()
         ), spack.repo.use_repositories(spack.paths.packages_path):
             # Default configuration scopes excluding command line

lib/spack/spack/bootstrap/core.py

@@ -37,21 +37,16 @@
 import spack.binary_distribution
 import spack.config
 import spack.detection
-import spack.environment
-import spack.modules
-import spack.paths
 import spack.platforms
-import spack.platforms.linux
-import spack.repo
 import spack.spec
 import spack.store
 import spack.user_environment
-import spack.util.environment
 import spack.util.executable
 import spack.util.path
 import spack.util.spack_yaml
 import spack.util.url
 import spack.version
+from spack.installer import PackageInstaller
 
 from ._common import _executables_in_store, _python_import, _root_spec, _try_import_from_store
 from .clingo import ClingoBootstrapConcretizer
@@ -283,7 +278,7 @@ def try_import(self, module: str, abstract_spec_str: str) -> bool:
 
         # Install the spec that should make the module importable
         with spack.config.override(self.mirror_scope):
-            concrete_spec.package.do_install(fail_fast=True)
+            PackageInstaller([concrete_spec.package], fail_fast=True).install()
 
         if _try_import_from_store(module, query_spec=concrete_spec, query_info=info):
             self.last_search = info
@@ -306,7 +301,7 @@ def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bo
         msg = "[BOOTSTRAP] Try installing '{0}' from sources"
         tty.debug(msg.format(abstract_spec_str))
         with spack.config.override(self.mirror_scope):
-            concrete_spec.package.do_install()
+            PackageInstaller([concrete_spec.package], fail_fast=True).install()
         if _executables_in_store(executables, concrete_spec, query_info=info):
             self.last_search = info
             return True
@@ -472,7 +467,8 @@ def ensure_clingo_importable_or_raise() -> None:
 
 def gnupg_root_spec() -> str:
     """Return the root spec used to bootstrap GnuPG"""
-    return _root_spec("gnupg@2.3:")
+    root_spec_name = "win-gpg" if IS_WINDOWS else "gnupg"
+    return _root_spec(f"{root_spec_name}@2.3:")
 
 
 def ensure_gpg_in_path_or_raise() -> None:
@@ -482,6 +478,19 @@ def ensure_gpg_in_path_or_raise() -> None:
     )
 
 
+def file_root_spec() -> str:
+    """Return the root spec used to bootstrap file"""
+    root_spec_name = "win-file" if IS_WINDOWS else "file"
+    return _root_spec(root_spec_name)
+
+
+def ensure_file_in_path_or_raise() -> None:
+    """Ensure file is in the PATH or raise"""
+    return ensure_executables_in_path_or_raise(
+        executables=["file"], abstract_spec=file_root_spec()
+    )
+
+
 def patchelf_root_spec() -> str:
     """Return the root spec used to bootstrap patchelf"""
     # 0.13.1 is the last version not to require C++17.
@@ -565,14 +574,15 @@ def ensure_core_dependencies() -> None:
     """Ensure the presence of all the core dependencies."""
     if sys.platform.lower() == "linux":
         ensure_patchelf_in_path_or_raise()
-    if not IS_WINDOWS:
-        ensure_gpg_in_path_or_raise()
+    elif sys.platform == "win32":
+        ensure_file_in_path_or_raise()
+    ensure_gpg_in_path_or_raise()
     ensure_clingo_importable_or_raise()
 
 
 def all_core_root_specs() -> List[str]:
     """Return a list of all the core root specs that may be used to bootstrap Spack"""
-    return [clingo_root_spec(), gnupg_root_spec(), patchelf_root_spec()]
+    return [clingo_root_spec(), gnupg_root_spec(), patchelf_root_spec(), file_root_spec()]
 
 
 def bootstrapping_sources(scope: Optional[str] = None):

lib/spack/spack/bootstrap/environment.py

@@ -14,9 +14,9 @@
 from llnl.util import tty
 
 import spack.environment
+import spack.spec
 import spack.tengine
-import spack.util.cpus
-import spack.util.executable
+import spack.util.path
 
 from ._common import _root_spec
 from .config import root_path, spec_for_current_python, store_path

lib/spack/spack/bootstrap/status.py

@@ -88,7 +88,7 @@ def _core_requirements() -> List[RequiredResponseType]:
 
 def _buildcache_requirements() -> List[RequiredResponseType]:
     _buildcache_exes = {
-        "file": _missing("file", "required to analyze files for buildcaches"),
+        "file": _missing("file", "required to analyze files for buildcaches", system_only=False),
         ("gpg2", "gpg"): _missing("gpg2", "required to sign/verify buildcaches", False),
     }
     if platform.system().lower() == "darwin":
@@ -124,7 +124,7 @@ def _development_requirements() -> List[RequiredResponseType]:
     # Ensure we trigger environment modifications if we have an environment
     if BootstrapEnvironment.spack_yaml().exists():
         with BootstrapEnvironment() as env:
-            env.update_syspath_and_environ()
+            env.load()
 
     return [
         _required_executable(

lib/spack/spack/build_environment.py

@@ -45,6 +45,8 @@
 from itertools import chain
 from typing import Dict, List, Set, Tuple
 
+import archspec.cpu
+
 import llnl.util.tty as tty
 from llnl.string import plural
 from llnl.util.filesystem import join_path
@@ -53,6 +55,7 @@
 from llnl.util.tty.color import cescape, colorize
 from llnl.util.tty.log import MultiProcessFd
 
+import spack.build_systems._checks
 import spack.build_systems.cmake
 import spack.build_systems.meson
 import spack.build_systems.python
@@ -61,26 +64,20 @@
 import spack.config
 import spack.deptypes as dt
 import spack.error
-import spack.main
+import spack.multimethod
 import spack.package_base
 import spack.paths
 import spack.platforms
-import spack.repo
 import spack.schema.environment
 import spack.spec
 import spack.stage
 import spack.store
 import spack.subprocess_context
-import spack.user_environment
 import spack.util.executable
-import spack.util.path
-import spack.util.pattern
 from spack import traverse
 from spack.context import Context
-from spack.error import NoHeadersError, NoLibrariesError
+from spack.error import InstallError, NoHeadersError, NoLibrariesError
 from spack.install_test import spack_install_test_log
-from spack.installer import InstallError
-from spack.util.cpus import determine_number_of_jobs
 from spack.util.environment import (
     SYSTEM_DIR_CASE_ENTRY,
     EnvironmentModifications,
@@ -363,7 +360,7 @@ def set_compiler_environment_variables(pkg, env):
     _add_werror_handling(keep_werror, env)
 
     # Set the target parameters that the compiler will add
-    isa_arg = spec.architecture.target.optimization_flags(compiler)
+    isa_arg = optimization_flags(compiler, spec.target)
     env.set("SPACK_TARGET_ARGS", isa_arg)
 
     # Trap spack-tracked compiler flags as appropriate.
@@ -408,6 +405,36 @@ def set_compiler_environment_variables(pkg, env):
     return env
 
 
+def optimization_flags(compiler, target):
+    if spack.compilers.is_mixed_toolchain(compiler):
+        msg = (
+            "microarchitecture specific optimizations are not "
+            "supported yet on mixed compiler toolchains [check"
+            f" {compiler.name}@{compiler.version} for further details]"
+        )
+        tty.debug(msg)
+        return ""
+
+    # Try to check if the current compiler comes with a version number or
+    # has an unexpected suffix. If so, treat it as a compiler with a
+    # custom spec.
+    compiler_version = compiler.version
+    version_number, suffix = archspec.cpu.version_components(compiler.version)
+    if not version_number or suffix:
+        try:
+            compiler_version = compiler.real_version
+        except spack.util.executable.ProcessError as e:
+            # log this and just return compiler.version instead
+            tty.debug(str(e))
+
+    try:
+        result = target.optimization_flags(compiler.name, compiler_version.dotted_numeric_string)
+    except (ValueError, archspec.cpu.UnsupportedMicroarchitecture):
+        result = ""
+
+    return result
+
+
 def set_wrapper_variables(pkg, env):
     """Set environment variables used by the Spack compiler wrapper (which have the prefix
     `SPACK_`) and also add the compiler wrappers to PATH.
@@ -455,11 +482,14 @@ def set_wrapper_variables(pkg, env):
         env.set(SPACK_DEBUG, "TRUE")
     env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec)
     env.set(SPACK_DEBUG_LOG_ID, pkg.spec.format("{name}-{hash:7}"))
-    env.set(SPACK_DEBUG_LOG_DIR, spack.main.spack_working_dir)
+    env.set(SPACK_DEBUG_LOG_DIR, spack.paths.spack_working_dir)
 
-    # Find ccache binary and hand it to build environment
     if spack.config.get("config:ccache"):
+        # Enable ccache in the compiler wrapper
         env.set(SPACK_CCACHE_BINARY, spack.util.executable.which_string("ccache", required=True))
+    else:
+        # Avoid cache pollution if a build system forces `ccache <compiler wrapper invocation>`.
+        env.set("CCACHE_DISABLE", "1")
 
     # Gather information about various types of dependencies
     link_deps = set(pkg.spec.traverse(root=False, deptype=("link")))
@@ -559,7 +589,7 @@ def set_package_py_globals(pkg, context: Context = Context.BUILD):
         module.std_meson_args = spack.build_systems.meson.MesonBuilder.std_args(pkg)
         module.std_pip_args = spack.build_systems.python.PythonPipBuilder.std_args(pkg)
 
-    jobs = determine_number_of_jobs(parallel=pkg.parallel)
+    jobs = spack.config.determine_number_of_jobs(parallel=pkg.parallel)
     module.make_jobs = jobs
 
     # TODO: make these build deps that can be installed if not found.
@@ -785,7 +815,6 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD):
     # Platform specific setup goes before package specific setup. This is for setting
     # defaults like MACOSX_DEPLOYMENT_TARGET on macOS.
     platform = spack.platforms.by_name(pkg.spec.architecture.platform)
-    target = platform.target(pkg.spec.architecture.target)
     platform.setup_platform_environment(pkg, env_mods)
 
     tty.debug("setup_package: grabbing modifications from dependencies")
@@ -810,9 +839,6 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD):
         for mod in pkg.compiler.modules:
             load_module(mod)
 
-    if target and target.module_name:
-        load_module(target.module_name)
-
     load_external_modules(pkg)
 
     implicit_rpaths = pkg.compiler.implicit_rpaths()
@@ -1136,7 +1162,7 @@ def _setup_pkg_and_run(
         return_value = function(pkg, kwargs)
         write_pipe.send(return_value)
 
-    except StopPhase as e:
+    except spack.error.StopPhase as e:
         # Do not create a full ChildError from this, it's not an error
         # it's a control statement.
         write_pipe.send(e)
@@ -1297,7 +1323,7 @@ def exitcode_msg(p):
     p.join()
 
     # If returns a StopPhase, raise it
-    if isinstance(child_result, StopPhase):
+    if isinstance(child_result, spack.error.StopPhase):
         # do not print
         raise child_result
 
@@ -1506,17 +1532,6 @@ def _make_child_error(msg, module, name, traceback, log, log_type, context):
     return ChildError(msg, module, name, traceback, log, log_type, context)
 
 
-class StopPhase(spack.error.SpackError):
-    """Pickle-able exception to control stopped builds."""
-
-    def __reduce__(self):
-        return _make_stop_phase, (self.message, self.long_message)
-
-
-def _make_stop_phase(msg, long_msg):
-    return StopPhase(msg, long_msg)
-
-
 def write_log_summary(out, log_type, log, last=None):
     errors, warnings = parse_log_events(log)
     nerr = len(errors)
@@ -1550,21 +1565,21 @@ class ModuleChangePropagator:
 
     _PROTECTED_NAMES = ("package", "current_module", "modules_in_mro", "_set_attributes")
 
-    def __init__(self, package):
+    def __init__(self, package: spack.package_base.PackageBase) -> None:
         self._set_self_attributes("package", package)
         self._set_self_attributes("current_module", package.module)
 
         #: Modules for the classes in the MRO up to PackageBase
         modules_in_mro = []
-        for cls in inspect.getmro(type(package)):
-            module = cls.module
+        for cls in package.__class__.__mro__:
+            module = getattr(cls, "module", None)
 
-            if module == self.current_module:
-                continue
-
-            if module == spack.package_base:
+            if module is None or module is spack.package_base:
                 break
 
+            if module is self.current_module:
+                continue
+
             modules_in_mro.append(module)
         self._set_self_attributes("modules_in_mro", modules_in_mro)
         self._set_self_attributes("_set_attributes", {})

lib/spack/spack/build_systems/_checks.py

@@ -8,7 +8,7 @@
 import llnl.util.lang
 
 import spack.builder
-import spack.installer
+import spack.error
 import spack.relocate
 import spack.spec
 import spack.store
@@ -34,7 +34,7 @@ def check_paths(path_list, filetype, predicate):
             if not predicate(abs_path):
                 msg = "Install failed for {0}. No such {1} in prefix: {2}"
                 msg = msg.format(pkg.name, filetype, path)
-                raise spack.installer.InstallError(msg)
+                raise spack.error.InstallError(msg)
 
     check_paths(pkg.sanity_check_is_file, "file", os.path.isfile)
     check_paths(pkg.sanity_check_is_dir, "directory", os.path.isdir)
@@ -42,7 +42,7 @@ def check_paths(path_list, filetype, predicate):
     ignore_file = llnl.util.lang.match_predicate(spack.store.STORE.layout.hidden_file_regexes)
     if all(map(ignore_file, os.listdir(pkg.prefix))):
         msg = "Install failed for {0}.  Nothing was installed!"
-        raise spack.installer.InstallError(msg.format(pkg.name))
+        raise spack.error.InstallError(msg.format(pkg.name))
 
 
 def apply_macos_rpath_fixups(builder: spack.builder.Builder):

lib/spack/spack/build_systems/autotools.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 import os
 import os.path
 import stat
@@ -14,6 +13,7 @@
 
 import spack.build_environment
 import spack.builder
+import spack.error
 import spack.package_base
 from spack.directives import build_system, conflicts, depends_on
 from spack.multimethod import when
@@ -249,7 +249,7 @@ def runs_ok(script_abs_path):
 
         # An external gnuconfig may not not have a prefix.
         if gnuconfig_dir is None:
-            raise spack.build_environment.InstallError(
+            raise spack.error.InstallError(
                 "Spack could not find substitutes for GNU config files because no "
                 "prefix is available for the `gnuconfig` package. Make sure you set a "
                 "prefix path instead of modules for external `gnuconfig`."
@@ -269,7 +269,7 @@ def runs_ok(script_abs_path):
                 msg += (
                     " or the `gnuconfig` package prefix is misconfigured as" " an external package"
                 )
-            raise spack.build_environment.InstallError(msg)
+            raise spack.error.InstallError(msg)
 
         # Filter working substitutes
         candidates = [f for f in candidates if runs_ok(f)]
@@ -294,9 +294,7 @@ def runs_ok(script_abs_path):
    and set the prefix to the directory containing the `config.guess` and
    `config.sub` files.
 """
-            raise spack.build_environment.InstallError(
-                msg.format(", ".join(to_be_found), self.name)
-            )
+            raise spack.error.InstallError(msg.format(", ".join(to_be_found), self.name))
 
         # Copy the good files over the bad ones
         for abs_path in to_be_patched:
@@ -549,13 +547,12 @@ def autoreconf(self, pkg, spec, prefix):
         tty.warn("*        a custom AUTORECONF phase in the package       *")
         tty.warn("*********************************************************")
         with fs.working_dir(self.configure_directory):
-            m = inspect.getmodule(self.pkg)
             # This line is what is needed most of the time
             # --install, --verbose, --force
             autoreconf_args = ["-ivf"]
             autoreconf_args += self.autoreconf_search_path_args
             autoreconf_args += self.autoreconf_extra_args
-            m.autoreconf(*autoreconf_args)
+            self.pkg.module.autoreconf(*autoreconf_args)
 
     @property
     def autoreconf_search_path_args(self):
@@ -579,7 +576,9 @@ def set_configure_or_die(self):
             raise RuntimeError(msg.format(self.configure_directory))
 
         # Monkey-patch the configure script in the corresponding module
-        inspect.getmodule(self.pkg).configure = Executable(self.configure_abs_path)
+        globals_for_pkg = spack.build_environment.ModuleChangePropagator(self.pkg)
+        globals_for_pkg.configure = Executable(self.configure_abs_path)
+        globals_for_pkg.propagate_changes_to_mro()
 
     def configure_args(self):
         """Return the list of all the arguments that must be passed to configure,
@@ -596,7 +595,7 @@ def configure(self, pkg, spec, prefix):
         options += self.configure_args()
 
         with fs.working_dir(self.build_directory, create=True):
-            inspect.getmodule(self.pkg).configure(*options)
+            pkg.module.configure(*options)
 
     def build(self, pkg, spec, prefix):
         """Run "make" on the build targets specified by the builder."""
@@ -604,12 +603,12 @@ def build(self, pkg, spec, prefix):
         params = ["V=1"]
         params += self.build_targets
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make(*params)
+            pkg.module.make(*params)
 
     def install(self, pkg, spec, prefix):
         """Run "make" on the install targets specified by the builder."""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make(*self.install_targets)
+            pkg.module.make(*self.install_targets)
 
     spack.builder.run_after("build")(execute_build_time_tests)
 
@@ -688,9 +687,8 @@ def _activate_or_not(
 
         variant = variant or name
 
-        # Defensively look that the name passed as argument is among
-        # variants
-        if variant not in self.pkg.variants:
+        # Defensively look that the name passed as argument is among variants
+        if not self.pkg.has_variant(variant):
             msg = '"{0}" is not a variant of "{1}"'
             raise KeyError(msg.format(variant, self.pkg.name))
 
@@ -699,27 +697,19 @@ def _activate_or_not(
 
         # Create a list of pairs. Each pair includes a configuration
         # option and whether or not that option is activated
-        variant_desc, _ = self.pkg.variants[variant]
-        if set(variant_desc.values) == set((True, False)):
+        vdef = self.pkg.get_variant(variant)
+        if set(vdef.values) == set((True, False)):
             # BoolValuedVariant carry information about a single option.
             # Nonetheless, for uniformity of treatment we'll package them
             # in an iterable of one element.
-            condition = "+{name}".format(name=variant)
-            options = [(name, condition in spec)]
+            options = [(name, f"+{variant}" in spec)]
         else:
-            condition = "{variant}={value}"
             # "feature_values" is used to track values which correspond to
             # features which can be enabled or disabled as understood by the
             # package's build system. It excludes values which have special
             # meanings and do not correspond to features (e.g. "none")
-            feature_values = (
-                getattr(variant_desc.values, "feature_values", None) or variant_desc.values
-            )
-
-            options = [
-                (value, condition.format(variant=variant, value=value) in spec)
-                for value in feature_values
-            ]
+            feature_values = getattr(vdef.values, "feature_values", None) or vdef.values
+            options = [(value, f"{variant}={value}" in spec) for value in feature_values]
 
         # For each allowed value in the list of values
         for option_value, activated in options:

lib/spack/spack/build_systems/cached_cmake.py

@@ -89,7 +89,7 @@ def define_cmake_cache_from_variant(self, cmake_var, variant=None, comment=""):
         if variant is None:
             variant = cmake_var.lower()
 
-        if variant not in self.pkg.variants:
+        if not self.pkg.has_variant(variant):
             raise KeyError('"{0}" is not a variant of "{1}"'.format(variant, self.pkg.name))
 
         if variant not in self.pkg.spec.variants:

lib/spack/spack/build_systems/cargo.py

@@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import inspect
-
 import llnl.util.filesystem as fs
 
 import spack.builder
@@ -72,9 +70,7 @@ def check_args(self):
     def build(self, pkg, spec, prefix):
         """Runs ``cargo install`` in the source directory"""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(pkg).cargo(
-                "install", "--root", "out", "--path", ".", *self.build_args
-            )
+            pkg.module.cargo("install", "--root", "out", "--path", ".", *self.build_args)
 
     def install(self, pkg, spec, prefix):
         """Copy build files into package prefix."""
@@ -86,4 +82,4 @@ def install(self, pkg, spec, prefix):
     def check(self):
         """Run "cargo test"."""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).cargo("test", *self.check_args)
+            self.pkg.module.cargo("test", *self.check_args)

lib/spack/spack/build_systems/cmake.py

@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import collections.abc
-import inspect
 import os
 import pathlib
 import platform
@@ -16,6 +15,7 @@
 import spack.build_environment
 import spack.builder
 import spack.deptypes as dt
+import spack.error
 import spack.package_base
 from spack.directives import build_system, conflicts, depends_on, variant
 from spack.multimethod import when
@@ -108,6 +108,11 @@ def _conditional_cmake_defaults(pkg: spack.package_base.PackageBase, args: List[
     if _supports_compilation_databases(pkg):
         args.append(CMakeBuilder.define("CMAKE_EXPORT_COMPILE_COMMANDS", True))
 
+    # Enable MACOSX_RPATH by default when cmake_minimum_required < 3
+    # https://cmake.org/cmake/help/latest/policy/CMP0042.html
+    if pkg.spec.satisfies("platform=darwin") and cmake.satisfies("@3:"):
+        args.append(CMakeBuilder.define("CMAKE_POLICY_DEFAULT_CMP0042", "NEW"))
+
 
 def generator(*names: str, default: Optional[str] = None):
     """The build system generator to use.
@@ -141,6 +146,7 @@ def _values(x):
         default=default,
         values=_values,
         description="the build system generator to use",
+        when="build_system=cmake",
     )
     for x in not_used:
         conflicts(f"generator={x}")
@@ -340,7 +346,7 @@ def std_args(pkg, generator=None):
             msg = "Invalid CMake generator: '{0}'\n".format(generator)
             msg += "CMakePackage currently supports the following "
             msg += "primary generators: '{0}'".format("', '".join(valid_primary_generators))
-            raise spack.package_base.InstallError(msg)
+            raise spack.error.InstallError(msg)
 
         try:
             build_type = pkg.spec.variants["build_type"].value
@@ -500,7 +506,7 @@ def define_from_variant(self, cmake_var, variant=None):
         if variant is None:
             variant = cmake_var.lower()
 
-        if variant not in self.pkg.variants:
+        if not self.pkg.has_variant(variant):
             raise KeyError('"{0}" is not a variant of "{1}"'.format(variant, self.pkg.name))
 
         if variant not in self.pkg.spec.variants:
@@ -539,24 +545,24 @@ def cmake(self, pkg, spec, prefix):
         options += self.cmake_args()
         options.append(os.path.abspath(self.root_cmakelists_dir))
         with fs.working_dir(self.build_directory, create=True):
-            inspect.getmodule(self.pkg).cmake(*options)
+            pkg.module.cmake(*options)
 
     def build(self, pkg, spec, prefix):
         """Make the build targets"""
         with fs.working_dir(self.build_directory):
             if self.generator == "Unix Makefiles":
-                inspect.getmodule(self.pkg).make(*self.build_targets)
+                pkg.module.make(*self.build_targets)
             elif self.generator == "Ninja":
                 self.build_targets.append("-v")
-                inspect.getmodule(self.pkg).ninja(*self.build_targets)
+                pkg.module.ninja(*self.build_targets)
 
     def install(self, pkg, spec, prefix):
         """Make the install targets"""
         with fs.working_dir(self.build_directory):
             if self.generator == "Unix Makefiles":
-                inspect.getmodule(self.pkg).make(*self.install_targets)
+                pkg.module.make(*self.install_targets)
             elif self.generator == "Ninja":
-                inspect.getmodule(self.pkg).ninja(*self.install_targets)
+                pkg.module.ninja(*self.install_targets)
 
     spack.builder.run_after("build")(execute_build_time_tests)
 

lib/spack/spack/build_systems/compiler.py

@@ -14,6 +14,7 @@
 
 import spack.compiler
 import spack.package_base
+import spack.util.executable
 
 # Local "type" for type hints
 Path = Union[str, pathlib.Path]

lib/spack/spack/build_systems/cuda.py

@@ -3,6 +3,9 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import re
+from typing import Iterable, List
+
 import spack.variant
 from spack.directives import conflicts, depends_on, variant
 from spack.multimethod import when
@@ -44,6 +47,7 @@ class CudaPackage(PackageBase):
         "87",
         "89",
         "90",
+        "90a",
     )
 
     # FIXME: keep cuda and cuda_arch separate to make usage easier until
@@ -70,6 +74,27 @@ def cuda_flags(arch_list):
             for s in arch_list
         ]
 
+    @staticmethod
+    def compute_capabilities(arch_list: Iterable[str]) -> List[str]:
+        """Adds a decimal place to each CUDA arch.
+
+        >>> compute_capabilities(['90', '90a'])
+        ['9.0', '9.0a']
+
+        Args:
+            arch_list: A list of integer strings, optionally followed by a suffix.
+
+        Returns:
+            A list of float strings, optionally followed by a suffix
+        """
+        pattern = re.compile(r"(\d+)")
+        capabilities = []
+        for arch in arch_list:
+            _, number, letter = re.split(pattern, arch)
+            number = "{0:.1f}".format(float(number) / 10.0)
+            capabilities.append(number + letter)
+        return capabilities
+
     depends_on("cuda", when="+cuda")
 
     # CUDA version vs Architecture
@@ -138,7 +163,7 @@ def cuda_flags(arch_list):
         conflicts("%gcc@11.2:", when="+cuda ^cuda@:11.5")
         conflicts("%gcc@12:", when="+cuda ^cuda@:11.8")
         conflicts("%gcc@13:", when="+cuda ^cuda@:12.3")
-        conflicts("%gcc@14:", when="+cuda ^cuda@:12.5")
+        conflicts("%gcc@14:", when="+cuda ^cuda@:12.6")
         conflicts("%clang@12:", when="+cuda ^cuda@:11.4.0")
         conflicts("%clang@13:", when="+cuda ^cuda@:11.5")
         conflicts("%clang@14:", when="+cuda ^cuda@:11.7")
@@ -146,6 +171,7 @@ def cuda_flags(arch_list):
         conflicts("%clang@16:", when="+cuda ^cuda@:12.1")
         conflicts("%clang@17:", when="+cuda ^cuda@:12.3")
         conflicts("%clang@18:", when="+cuda ^cuda@:12.5")
+        conflicts("%clang@19:", when="+cuda ^cuda@:12.6")
 
         # https://gist.github.com/ax3l/9489132#gistcomment-3860114
         conflicts("%gcc@10", when="+cuda ^cuda@:11.4.0")

lib/spack/spack/build_systems/go.py

@@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import inspect
-
 import llnl.util.filesystem as fs
 
 import spack.builder
@@ -82,7 +80,7 @@ def check_args(self):
     def build(self, pkg, spec, prefix):
         """Runs ``go build`` in the source directory"""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(pkg).go("build", *self.build_args)
+            pkg.module.go("build", *self.build_args)
 
     def install(self, pkg, spec, prefix):
         """Install built binaries into prefix bin."""
@@ -95,4 +93,4 @@ def install(self, pkg, spec, prefix):
     def check(self):
         """Run ``go test .`` in the source directory"""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).go("test", *self.check_args)
+            self.pkg.module.go("test", *self.check_args)

lib/spack/spack/build_systems/intel.py

@@ -22,9 +22,10 @@
     install,
 )
 
+import spack.builder
 import spack.error
 from spack.build_environment import dso_suffix
-from spack.package_base import InstallError
+from spack.error import InstallError
 from spack.util.environment import EnvironmentModifications
 from spack.util.executable import Executable
 from spack.util.prefix import Prefix

lib/spack/spack/build_systems/makefile.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 from typing import List
 
 import llnl.util.filesystem as fs
@@ -103,12 +102,12 @@ def edit(self, pkg, spec, prefix):
     def build(self, pkg, spec, prefix):
         """Run "make" on the build targets specified by the builder."""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make(*self.build_targets)
+            pkg.module.make(*self.build_targets)
 
     def install(self, pkg, spec, prefix):
         """Run "make" on the install targets specified by the builder."""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make(*self.install_targets)
+            pkg.module.make(*self.install_targets)
 
     spack.builder.run_after("build")(execute_build_time_tests)
 

lib/spack/spack/build_systems/meson.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 import os
 from typing import List
 
@@ -195,19 +194,19 @@ def meson(self, pkg, spec, prefix):
         options += self.std_meson_args
         options += self.meson_args()
         with fs.working_dir(self.build_directory, create=True):
-            inspect.getmodule(self.pkg).meson(*options)
+            pkg.module.meson(*options)
 
     def build(self, pkg, spec, prefix):
         """Make the build targets"""
         options = ["-v"]
         options += self.build_targets
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).ninja(*options)
+            pkg.module.ninja(*options)
 
     def install(self, pkg, spec, prefix):
         """Make the install targets"""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).ninja(*self.install_targets)
+            pkg.module.ninja(*self.install_targets)
 
     spack.builder.run_after("build")(execute_build_time_tests)
 

lib/spack/spack/build_systems/msbuild.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 from typing import List  # novm
 
 import llnl.util.filesystem as fs
@@ -104,7 +103,7 @@ def msbuild_install_args(self):
     def build(self, pkg, spec, prefix):
         """Run "msbuild" on the build targets specified by the builder."""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).msbuild(
+            pkg.module.msbuild(
                 *self.std_msbuild_args,
                 *self.msbuild_args(),
                 self.define_targets(*self.build_targets),
@@ -114,6 +113,6 @@ def install(self, pkg, spec, prefix):
         """Run "msbuild" on the install targets specified by the builder.
         This is INSTALL by default"""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).msbuild(
+            pkg.module.msbuild(
                 *self.msbuild_install_args(), self.define_targets(*self.install_targets)
             )

lib/spack/spack/build_systems/nmake.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 from typing import List  # novm
 
 import llnl.util.filesystem as fs
@@ -132,9 +131,7 @@ def build(self, pkg, spec, prefix):
         if self.makefile_name:
             opts.append("/F{}".format(self.makefile_name))
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).nmake(
-                *opts, *self.build_targets, ignore_quotes=self.ignore_quotes
-            )
+            pkg.module.nmake(*opts, *self.build_targets, ignore_quotes=self.ignore_quotes)
 
     def install(self, pkg, spec, prefix):
         """Run "nmake" on the install targets specified by the builder.
@@ -146,6 +143,4 @@ def install(self, pkg, spec, prefix):
             opts.append("/F{}".format(self.makefile_name))
         opts.append(self.define("PREFIX", fs.windows_sfn(prefix)))
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).nmake(
-                *opts, *self.install_targets, ignore_quotes=self.ignore_quotes
-            )
+            pkg.module.nmake(*opts, *self.install_targets, ignore_quotes=self.ignore_quotes)

lib/spack/spack/build_systems/octave.py

@@ -2,8 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
-
 import spack.builder
 import spack.package_base
 from spack.directives import build_system, extends
@@ -47,7 +45,7 @@ class OctaveBuilder(BaseBuilder):
 
     def install(self, pkg, spec, prefix):
         """Install the package from the archive file"""
-        inspect.getmodule(self.pkg).octave(
+        pkg.module.octave(
             "--quiet",
             "--norc",
             "--built-in-docstrings-file=/dev/null",

lib/spack/spack/build_systems/oneapi.py

@@ -15,7 +15,7 @@
 import spack.util.path
 from spack.build_environment import dso_suffix
 from spack.directives import conflicts, license, redistribute, variant
-from spack.package_base import InstallError
+from spack.error import InstallError
 from spack.util.environment import EnvironmentModifications
 from spack.util.executable import Executable
 

lib/spack/spack/build_systems/perl.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 import os
 from typing import Iterable
 
@@ -134,7 +133,7 @@ def build_method(self):
     def build_executable(self):
         """Returns the executable method to build the perl package"""
         if self.build_method == "Makefile.PL":
-            build_executable = inspect.getmodule(self.pkg).make
+            build_executable = self.pkg.module.make
         elif self.build_method == "Build.PL":
             build_executable = Executable(os.path.join(self.pkg.stage.source_path, "Build"))
         return build_executable
@@ -158,7 +157,7 @@ def configure(self, pkg, spec, prefix):
             options = ["Build.PL", "--install_base", prefix]
         options += self.configure_args()
 
-        inspect.getmodule(self.pkg).perl(*options)
+        pkg.module.perl(*options)
 
     # It is possible that the shebang in the Build script that is created from
     # Build.PL may be too long causing the build to fail. Patching the shebang

lib/spack/spack/build_systems/python.py

@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import functools
-import inspect
 import operator
 import os
 import re
@@ -17,7 +16,7 @@
 import llnl.util.filesystem as fs
 import llnl.util.lang as lang
 import llnl.util.tty as tty
-from llnl.util.filesystem import HeaderList, LibraryList
+from llnl.util.filesystem import HeaderList, LibraryList, join_path
 
 import spack.builder
 import spack.config
@@ -25,6 +24,8 @@
 import spack.detection
 import spack.multimethod
 import spack.package_base
+import spack.platforms
+import spack.repo
 import spack.spec
 import spack.store
 from spack.directives import build_system, depends_on, extends
@@ -120,6 +121,12 @@ def skip_modules(self) -> Iterable[str]:
         """
         return []
 
+    @property
+    def bindir(self) -> str:
+        """Path to Python package's bindir, bin on unix like OS's Scripts on Windows"""
+        windows = self.spec.satisfies("platform=windows")
+        return join_path(self.spec.prefix, "Scripts" if windows else "bin")
+
     def view_file_conflicts(self, view, merge_map):
         """Report all file conflicts, excepting special cases for python.
         Specifically, this does not report errors for duplicate
@@ -222,7 +229,7 @@ def test_imports(self) -> None:
 
         # Make sure we are importing the installed modules,
         # not the ones in the source directory
-        python = inspect.getmodule(self).python  # type: ignore[union-attr]
+        python = self.module.python
         for module in self.import_modules:
             with test_part(
                 self,
@@ -309,9 +316,9 @@ def get_external_python_for_prefix(self):
         )
 
         python_externals_detected = [
-            d.spec
-            for d in python_externals_detection.get("python", [])
-            if d.prefix == self.spec.external_path
+            spec
+            for spec in python_externals_detection.get("python", [])
+            if spec.external_path == self.spec.external_path
         ]
         if python_externals_detected:
             return python_externals_detected[0]

lib/spack/spack/build_systems/qmake.py

@@ -2,8 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
-
 from llnl.util.filesystem import working_dir
 
 import spack.builder
@@ -66,17 +64,17 @@ def qmake_args(self):
     def qmake(self, pkg, spec, prefix):
         """Run ``qmake`` to configure the project and generate a Makefile."""
         with working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).qmake(*self.qmake_args())
+            pkg.module.qmake(*self.qmake_args())
 
     def build(self, pkg, spec, prefix):
         """Make the build targets"""
         with working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make()
+            pkg.module.make()
 
     def install(self, pkg, spec, prefix):
         """Make the install targets"""
         with working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make("install")
+            pkg.module.make("install")
 
     def check(self):
         """Search the Makefile for a ``check:`` target and runs it if found."""

lib/spack/spack/build_systems/r.py

@@ -2,10 +2,10 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 from typing import Optional, Tuple
 
 import llnl.util.lang as lang
+from llnl.util.filesystem import mkdirp
 
 from spack.directives import extends
 
@@ -37,6 +37,7 @@ def configure_vars(self):
 
     def install(self, pkg, spec, prefix):
         """Installs an R package."""
+        mkdirp(pkg.module.r_lib_dir)
 
         config_args = self.configure_args()
         config_vars = self.configure_vars()
@@ -44,14 +45,14 @@ def install(self, pkg, spec, prefix):
         args = ["--vanilla", "CMD", "INSTALL"]
 
         if config_args:
-            args.append("--configure-args={0}".format(" ".join(config_args)))
+            args.append(f"--configure-args={' '.join(config_args)}")
 
         if config_vars:
-            args.append("--configure-vars={0}".format(" ".join(config_vars)))
+            args.append(f"--configure-vars={' '.join(config_vars)}")
 
-        args.extend(["--library={0}".format(self.pkg.module.r_lib_dir), self.stage.source_path])
+        args.extend([f"--library={pkg.module.r_lib_dir}", self.stage.source_path])
 
-        inspect.getmodule(self.pkg).R(*args)
+        pkg.module.R(*args)
 
 
 class RPackage(Package):
@@ -80,27 +81,21 @@ class RPackage(Package):
     @lang.classproperty
     def homepage(cls):
         if cls.cran:
-            return "https://cloud.r-project.org/package=" + cls.cran
+            return f"https://cloud.r-project.org/package={cls.cran}"
         elif cls.bioc:
-            return "https://bioconductor.org/packages/" + cls.bioc
+            return f"https://bioconductor.org/packages/{cls.bioc}"
 
     @lang.classproperty
     def url(cls):
         if cls.cran:
-            return (
-                "https://cloud.r-project.org/src/contrib/"
-                + cls.cran
-                + "_"
-                + str(list(cls.versions)[0])
-                + ".tar.gz"
-            )
+            return f"https://cloud.r-project.org/src/contrib/{cls.cran}_{str(list(cls.versions)[0])}.tar.gz"
 
     @lang.classproperty
     def list_url(cls):
         if cls.cran:
-            return "https://cloud.r-project.org/src/contrib/Archive/" + cls.cran + "/"
+            return f"https://cloud.r-project.org/src/contrib/Archive/{cls.cran}/"
 
     @property
     def git(self):
         if self.bioc:
-            return "https://git.bioconductor.org/packages/" + self.bioc
+            return f"https://git.bioconductor.org/packages/{self.bioc}"

lib/spack/spack/build_systems/racket.py

@@ -11,9 +11,9 @@
 
 import spack.builder
 from spack.build_environment import SPACK_NO_PARALLEL_MAKE
+from spack.config import determine_number_of_jobs
 from spack.directives import build_system, extends, maintainers
 from spack.package_base import PackageBase
-from spack.util.cpus import determine_number_of_jobs
 from spack.util.environment import env_flag
 from spack.util.executable import Executable, ProcessError
 

lib/spack/spack/build_systems/ruby.py

@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import glob
-import inspect
 
 import spack.builder
 import spack.package_base
@@ -52,10 +51,10 @@ def build(self, pkg, spec, prefix):
         gemspecs = glob.glob("*.gemspec")
         rakefiles = glob.glob("Rakefile")
         if gemspecs:
-            inspect.getmodule(self.pkg).gem("build", "--norc", gemspecs[0])
+            pkg.module.gem("build", "--norc", gemspecs[0])
         elif rakefiles:
-            jobs = inspect.getmodule(self.pkg).make_jobs
-            inspect.getmodule(self.pkg).rake("package", "-j{0}".format(jobs))
+            jobs = pkg.module.make_jobs
+            pkg.module.rake("package", "-j{0}".format(jobs))
         else:
             # Some Ruby packages only ship `*.gem` files, so nothing to build
             pass
@@ -70,6 +69,6 @@ def install(self, pkg, spec, prefix):
             # if --install-dir is not used, GEM_PATH is deleted from the
             # environement, and Gems required to build native extensions will
             # not be found. Those extensions are built during `gem install`.
-            inspect.getmodule(self.pkg).gem(
+            pkg.module.gem(
                 "install", "--norc", "--ignore-dependencies", "--install-dir", prefix, gems[0]
             )

lib/spack/spack/build_systems/scons.py

@@ -2,8 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
-
 import spack.builder
 import spack.package_base
 from spack.directives import build_system, depends_on
@@ -63,8 +61,7 @@ def build_args(self, spec, prefix):
 
     def build(self, pkg, spec, prefix):
         """Build the package."""
-        args = self.build_args(spec, prefix)
-        inspect.getmodule(self.pkg).scons(*args)
+        pkg.module.scons(*self.build_args(spec, prefix))
 
     def install_args(self, spec, prefix):
         """Arguments to pass to install."""
@@ -72,9 +69,7 @@ def install_args(self, spec, prefix):
 
     def install(self, pkg, spec, prefix):
         """Install the package."""
-        args = self.install_args(spec, prefix)
-
-        inspect.getmodule(self.pkg).scons("install", *args)
+        pkg.module.scons("install", *self.install_args(spec, prefix))
 
     def build_test(self):
         """Run unit tests after build.

lib/spack/spack/build_systems/sip.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 import os
 import re
 
@@ -86,14 +85,13 @@ def import_modules(self):
 
     def python(self, *args, **kwargs):
         """The python ``Executable``."""
-        inspect.getmodule(self).python(*args, **kwargs)
+        self.pkg.module.python(*args, **kwargs)
 
     def test_imports(self):
         """Attempts to import modules of the installed package."""
 
         # Make sure we are importing the installed modules,
         # not the ones in the source directory
-        python = inspect.getmodule(self).python
         for module in self.import_modules:
             with spack.install_test.test_part(
                 self,
@@ -101,7 +99,7 @@ def test_imports(self):
                 purpose="checking import of {0}".format(module),
                 work_dir="spack-test",
             ):
-                python("-c", "import {0}".format(module))
+                self.python("-c", "import {0}".format(module))
 
 
 @spack.builder.builder("sip")
@@ -136,7 +134,7 @@ def configure(self, pkg, spec, prefix):
         """Configure the package."""
 
         # https://www.riverbankcomputing.com/static/Docs/sip/command_line_tools.html
-        args = ["--verbose", "--target-dir", inspect.getmodule(self.pkg).python_platlib]
+        args = ["--verbose", "--target-dir", pkg.module.python_platlib]
         args.extend(self.configure_args())
 
         # https://github.com/Python-SIP/sip/commit/cb0be6cb6e9b756b8b0db3136efb014f6fb9b766
@@ -155,7 +153,7 @@ def build(self, pkg, spec, prefix):
         args = self.build_args()
 
         with working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make(*args)
+            pkg.module.make(*args)
 
     def build_args(self):
         """Arguments to pass to build."""
@@ -166,7 +164,7 @@ def install(self, pkg, spec, prefix):
         args = self.install_args()
 
         with working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make("install", *args)
+            pkg.module.make("install", *args)
 
     def install_args(self):
         """Arguments to pass to install."""

lib/spack/spack/build_systems/waf.py

@@ -2,8 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
-
 from llnl.util.filesystem import working_dir
 
 import spack.builder
@@ -90,11 +88,11 @@ def build_directory(self):
 
     def python(self, *args, **kwargs):
         """The python ``Executable``."""
-        inspect.getmodule(self.pkg).python(*args, **kwargs)
+        self.pkg.module.python(*args, **kwargs)
 
     def waf(self, *args, **kwargs):
         """Runs the waf ``Executable``."""
-        jobs = inspect.getmodule(self.pkg).make_jobs
+        jobs = self.pkg.module.make_jobs
 
         with working_dir(self.build_directory):
             self.python("waf", "-j{0}".format(jobs), *args, **kwargs)

lib/spack/spack/builder.py

@@ -6,12 +6,12 @@
 import collections.abc
 import copy
 import functools
-import inspect
 from typing import List, Optional, Tuple
 
 from llnl.util import lang
 
-import spack.build_environment
+import spack.error
+import spack.multimethod
 
 #: Builder classes, as registered by the "builder" decorator
 BUILDER_CLS = {}
@@ -96,11 +96,10 @@ class hierarchy (look at AspellDictPackage for an example of that)
     Args:
         pkg (spack.package_base.PackageBase): package object for which we need a builder
     """
-    package_module = inspect.getmodule(pkg)
     package_buildsystem = buildsystem_name(pkg)
     default_builder_cls = BUILDER_CLS[package_buildsystem]
     builder_cls_name = default_builder_cls.__name__
-    builder_cls = getattr(package_module, builder_cls_name, None)
+    builder_cls = getattr(pkg.module, builder_cls_name, None)
     if builder_cls:
         return builder_cls(pkg)
 
@@ -295,7 +294,11 @@ def _decorator(fn):
         return _decorator
 
 
-class BuilderMeta(PhaseCallbacksMeta, type(collections.abc.Sequence)):  # type: ignore
+class BuilderMeta(
+    PhaseCallbacksMeta,
+    spack.multimethod.MultiMethodMeta,
+    type(collections.abc.Sequence),  # type: ignore
+):
     pass
 
 
@@ -458,15 +461,13 @@ def _on_phase_start(self, instance):
         # If a phase has a matching stop_before_phase attribute,
         # stop the installation process raising a StopPhase
         if getattr(instance, "stop_before_phase", None) == self.name:
-            raise spack.build_environment.StopPhase(
-                "Stopping before '{0}' phase".format(self.name)
-            )
+            raise spack.error.StopPhase("Stopping before '{0}' phase".format(self.name))
 
     def _on_phase_exit(self, instance):
         # If a phase has a matching last_phase attribute,
         # stop the installation process raising a StopPhase
         if getattr(instance, "last_phase", None) == self.name:
-            raise spack.build_environment.StopPhase("Stopping at '{0}' phase".format(self.name))
+            raise spack.error.StopPhase("Stopping at '{0}' phase".format(self.name))
 
     def copy(self):
         return copy.deepcopy(self)

lib/spack/spack/caches.py

@@ -9,10 +9,8 @@
 
 import llnl.util.lang
 from llnl.util.filesystem import mkdirp
-from llnl.util.symlink import symlink
 
 import spack.config
-import spack.error
 import spack.fetch_strategy
 import spack.paths
 import spack.util.file_cache
@@ -74,23 +72,6 @@ def store(self, fetcher, relative_dest):
         mkdirp(os.path.dirname(dst))
         fetcher.archive(dst)
 
-    def symlink(self, mirror_ref):
-        """Symlink a human readible path in our mirror to the actual
-        storage location."""
-
-        cosmetic_path = os.path.join(self.root, mirror_ref.cosmetic_path)
-        storage_path = os.path.join(self.root, mirror_ref.storage_path)
-        relative_dst = os.path.relpath(storage_path, start=os.path.dirname(cosmetic_path))
-
-        if not os.path.exists(cosmetic_path):
-            if os.path.lexists(cosmetic_path):
-                # In this case the link itself exists but it is broken: remove
-                # it and recreate it (in order to fix any symlinks broken prior
-                # to https://github.com/spack/spack/pull/13908)
-                os.unlink(cosmetic_path)
-            mkdirp(os.path.dirname(cosmetic_path))
-            symlink(relative_dst, cosmetic_path)
-
 
 #: Spack's local cache for downloaded source archives
 FETCH_CACHE: Union[spack.fetch_strategy.FsCache, llnl.util.lang.Singleton] = (

lib/spack/spack/ci.py

@@ -31,6 +31,7 @@
 
 import spack
 import spack.binary_distribution as bindist
+import spack.concretize
 import spack.config as cfg
 import spack.environment as ev
 import spack.main
@@ -38,7 +39,6 @@
 import spack.paths
 import spack.repo
 import spack.spec
-import spack.stage
 import spack.util.git
 import spack.util.gpg as gpg_util
 import spack.util.spack_yaml as syaml
@@ -1110,7 +1110,8 @@ def main_script_replacements(cmd):
             cdash_handler.populate_buildgroup(all_job_names)
         except (SpackError, HTTPError, URLError, TimeoutError) as err:
             tty.warn(f"Problem populating buildgroup: {err}")
-    else:
+    elif cdash_config:
+        # warn only if there was actually a CDash configuration.
         tty.warn("Unable to populate buildgroup without CDash credentials")
 
     service_job_retries = {
@@ -1218,8 +1219,8 @@ def main_script_replacements(cmd):
         # Capture the version of Spack used to generate the pipeline, that can be
         # passed to `git checkout` for version consistency. If we aren't in a Git
         # repository, presume we are a Spack release and use the Git tag instead.
-        spack_version = spack.main.get_version()
-        version_to_clone = spack.main.get_spack_commit() or f"v{spack.spack_version}"
+        spack_version = spack.get_version()
+        version_to_clone = spack.get_spack_commit() or f"v{spack.spack_version}"
 
         output_object["variables"] = {
             "SPACK_ARTIFACTS_ROOT": rel_artifacts_root,
@@ -1382,8 +1383,10 @@ def push_to_build_cache(spec: spack.spec.Spec, mirror_url: str, sign_binaries: b
     """
     tty.debug(f"Pushing to build cache ({'signed' if sign_binaries else 'unsigned'})")
     signing_key = bindist.select_signing_key() if sign_binaries else None
+    mirror = spack.mirror.Mirror.from_url(mirror_url)
     try:
-        bindist.push_or_raise([spec], out_url=mirror_url, signing_key=signing_key)
+        with bindist.make_uploader(mirror, signing_key=signing_key) as uploader:
+            uploader.push_or_raise([spec])
         return True
     except bindist.PushToBuildCacheError as e:
         tty.error(f"Problem writing to {mirror_url}: {e}")
@@ -1433,10 +1436,6 @@ def copy_stage_logs_to_artifacts(job_spec: spack.spec.Spec, job_log_dir: str) ->
         job_log_dir: path into which build log should be copied
     """
     tty.debug(f"job spec: {job_spec}")
-    if not job_spec:
-        msg = f"Cannot copy stage logs: job spec ({job_spec}) is required"
-        tty.error(msg)
-        return
 
     try:
         pkg_cls = spack.repo.PATH.get_pkg_class(job_spec.name)

lib/spack/spack/cmd/__init__.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import argparse
+import importlib
 import os
 import re
 import sys
@@ -16,7 +17,7 @@
 from llnl.util.tty.colify import colify
 from llnl.util.tty.color import colorize
 
-import spack.config
+import spack.config  # breaks a cycle.
 import spack.environment as ev
 import spack.error
 import spack.extensions
@@ -114,8 +115,8 @@ def get_module(cmd_name):
 
     try:
         # Try to import the command from the built-in directory
-        module_name = "%s.%s" % (__name__, pname)
-        module = __import__(module_name, fromlist=[pname, SETUP_PARSER, DESCRIPTION], level=0)
+        module_name = f"{__name__}.{pname}"
+        module = importlib.import_module(module_name)
         tty.debug("Imported {0} from built-in commands".format(pname))
     except ImportError:
         module = spack.extensions.get_module(cmd_name)

lib/spack/spack/cmd/arch.py

@@ -11,6 +11,7 @@
 import llnl.util.tty.color as color
 
 import spack.platforms
+import spack.spec
 
 description = "print architecture information about this machine"
 section = "system"

lib/spack/spack/cmd/audit.py

@@ -115,15 +115,11 @@ def audit(parser, args):
 def _process_reports(reports):
     for check, errors in reports:
         if errors:
-            msg = "{0}: {1} issue{2} found".format(
-                check, len(errors), "" if len(errors) == 1 else "s"
-            )
-            header = "@*b{" + msg + "}"
-            print(cl.colorize(header))
+            status = f"{len(errors)} issue{'' if len(errors) == 1 else 's'} found"
+            print(cl.colorize(f"{check}: @*r{{{status}}}"))
+            numdigits = len(str(len(errors)))
             for idx, error in enumerate(errors):
-                print(str(idx + 1) + ". " + str(error))
+                print(f"{idx + 1:>{numdigits}}. {error}")
             raise SystemExit(1)
         else:
-            msg = "{0}: 0 issues found.".format(check)
-            header = "@*b{" + msg + "}"
-            print(cl.colorize(header))
+            print(cl.colorize(f"{check}: @*g{{passed}}"))

lib/spack/spack/cmd/bootstrap.py

@@ -16,11 +16,11 @@
 import spack.bootstrap.config
 import spack.bootstrap.core
 import spack.config
-import spack.main
 import spack.mirror
 import spack.spec
 import spack.stage
 import spack.util.path
+import spack.util.spack_yaml
 from spack.cmd.common import arguments
 
 description = "manage bootstrap configuration"

lib/spack/spack/cmd/buildcache.py

@@ -23,21 +23,15 @@
 import spack.error
 import spack.mirror
 import spack.oci.oci
-import spack.oci.opener
-import spack.relocate
-import spack.repo
 import spack.spec
 import spack.stage
 import spack.store
-import spack.user_environment
-import spack.util.crypto
 import spack.util.parallel
 import spack.util.url as url_util
 import spack.util.web as web_util
 from spack import traverse
 from spack.cmd import display_specs
 from spack.cmd.common import arguments
-from spack.oci.image import ImageReference
 from spack.spec import Spec, save_dependency_specfiles
 
 description = "create, download and install binary packages"
@@ -392,13 +386,8 @@ def push_fn(args):
     else:
         roots = spack.cmd.require_active_env(cmd_name="buildcache push").concrete_roots()
 
-    mirror: spack.mirror.Mirror = args.mirror
-
-    # Check if this is an OCI image.
-    try:
-        target_image = spack.oci.oci.image_from_mirror(mirror)
-    except ValueError:
-        target_image = None
+    mirror = args.mirror
+    assert isinstance(mirror, spack.mirror.Mirror)
 
     push_url = mirror.push_url
 
@@ -409,14 +398,11 @@ def push_fn(args):
         unsigned = not (args.key or args.signed)
 
     # For OCI images, we require dependencies to be pushed for now.
-    if target_image:
-        if "dependencies" not in args.things_to_install:
-            tty.die("Dependencies must be pushed for OCI images.")
-        if not unsigned:
-            tty.warn(
-                "Code signing is currently not supported for OCI images. "
-                "Use --unsigned to silence this warning."
-            )
+    if mirror.push_url.startswith("oci://") and not unsigned:
+        tty.warn(
+            "Code signing is currently not supported for OCI images. "
+            "Use --unsigned to silence this warning."
+        )
         unsigned = True
 
     # Select a signing key, or None if unsigned.
@@ -447,49 +433,17 @@ def push_fn(args):
                     (s, PackageNotInstalledError("package not installed")) for s in not_installed
                 )
 
-    with bindist.default_push_context() as (tmpdir, executor):
-        if target_image:
-            base_image = ImageReference.from_string(args.base_image) if args.base_image else None
-            skipped, base_images, checksums, upload_errors = bindist._push_oci(
-                target_image=target_image,
-                base_image=base_image,
-                installed_specs_with_deps=specs,
-                force=args.force,
-                tmpdir=tmpdir,
-                executor=executor,
-            )
-
-            if upload_errors:
-                failed.extend(upload_errors)
-
-            # Apart from creating manifests for each individual spec, we allow users to create a
-            # separate image tag for all root specs and their runtime dependencies.
-            elif args.tag:
-                tagged_image = target_image.with_tag(args.tag)
-                # _push_oci may not populate base_images if binaries were already in the registry
-                for spec in roots:
-                    bindist._oci_update_base_images(
-                        base_image=base_image,
-                        target_image=target_image,
-                        spec=spec,
-                        base_image_cache=base_images,
-                    )
-                bindist._oci_put_manifest(
-                    base_images, checksums, tagged_image, tmpdir, None, None, *roots
-                )
-                tty.info(f"Tagged {tagged_image}")
-
-        else:
-            skipped, upload_errors = bindist._push(
-                specs,
-                out_url=push_url,
-                force=args.force,
-                update_index=args.update_index,
-                signing_key=signing_key,
-                tmpdir=tmpdir,
-                executor=executor,
-            )
-            failed.extend(upload_errors)
+    with bindist.make_uploader(
+        mirror=mirror,
+        force=args.force,
+        update_index=args.update_index,
+        signing_key=signing_key,
+        base_image=args.base_image,
+    ) as uploader:
+        skipped, upload_errors = uploader.push(specs=specs)
+        failed.extend(upload_errors)
+        if not upload_errors and args.tag:
+            uploader.tag(args.tag, roots)
 
     if skipped:
         if len(specs) == 1:
@@ -501,7 +455,7 @@ def push_fn(args):
                 "The following {} specs were skipped as they already exist in the buildcache:\n"
                 "    {}\n"
                 "    Use --force to overwrite them.".format(
-                    len(skipped), ", ".join(elide_list(skipped, 5))
+                    len(skipped), ", ".join(elide_list([_format_spec(s) for s in skipped], 5))
                 )
             )
 
@@ -522,13 +476,6 @@ def push_fn(args):
             ),
         )
 
-    # Update the OCI index if requested
-    if target_image and len(skipped) < len(specs) and args.update_index:
-        with tempfile.TemporaryDirectory(
-            dir=spack.stage.get_stage_root()
-        ) as tmpdir, spack.util.parallel.make_concurrent_executor() as executor:
-            bindist._oci_update_index(target_image, tmpdir, executor)
-
 
 def install_fn(args):
     """install from a binary package"""
@@ -816,7 +763,7 @@ def update_index(mirror: spack.mirror.Mirror, update_keys=False):
     url = mirror.push_url
 
     with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:
-        bindist.generate_package_index(url, tmpdir)
+        bindist._url_generate_package_index(url, tmpdir)
 
     if update_keys:
         keys_url = url_util.join(

lib/spack/spack/cmd/change.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import spack.cmd
+import spack.spec
 from spack.cmd.common import arguments
 
 description = "change an existing spec in an environment"

lib/spack/spack/cmd/checksum.py

@@ -15,7 +15,6 @@
 import spack.repo
 import spack.spec
 import spack.stage
-import spack.util.crypto
 import spack.util.web as web_util
 from spack.cmd.common import arguments
 from spack.package_base import (

lib/spack/spack/cmd/ci.py

@@ -19,7 +19,6 @@
 import spack.cmd.buildcache as buildcache
 import spack.config as cfg
 import spack.environment as ev
-import spack.environment.depfile
 import spack.hash_types as ht
 import spack.mirror
 import spack.util.gpg as gpg_util

lib/spack/spack/cmd/clean.py

@@ -10,11 +10,9 @@
 import llnl.util.filesystem
 import llnl.util.tty as tty
 
-import spack.bootstrap
 import spack.caches
-import spack.cmd.test
+import spack.cmd
 import spack.config
-import spack.repo
 import spack.stage
 import spack.store
 import spack.util.path

lib/spack/spack/cmd/commands.py

@@ -7,6 +7,7 @@
 import copy
 import os
 import re
+import shlex
 import sys
 from argparse import ArgumentParser, Namespace
 from typing import IO, Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
@@ -16,8 +17,10 @@
 from llnl.util.tty.colify import colify
 
 import spack.cmd
+import spack.config
 import spack.main
 import spack.paths
+import spack.platforms
 from spack.main import section_descriptions
 
 description = "list available spack commands"
@@ -139,7 +142,7 @@ def usage(self, usage: str) -> str:
 
         cmd = self.parser.prog.replace(" ", "-")
         if cmd in self.documented:
-            string += "\n:ref:`More documentation <cmd-{0}>`\n".format(cmd)
+            string = f"{string}\n:ref:`More documentation <cmd-{cmd}>`\n"
 
         return string
 
@@ -249,33 +252,27 @@ def body(
             Function body.
         """
         if positionals:
-            return """
+            return f"""
     if $list_options
     then
-        {0}
+        {self.optionals(optionals)}
     else
-        {1}
+        {self.positionals(positionals)}
     fi
-""".format(
-                self.optionals(optionals), self.positionals(positionals)
-            )
+"""
         elif subcommands:
-            return """
+            return f"""
     if $list_options
     then
-        {0}
+        {self.optionals(optionals)}
     else
-        {1}
+        {self.subcommands(subcommands)}
     fi
-""".format(
-                self.optionals(optionals), self.subcommands(subcommands)
-            )
+"""
         else:
-            return """
-    {0}
-""".format(
-                self.optionals(optionals)
-            )
+            return f"""
+    {self.optionals(optionals)}
+"""
 
     def positionals(self, positionals: Sequence[str]) -> str:
         """Return the syntax for reporting positional arguments.
@@ -304,7 +301,7 @@ def optionals(self, optionals: Sequence[str]) -> str:
         Returns:
             Syntax for optional flags.
         """
-        return 'SPACK_COMPREPLY="{0}"'.format(" ".join(optionals))
+        return f'SPACK_COMPREPLY="{" ".join(optionals)}"'
 
     def subcommands(self, subcommands: Sequence[str]) -> str:
         """Return the syntax for reporting subcommands.
@@ -315,7 +312,7 @@ def subcommands(self, subcommands: Sequence[str]) -> str:
         Returns:
             Syntax for subcommand parsers
         """
-        return 'SPACK_COMPREPLY="{0}"'.format(" ".join(subcommands))
+        return f'SPACK_COMPREPLY="{" ".join(subcommands)}"'
 
 
 # Map argument destination names to their complete commands
@@ -395,7 +392,7 @@ def _fish_dest_get_complete(prog: str, dest: str) -> Optional[str]:
     subcmd = s[1] if len(s) == 2 else ""
 
     for (prog_key, pos_key), value in _dest_to_fish_complete.items():
-        if subcmd.startswith(prog_key) and re.match("^" + pos_key + "$", dest):
+        if subcmd.startswith(prog_key) and re.match(f"^{pos_key}$", dest):
             return value
     return None
 
@@ -427,24 +424,6 @@ def format(self, cmd: Command) -> str:
             + self.complete(cmd.prog, positionals, optionals, subcommands)
         )
 
-    def _quote(self, string: str) -> str:
-        """Quote string and escape special characters if necessary.
-
-        Args:
-            string: Input string.
-
-        Returns:
-            Quoted string.
-        """
-        # Goal here is to match fish_indent behavior
-
-        # Strings without spaces (or other special characters) do not need to be escaped
-        if not any([sub in string for sub in [" ", "'", '"']]):
-            return string
-
-        string = string.replace("'", r"\'")
-        return f"'{string}'"
-
     def optspecs(
         self,
         prog: str,
@@ -463,7 +442,7 @@ def optspecs(
         optspec_var = "__fish_spack_optspecs_" + prog.replace(" ", "_").replace("-", "_")
 
         if optionals is None:
-            return "set -g %s\n" % optspec_var
+            return f"set -g {optspec_var}\n"
 
         # Build optspec by iterating over options
         args = []
@@ -490,11 +469,11 @@ def optspecs(
             long = [f[2:] for f in flags if f.startswith("--")]
 
             while len(short) > 0 and len(long) > 0:
-                arg = "%s/%s%s" % (short.pop(), long.pop(), required)
+                arg = f"{short.pop()}/{long.pop()}{required}"
             while len(short) > 0:
-                arg = "%s/%s" % (short.pop(), required)
+                arg = f"{short.pop()}/{required}"
             while len(long) > 0:
-                arg = "%s%s" % (long.pop(), required)
+                arg = f"{long.pop()}{required}"
 
             args.append(arg)
 
@@ -503,7 +482,7 @@ def optspecs(
         # indicate that such subcommand exists.
         args = " ".join(args)
 
-        return "set -g %s %s\n" % (optspec_var, args)
+        return f"set -g {optspec_var} {args}\n"
 
     @staticmethod
     def complete_head(
@@ -524,12 +503,14 @@ def complete_head(
         subcmd = s[1] if len(s) == 2 else ""
 
         if index is None:
-            return "complete -c %s -n '__fish_spack_using_command %s'" % (s[0], subcmd)
+            return f"complete -c {s[0]} -n '__fish_spack_using_command {subcmd}'"
         elif nargs in [argparse.ZERO_OR_MORE, argparse.ONE_OR_MORE, argparse.REMAINDER]:
-            head = "complete -c %s -n '__fish_spack_using_command_pos_remainder %d %s'"
+            return (
+                f"complete -c {s[0]} -n '__fish_spack_using_command_pos_remainder "
+                f"{index} {subcmd}'"
+            )
         else:
-            head = "complete -c %s -n '__fish_spack_using_command_pos %d %s'"
-        return head % (s[0], index, subcmd)
+            return f"complete -c {s[0]} -n '__fish_spack_using_command_pos {index} {subcmd}'"
 
     def complete(
         self,
@@ -597,25 +578,18 @@ def positionals(
 
             if choices is not None:
                 # If there are choices, we provide a completion for all possible values.
-                commands.append(head + " -f -a %s" % self._quote(" ".join(choices)))
+                commands.append(f"{head} -f -a {shlex.quote(' '.join(choices))}")
             else:
                 # Otherwise, we try to find a predefined completion for it
                 value = _fish_dest_get_complete(prog, args)
                 if value is not None:
-                    commands.append(head + " " + value)
+                    commands.append(f"{head} {value}")
 
         return "\n".join(commands) + "\n"
 
     def prog_comment(self, prog: str) -> str:
-        """Return a comment line for the command.
-
-        Args:
-            prog: Program name.
-
-        Returns:
-            Comment line.
-        """
-        return "\n# %s\n" % prog
+        """Return a comment line for the command."""
+        return f"\n# {prog}\n"
 
     def optionals(
         self,
@@ -658,28 +632,28 @@ def optionals(
             for f in flags:
                 if f.startswith("--"):
                     long = f[2:]
-                    prefix += " -l %s" % long
+                    prefix = f"{prefix} -l {long}"
                 elif f.startswith("-"):
                     short = f[1:]
                     assert len(short) == 1
-                    prefix += " -s %s" % short
+                    prefix = f"{prefix} -s {short}"
 
             # Check if option require argument.
             # Currently multi-argument options are not supported, so we treat it like one argument.
             if nargs != 0:
-                prefix += " -r"
+                prefix = f"{prefix} -r"
 
             if dest is not None:
                 # If there are choices, we provide a completion for all possible values.
-                commands.append(prefix + " -f -a %s" % self._quote(" ".join(dest)))
+                commands.append(f"{prefix} -f -a {shlex.quote(' '.join(dest))}")
             else:
                 # Otherwise, we try to find a predefined completion for it
                 value = _fish_dest_get_complete(prog, dest)
                 if value is not None:
-                    commands.append(prefix + " " + value)
+                    commands.append(f"{prefix} {value}")
 
             if help:
-                commands.append(prefix + " -d %s" % self._quote(help))
+                commands.append(f"{prefix} -d {shlex.quote(help)}")
 
         return "\n".join(commands) + "\n"
 
@@ -697,11 +671,11 @@ def subcommands(self, prog: str, subcommands: List[Tuple[ArgumentParser, str, st
         head = self.complete_head(prog, 0)
 
         for _, subcommand, help in subcommands:
-            command = head + " -f -a %s" % self._quote(subcommand)
+            command = f"{head} -f -a {shlex.quote(subcommand)}"
 
             if help is not None and len(help) > 0:
                 help = help.split("\n")[0]
-                command += " -d %s" % self._quote(help)
+                command = f"{command} -d {shlex.quote(help)}"
 
             commands.append(command)
 
@@ -747,7 +721,7 @@ def rst_index(out: IO) -> None:
 
         for i, cmd in enumerate(sorted(commands)):
             description = description.capitalize() if i == 0 else ""
-            ref = ":ref:`%s <spack-%s>`" % (cmd, cmd)
+            ref = f":ref:`{cmd} <spack-{cmd}>`"
             comma = "," if i != len(commands) - 1 else ""
             bar = "| " if i % 8 == 0 else "  "
             out.write(line % (description, bar + ref + comma))
@@ -858,10 +832,10 @@ def _commands(parser: ArgumentParser, args: Namespace) -> None:
 
     # check header first so we don't open out files unnecessarily
     if args.header and not os.path.exists(args.header):
-        tty.die("No such file: '%s'" % args.header)
+        tty.die(f"No such file: '{args.header}'")
 
     if args.update:
-        tty.msg("Updating file: %s" % args.update)
+        tty.msg(f"Updating file: {args.update}")
         with open(args.update, "w") as f:
             prepend_header(args, f)
             formatter(args, f)

lib/spack/spack/cmd/common/arguments.py

@@ -15,7 +15,6 @@
 import spack.deptypes as dt
 import spack.environment as ev
 import spack.mirror
-import spack.modules
 import spack.reporters
 import spack.spec
 import spack.store

lib/spack/spack/cmd/common/confirmation.py

@@ -9,6 +9,7 @@
 import llnl.util.tty as tty
 
 import spack.cmd
+import spack.spec
 
 display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4}
 

lib/spack/spack/cmd/common/env_utility.py

@@ -10,7 +10,6 @@
 import spack.cmd
 import spack.deptypes as dt
 import spack.error
-import spack.paths
 import spack.spec
 import spack.store
 from spack import build_environment, traverse

lib/spack/spack/cmd/compiler.py

@@ -50,6 +50,7 @@ def setup_parser(subparser):
         default=lambda: spack.config.default_modify_scope("compilers"),
         help="configuration scope to modify",
     )
+    arguments.add_common_arguments(find_parser, ["jobs"])
 
     # Remove
     remove_parser = sp.add_parser("remove", aliases=["rm"], help="remove compiler by spec")
@@ -78,25 +79,21 @@ def setup_parser(subparser):
 def compiler_find(args):
     """Search either $PATH or a list of paths OR MODULES for compilers and
     add them to Spack's configuration.
-
     """
-    # None signals spack.compiler.find_compilers to use its default logic
     paths = args.add_paths or None
-
-    # Below scope=None because we want new compilers that don't appear
-    # in any other configuration.
-    new_compilers = spack.compilers.find_new_compilers(
-        paths, scope=None, mixed_toolchain=args.mixed_toolchain
+    new_compilers = spack.compilers.find_compilers(
+        path_hints=paths,
+        scope=args.scope,
+        mixed_toolchain=args.mixed_toolchain,
+        max_workers=args.jobs,
     )
     if new_compilers:
-        spack.compilers.add_compilers_to_config(new_compilers, scope=args.scope)
         n = len(new_compilers)
         s = "s" if n > 1 else ""
-
-        config = spack.config.CONFIG
-        filename = config.get_config_filename(args.scope, "compilers")
-        tty.msg("Added %d new compiler%s to %s" % (n, s, filename))
-        colify(reversed(sorted(c.spec.display_str for c in new_compilers)), indent=4)
+        filename = spack.config.CONFIG.get_config_filename(args.scope, "compilers")
+        tty.msg(f"Added {n:d} new compiler{s} to {filename}")
+        compiler_strs = sorted(f"{c.spec.name}@{c.spec.version}" for c in new_compilers)
+        colify(reversed(compiler_strs), indent=4)
     else:
         tty.msg("Found no new compilers")
     tty.msg("Compilers are defined in the following files:")

lib/spack/spack/cmd/config.py

@@ -13,9 +13,9 @@
 
 import spack.config
 import spack.environment as ev
-import spack.repo
+import spack.error
 import spack.schema.env
-import spack.schema.packages
+import spack.spec
 import spack.store
 import spack.util.spack_yaml as syaml
 from spack.cmd.common import arguments
@@ -256,7 +256,7 @@ def config_remove(args):
         existing.pop(value, None)
     else:
         # This should be impossible to reach
-        raise spack.config.ConfigError("Config has nested non-dict values")
+        raise spack.error.ConfigError("Config has nested non-dict values")
 
     spack.config.set(path, existing, scope)
 
@@ -340,7 +340,7 @@ def _config_change(config_path, match_spec_str=None):
         if not changed:
             existing_requirements = spack.config.get(key_path)
             if isinstance(existing_requirements, str):
-                raise spack.config.ConfigError(
+                raise spack.error.ConfigError(
                     "'config change' needs to append a requirement,"
                     " but existing require: config is not a list"
                 )
@@ -536,11 +536,11 @@ def config_prefer_upstream(args):
         # Get and list all the variants that differ from the default.
         variants = []
         for var_name, variant in spec.variants.items():
-            if var_name in ["patches"] or var_name not in spec.package.variants:
+            if var_name in ["patches"] or not spec.package.has_variant(var_name):
                 continue
 
-            variant_desc, _ = spec.package.variants[var_name]
-            if variant.value != variant_desc.default:
+            vdef = spec.package.get_variant(var_name)
+            if variant.value != vdef.default:
                 variants.append(str(variant))
         variants.sort()
         variants = " ".join(variants)

lib/spack/spack/cmd/create.py

@@ -6,17 +6,23 @@
 import re
 import sys
 import urllib.parse
+from typing import List
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import mkdirp
 
 import spack.repo
 import spack.stage
-import spack.util.web
 from spack.spec import Spec
-from spack.url import UndetectableNameError, UndetectableVersionError, parse_name, parse_version
+from spack.url import (
+    UndetectableNameError,
+    UndetectableVersionError,
+    find_versions_of_archive,
+    parse_name,
+    parse_version,
+)
 from spack.util.editor import editor
-from spack.util.executable import ProcessError, which
+from spack.util.executable import which
 from spack.util.format import get_version_lines
 from spack.util.naming import mod_to_class, simplify_name, valid_fully_qualified_module_name
 
@@ -89,14 +95,20 @@ class BundlePackageTemplate:
     url_def = "    # There is no URL since there is no code to download."
     body_def = "    # There is no need for install() since there is no code."
 
-    def __init__(self, name, versions):
+    def __init__(self, name: str, versions, languages: List[str]):
         self.name = name
         self.class_name = mod_to_class(name)
         self.versions = versions
+        self.languages = languages
 
     def write(self, pkg_path):
         """Writes the new package file."""
 
+        all_deps = [f'    depends_on("{lang}", type="build")' for lang in self.languages]
+        if all_deps and self.dependencies:
+            all_deps.append("")
+        all_deps.append(self.dependencies)
+
         # Write out a template for the file
         with open(pkg_path, "w") as pkg_file:
             pkg_file.write(
@@ -106,7 +118,7 @@ def write(self, pkg_path):
                     base_class_name=self.base_class_name,
                     url_def=self.url_def,
                     versions=self.versions,
-                    dependencies=self.dependencies,
+                    dependencies="\n".join(all_deps),
                     body_def=self.body_def,
                 )
             )
@@ -125,8 +137,8 @@ def install(self, spec, prefix):
 
     url_line = '    url = "{url}"'
 
-    def __init__(self, name, url, versions):
-        super().__init__(name, versions)
+    def __init__(self, name, url, versions, languages: List[str]):
+        super().__init__(name, versions, languages)
 
         self.url_def = self.url_line.format(url=url)
 
@@ -214,13 +226,13 @@ def luarocks_args(self):
         args = []
         return args"""
 
-    def __init__(self, name, url, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name lua-lpeg`, don't rename it lua-lua-lpeg
         if not name.startswith("lua-"):
             # Make it more obvious that we are renaming the package
             tty.msg("Changing package name from {0} to lua-{0}".format(name))
             name = "lua-{0}".format(name)
-        super().__init__(name, url, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class MesonPackageTemplate(PackageTemplate):
@@ -321,14 +333,14 @@ class RacketPackageTemplate(PackageTemplate):
     # subdirectory = None
     """
 
-    def __init__(self, name, url, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name rkt-scribble`, don't rename it rkt-rkt-scribble
         if not name.startswith("rkt-"):
             # Make it more obvious that we are renaming the package
             tty.msg("Changing package name from {0} to rkt-{0}".format(name))
             name = "rkt-{0}".format(name)
         self.body_def = self.body_def.format(name[4:])
-        super().__init__(name, url, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class PythonPackageTemplate(PackageTemplate):
@@ -361,7 +373,7 @@ def config_settings(self, spec, prefix):
         settings = {}
         return settings"""
 
-    def __init__(self, name, url, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name py-numpy`, don't rename it py-py-numpy
         if not name.startswith("py-"):
             # Make it more obvious that we are renaming the package
@@ -415,7 +427,7 @@ def __init__(self, name, url, *args, **kwargs):
                 + self.url_line
             )
 
-        super().__init__(name, url, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class RPackageTemplate(PackageTemplate):
@@ -434,7 +446,7 @@ def configure_args(self):
         args = []
         return args"""
 
-    def __init__(self, name, url, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name r-rcpp`, don't rename it r-r-rcpp
         if not name.startswith("r-"):
             # Make it more obvious that we are renaming the package
@@ -454,7 +466,7 @@ def __init__(self, name, url, *args, **kwargs):
         if bioc:
             self.url_line = '    url = "{0}"\n' '    bioc = "{1}"'.format(url, r_name)
 
-        super().__init__(name, url, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class PerlmakePackageTemplate(PackageTemplate):
@@ -474,14 +486,14 @@ def configure_args(self):
         args = []
         return args"""
 
-    def __init__(self, name, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name perl-cpp`, don't rename it perl-perl-cpp
         if not name.startswith("perl-"):
             # Make it more obvious that we are renaming the package
             tty.msg("Changing package name from {0} to perl-{0}".format(name))
             name = "perl-{0}".format(name)
 
-        super().__init__(name, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class PerlbuildPackageTemplate(PerlmakePackageTemplate):
@@ -506,7 +518,7 @@ class OctavePackageTemplate(PackageTemplate):
     # FIXME: Add additional dependencies if required.
     # depends_on("octave-foo", type=("build", "run"))"""
 
-    def __init__(self, name, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name octave-splines`, don't rename it
         # octave-octave-splines
         if not name.startswith("octave-"):
@@ -514,7 +526,7 @@ def __init__(self, name, *args, **kwargs):
             tty.msg("Changing package name from {0} to octave-{0}".format(name))
             name = "octave-{0}".format(name)
 
-        super().__init__(name, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class RubyPackageTemplate(PackageTemplate):
@@ -534,7 +546,7 @@ def build(self, spec, prefix):
         # FIXME: If not needed delete this function
         pass"""
 
-    def __init__(self, name, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name ruby-numpy`, don't rename it
         # ruby-ruby-numpy
         if not name.startswith("ruby-"):
@@ -542,7 +554,7 @@ def __init__(self, name, *args, **kwargs):
             tty.msg("Changing package name from {0} to ruby-{0}".format(name))
             name = "ruby-{0}".format(name)
 
-        super().__init__(name, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class MakefilePackageTemplate(PackageTemplate):
@@ -580,14 +592,14 @@ def configure_args(self, spec, prefix):
         args = []
         return args"""
 
-    def __init__(self, name, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name py-pyqt4`, don't rename it py-py-pyqt4
         if not name.startswith("py-"):
             # Make it more obvious that we are renaming the package
             tty.msg("Changing package name from {0} to py-{0}".format(name))
             name = "py-{0}".format(name)
 
-        super().__init__(name, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 templates = {
@@ -658,8 +670,48 @@ def setup_parser(subparser):
     )
 
 
-class BuildSystemGuesser:
-    """An instance of BuildSystemGuesser provides a callable object to be used
+#: C file extensions
+C_EXT = {".c"}
+
+#: C++ file extensions
+CXX_EXT = {
+    ".C",
+    ".c++",
+    ".cc",
+    ".ccm",
+    ".cpp",
+    ".CPP",
+    ".cxx",
+    ".h++",
+    ".hh",
+    ".hpp",
+    ".hxx",
+    ".inl",
+    ".ipp",
+    ".ixx",
+    ".tcc",
+    ".tpp",
+}
+
+#: Fortran file extensions
+FORTRAN_EXT = {
+    ".f77",
+    ".F77",
+    ".f90",
+    ".F90",
+    ".f95",
+    ".F95",
+    ".f",
+    ".F",
+    ".for",
+    ".FOR",
+    ".ftn",
+    ".FTN",
+}
+
+
+class BuildSystemAndLanguageGuesser:
+    """An instance of BuildSystemAndLanguageGuesser provides a callable object to be used
     during ``spack create``. By passing this object to ``spack checksum``, we
     can take a peek at the fetched tarball and discern the build system it uses
     """
@@ -667,81 +719,119 @@ class BuildSystemGuesser:
     def __init__(self):
         """Sets the default build system."""
         self.build_system = "generic"
+        self._c = False
+        self._cxx = False
+        self._fortran = False
 
-    def __call__(self, stage, url):
+        # List of files in the archive ordered by their depth in the directory tree.
+        self._file_entries: List[str] = []
+
+    def __call__(self, archive: str, url: str) -> None:
         """Try to guess the type of build system used by a project based on
         the contents of its archive or the URL it was downloaded from."""
 
-        if url is not None:
-            # Most octave extensions are hosted on Octave-Forge:
-            #     https://octave.sourceforge.net/index.html
-            # They all have the same base URL.
-            if "downloads.sourceforge.net/octave/" in url:
-                self.build_system = "octave"
-                return
-            if url.endswith(".gem"):
-                self.build_system = "ruby"
-                return
-            if url.endswith(".whl") or ".whl#" in url:
-                self.build_system = "python"
-                return
-            if url.endswith(".rock"):
-                self.build_system = "lua"
-                return
-
-        # A list of clues that give us an idea of the build system a package
-        # uses. If the regular expression matches a file contained in the
-        # archive, the corresponding build system is assumed.
-        # NOTE: Order is important here. If a package supports multiple
-        # build systems, we choose the first match in this list.
-        clues = [
-            (r"/CMakeLists\.txt$", "cmake"),
-            (r"/NAMESPACE$", "r"),
-            (r"/Cargo\.toml$", "cargo"),
-            (r"/go\.mod$", "go"),
-            (r"/configure$", "autotools"),
-            (r"/configure\.(in|ac)$", "autoreconf"),
-            (r"/Makefile\.am$", "autoreconf"),
-            (r"/pom\.xml$", "maven"),
-            (r"/SConstruct$", "scons"),
-            (r"/waf$", "waf"),
-            (r"/pyproject.toml", "python"),
-            (r"/setup\.(py|cfg)$", "python"),
-            (r"/WORKSPACE$", "bazel"),
-            (r"/Build\.PL$", "perlbuild"),
-            (r"/Makefile\.PL$", "perlmake"),
-            (r"/.*\.gemspec$", "ruby"),
-            (r"/Rakefile$", "ruby"),
-            (r"/setup\.rb$", "ruby"),
-            (r"/.*\.pro$", "qmake"),
-            (r"/.*\.rockspec$", "lua"),
-            (r"/(GNU)?[Mm]akefile$", "makefile"),
-            (r"/DESCRIPTION$", "octave"),
-            (r"/meson\.build$", "meson"),
-            (r"/configure\.py$", "sip"),
-        ]
-
         # Peek inside the compressed file.
-        if stage.archive_file.endswith(".zip") or ".zip#" in stage.archive_file:
+        if archive.endswith(".zip") or ".zip#" in archive:
             try:
                 unzip = which("unzip")
-                output = unzip("-lq", stage.archive_file, output=str)
-            except ProcessError:
+                assert unzip is not None
+                output = unzip("-lq", archive, output=str)
+            except Exception:
                 output = ""
         else:
             try:
                 tar = which("tar")
-                output = tar("--exclude=*/*/*", "-tf", stage.archive_file, output=str)
-            except ProcessError:
+                assert tar is not None
+                output = tar("tf", archive, output=str)
+            except Exception:
                 output = ""
-        lines = output.splitlines()
+        self._file_entries[:] = output.splitlines()
 
-        # Determine the build system based on the files contained
-        # in the archive.
-        for pattern, bs in clues:
-            if any(re.search(pattern, line) for line in lines):
-                self.build_system = bs
-                break
+        # Files closest to the root should be considered first when determining build system.
+        self._file_entries.sort(key=lambda p: p.count("/"))
+
+        self._determine_build_system(url)
+        self._determine_language()
+
+    def _determine_build_system(self, url: str) -> None:
+        # Most octave extensions are hosted on Octave-Forge:
+        #     https://octave.sourceforge.net/index.html
+        # They all have the same base URL.
+        if "downloads.sourceforge.net/octave/" in url:
+            self.build_system = "octave"
+        elif url.endswith(".gem"):
+            self.build_system = "ruby"
+        elif url.endswith(".whl") or ".whl#" in url:
+            self.build_system = "python"
+        elif url.endswith(".rock"):
+            self.build_system = "lua"
+        elif self._file_entries:
+            # A list of clues that give us an idea of the build system a package
+            # uses. If the regular expression matches a file contained in the
+            # archive, the corresponding build system is assumed.
+            # NOTE: Order is important here. If a package supports multiple
+            # build systems, we choose the first match in this list.
+            clues = [
+                (re.compile(pattern), build_system)
+                for pattern, build_system in (
+                    (r"/CMakeLists\.txt$", "cmake"),
+                    (r"/NAMESPACE$", "r"),
+                    (r"/Cargo\.toml$", "cargo"),
+                    (r"/go\.mod$", "go"),
+                    (r"/configure$", "autotools"),
+                    (r"/configure\.(in|ac)$", "autoreconf"),
+                    (r"/Makefile\.am$", "autoreconf"),
+                    (r"/pom\.xml$", "maven"),
+                    (r"/SConstruct$", "scons"),
+                    (r"/waf$", "waf"),
+                    (r"/pyproject.toml", "python"),
+                    (r"/setup\.(py|cfg)$", "python"),
+                    (r"/WORKSPACE$", "bazel"),
+                    (r"/Build\.PL$", "perlbuild"),
+                    (r"/Makefile\.PL$", "perlmake"),
+                    (r"/.*\.gemspec$", "ruby"),
+                    (r"/Rakefile$", "ruby"),
+                    (r"/setup\.rb$", "ruby"),
+                    (r"/.*\.pro$", "qmake"),
+                    (r"/.*\.rockspec$", "lua"),
+                    (r"/(GNU)?[Mm]akefile$", "makefile"),
+                    (r"/DESCRIPTION$", "octave"),
+                    (r"/meson\.build$", "meson"),
+                    (r"/configure\.py$", "sip"),
+                )
+            ]
+
+            # Determine the build system based on the files contained in the archive.
+            for file in self._file_entries:
+                for pattern, build_system in clues:
+                    if pattern.search(file):
+                        self.build_system = build_system
+                        return
+
+    def _determine_language(self):
+        for entry in self._file_entries:
+            _, ext = os.path.splitext(entry)
+
+            if not self._c and ext in C_EXT:
+                self._c = True
+            elif not self._cxx and ext in CXX_EXT:
+                self._cxx = True
+            elif not self._fortran and ext in FORTRAN_EXT:
+                self._fortran = True
+
+            if self._c and self._cxx and self._fortran:
+                return
+
+    @property
+    def languages(self) -> List[str]:
+        langs: List[str] = []
+        if self._c:
+            langs.append("c")
+        if self._cxx:
+            langs.append("cxx")
+        if self._fortran:
+            langs.append("fortran")
+        return langs
 
 
 def get_name(name, url):
@@ -811,7 +901,7 @@ def get_url(url):
 def get_versions(args, name):
     """Returns a list of versions and hashes for a package.
 
-    Also returns a BuildSystemGuesser object.
+    Also returns a BuildSystemAndLanguageGuesser object.
 
     Returns default values if no URL is provided.
 
@@ -820,7 +910,7 @@ def get_versions(args, name):
         name (str): The name of the package
 
     Returns:
-        tuple: versions and hashes, and a BuildSystemGuesser object
+        tuple: versions and hashes, and a BuildSystemAndLanguageGuesser object
     """
 
     # Default version with hash
@@ -834,7 +924,7 @@ def get_versions(args, name):
     # version("1.2.4")"""
 
     # Default guesser
-    guesser = BuildSystemGuesser()
+    guesser = BuildSystemAndLanguageGuesser()
 
     valid_url = True
     try:
@@ -847,7 +937,7 @@ def get_versions(args, name):
     if args.url is not None and args.template != "bundle" and valid_url:
         # Find available versions
         try:
-            url_dict = spack.url.find_versions_of_archive(args.url)
+            url_dict = find_versions_of_archive(args.url)
             if len(url_dict) > 1 and not args.batch and sys.stdin.isatty():
                 url_dict_filtered = spack.stage.interactive_version_filter(url_dict)
                 if url_dict_filtered is None:
@@ -874,7 +964,7 @@ def get_versions(args, name):
     return versions, guesser
 
 
-def get_build_system(template, url, guesser):
+def get_build_system(template: str, url: str, guesser: BuildSystemAndLanguageGuesser) -> str:
     """Determine the build system template.
 
     If a template is specified, always use that. Otherwise, if a URL
@@ -882,11 +972,10 @@ def get_build_system(template, url, guesser):
     build system it uses. Otherwise, use a generic template by default.
 
     Args:
-        template (str): ``--template`` argument given to ``spack create``
-        url (str): ``url`` argument given to ``spack create``
-        args (argparse.Namespace): The arguments given to ``spack create``
-        guesser (BuildSystemGuesser): The first_stage_function given to
-            ``spack checksum`` which records the build system it detects
+        template: ``--template`` argument given to ``spack create``
+        url: ``url`` argument given to ``spack create``
+        guesser: The first_stage_function given to ``spack checksum`` which records the build
+            system it detects
 
     Returns:
         str: The name of the build system template to use
@@ -960,7 +1049,7 @@ def create(parser, args):
     build_system = get_build_system(args.template, url, guesser)
 
     # Create the package template object
-    constr_args = {"name": name, "versions": versions}
+    constr_args = {"name": name, "versions": versions, "languages": guesser.languages}
     package_class = templates[build_system]
     if package_class != BundlePackageTemplate:
         constr_args["url"] = url

lib/spack/spack/cmd/debug.py

@@ -13,11 +13,12 @@
 import llnl.util.tty as tty
 from llnl.util.filesystem import working_dir
 
-import spack.config
+import spack
 import spack.paths
 import spack.platforms
+import spack.spec
+import spack.store
 import spack.util.git
-from spack.main import get_version
 from spack.util.executable import which
 
 description = "debugging commands for troubleshooting Spack"
@@ -89,7 +90,7 @@ def report(args):
     host_os = host_platform.operating_system("frontend")
     host_target = host_platform.target("frontend")
     architecture = spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))
-    print("* **Spack:**", get_version())
+    print("* **Spack:**", spack.get_version())
     print("* **Python:**", platform.python_version())
     print("* **Platform:**", architecture)
 

lib/spack/spack/cmd/dependencies.py

@@ -11,7 +11,6 @@
 import spack.cmd
 import spack.environment as ev
 import spack.package_base
-import spack.repo
 import spack.store
 from spack.cmd.common import arguments
 

lib/spack/spack/cmd/deprecate.py

@@ -14,13 +14,13 @@
 installation and its deprecator.
 """
 import argparse
-import os
 
 import llnl.util.tty as tty
 from llnl.util.symlink import symlink
 
 import spack.cmd
 import spack.environment as ev
+import spack.installer
 import spack.store
 from spack.cmd.common import arguments
 from spack.database import InstallStatuses
@@ -76,12 +76,7 @@ def setup_parser(sp):
     )
 
     sp.add_argument(
-        "-l",
-        "--link-type",
-        type=str,
-        default="soft",
-        choices=["soft", "hard"],
-        help="type of filesystem link to use for deprecation (default soft)",
+        "-l", "--link-type", type=str, default=None, choices=["soft", "hard"], help="(deprecated)"
     )
 
     sp.add_argument(
@@ -91,6 +86,9 @@ def setup_parser(sp):
 
 def deprecate(parser, args):
     """Deprecate one spec in favor of another"""
+    if args.link_type is not None:
+        tty.warn("The --link-type option is deprecated and will be removed in a future release.")
+
     env = ev.active_environment()
     specs = spack.cmd.parse_specs(args.specs)
 
@@ -144,7 +142,5 @@ def deprecate(parser, args):
         if not answer:
             tty.die("Will not deprecate any packages.")
 
-    link_fn = os.link if args.link_type == "hard" else symlink
-
     for dcate, dcator in zip(all_deprecate, all_deprecators):
-        dcate.package.do_deprecate(dcator, link_fn)
+        spack.installer.deprecate(dcate, dcator, symlink)

lib/spack/spack/cmd/dev_build.py

@@ -8,10 +8,13 @@
 
 import llnl.util.tty as tty
 
+import spack.build_environment
 import spack.cmd
+import spack.cmd.common.arguments
 import spack.config
 import spack.repo
 from spack.cmd.common import arguments
+from spack.installer import PackageInstaller
 
 description = "developer build: build from code in current working directory"
 section = "build"
@@ -129,9 +132,9 @@ def dev_build(self, args):
     elif args.test == "root":
         tests = [spec.name for spec in specs]
 
-    spec.package.do_install(
+    PackageInstaller(
+        [spec.package],
         tests=tests,
-        make_jobs=args.jobs,
         keep_prefix=args.keep_prefix,
         install_deps=not args.ignore_deps,
         verbose=not args.quiet,
@@ -139,7 +142,7 @@ def dev_build(self, args):
         stop_before=args.before,
         skip_patch=args.skip_patch,
         stop_at=args.until,
-    )
+    ).install()
 
     # drop into the build environment of the package?
     if args.shell is not None:

lib/spack/spack/cmd/develop.py

@@ -12,6 +12,7 @@
 import spack.fetch_strategy
 import spack.repo
 import spack.spec
+import spack.stage
 import spack.util.path
 import spack.version
 from spack.cmd.common import arguments
@@ -62,7 +63,7 @@ def change_fn(section):
     spack.config.change_or_add("develop", find_fn, change_fn)
 
 
-def _retrieve_develop_source(spec, abspath):
+def _retrieve_develop_source(spec: spack.spec.Spec, abspath: str) -> None:
     # "steal" the source code via staging API. We ask for a stage
     # to be created, then copy it afterwards somewhere else. It would be
     # better if we can create the `source_path` directly into its final
@@ -71,13 +72,13 @@ def _retrieve_develop_source(spec, abspath):
     # We construct a package class ourselves, rather than asking for
     # Spec.package, since Spec only allows this when it is concrete
     package = pkg_cls(spec)
-    source_stage = package.stage[0]
+    source_stage: spack.stage.Stage = package.stage[0]
     if isinstance(source_stage.fetcher, spack.fetch_strategy.GitFetchStrategy):
         source_stage.fetcher.get_full_repo = True
         # If we retrieved this version before and cached it, we may have
         # done so without cloning the full git repo; likewise, any
         # mirror might store an instance with truncated history.
-        source_stage.disable_mirrors()
+        source_stage.default_fetcher_only = True
 
     source_stage.fetcher.set_package(package)
     package.stage.steal_source(abspath)

lib/spack/spack/cmd/diff.py

@@ -12,7 +12,6 @@
 import spack.cmd
 import spack.environment as ev
 import spack.solver.asp as asp
-import spack.util.environment
 import spack.util.spack_json as sjson
 from spack.cmd.common import arguments
 

lib/spack/spack/cmd/env.py

@@ -21,15 +21,12 @@
 import spack.cmd
 import spack.cmd.common
 import spack.cmd.common.arguments
-import spack.cmd.install
 import spack.cmd.modules
-import spack.cmd.uninstall
 import spack.config
 import spack.environment as ev
 import spack.environment.depfile as depfile
+import spack.environment.environment
 import spack.environment.shell
-import spack.schema.env
-import spack.spec
 import spack.tengine
 from spack.cmd.common import arguments
 from spack.util.environment import EnvironmentModifications
@@ -468,32 +465,30 @@ def env_remove(args):
     This removes an environment managed by Spack. Directory environments
     and manifests embedded in repositories should be removed manually.
     """
-    read_envs = []
+    remove_envs = []
     valid_envs = []
     bad_envs = []
-    invalid_envs = []
 
     for env_name in ev.all_environment_names():
         try:
             env = ev.read(env_name)
-            valid_envs.append(env_name)
+            valid_envs.append(env)
 
             if env_name in args.rm_env:
-                read_envs.append(env)
+                remove_envs.append(env)
         except (spack.config.ConfigFormatError, ev.SpackEnvironmentConfigError):
-            invalid_envs.append(env_name)
-
             if env_name in args.rm_env:
                 bad_envs.append(env_name)
 
-        # Check if env is linked to another before trying to remove
-        for name in valid_envs:
+    # Check if remove_env is included from another env before trying to remove
+    for env in valid_envs:
+        for remove_env in remove_envs:
             # don't check if environment is included to itself
-            if name == env_name:
+            if env.name == remove_env.name:
                 continue
-            environ = ev.Environment(ev.root(name))
-            if ev.root(env_name) in environ.included_concrete_envs:
-                msg = f'Environment "{env_name}" is being used by environment "{name}"'
+
+            if remove_env.path in env.included_concrete_envs:
+                msg = f'Environment "{remove_env.name}" is being used by environment "{env.name}"'
                 if args.force:
                     tty.warn(msg)
                 else:
@@ -506,7 +501,7 @@ def env_remove(args):
         if not answer:
             tty.die("Will not remove any environments")
 
-    for env in read_envs:
+    for env in remove_envs:
         name = env.name
         if env.active:
             tty.die(f"Environment {name} can't be removed while activated.")

lib/spack/spack/cmd/external.py

@@ -18,9 +18,9 @@
 import spack.cray_manifest as cray_manifest
 import spack.detection
 import spack.error
+import spack.package_base
 import spack.repo
 import spack.spec
-import spack.util.environment
 from spack.cmd.common import arguments
 
 description = "manage external packages in Spack configuration"

lib/spack/spack/cmd/fetch.py

@@ -8,7 +8,6 @@
 import spack.cmd
 import spack.config
 import spack.environment as ev
-import spack.repo
 import spack.traverse
 from spack.cmd.common import arguments
 

lib/spack/spack/cmd/find.py

@@ -10,10 +10,11 @@
 import llnl.util.tty as tty
 import llnl.util.tty.color as color
 
-import spack.bootstrap
 import spack.cmd as cmd
+import spack.config
 import spack.environment as ev
 import spack.repo
+import spack.spec
 import spack.store
 from spack.cmd.common import arguments
 from spack.database import InstallStatuses

lib/spack/spack/cmd/gpg.py

@@ -224,7 +224,7 @@ def gpg_publish(args):
         mirror = spack.mirror.Mirror(args.mirror_url, args.mirror_url)
 
     with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:
-        spack.binary_distribution.push_keys(
+        spack.binary_distribution._url_push_keys(
             mirror, keys=args.keys, tmpdir=tmpdir, update_index=args.update_index
         )
 

lib/spack/spack/cmd/info.py

@@ -16,6 +16,7 @@
 import spack.install_test
 import spack.repo
 import spack.spec
+import spack.variant
 import spack.version
 from spack.cmd.common import arguments
 from spack.package_base import preferred_version
@@ -48,6 +49,7 @@ def setup_parser(subparser):
     options = [
         ("--detectable", print_detectable.__doc__),
         ("--maintainers", print_maintainers.__doc__),
+        ("--namespace", print_namespace.__doc__),
         ("--no-dependencies", "do not " + print_dependencies.__doc__),
         ("--no-variants", "do not " + print_variants.__doc__),
         ("--no-versions", "do not " + print_versions.__doc__),
@@ -189,6 +191,15 @@ def print_maintainers(pkg, args):
         color.cprint(section_title("Maintainers: ") + mnt)
 
 
+def print_namespace(pkg, args):
+    """output package namespace"""
+
+    repo = spack.repo.PATH.get_repo(pkg.namespace)
+    color.cprint("")
+    color.cprint(section_title("Namespace:"))
+    color.cprint(f"    @c{{{repo.namespace}}} at {repo.root}")
+
+
 def print_phases(pkg, args):
     """output installation phases"""
 
@@ -323,26 +334,6 @@ def _fmt_variant(variant, max_name_default_len, indent, when=None, out=None):
     out.write("\n")
 
 
-def _variants_by_name_when(pkg):
-    """Adaptor to get variants keyed by { name: { when: { [Variant...] } }."""
-    # TODO: replace with pkg.variants_by_name(when=True) when unified directive dicts are merged.
-    variants = {}
-    for name, (variant, whens) in sorted(pkg.variants.items()):
-        for when in whens:
-            variants.setdefault(name, {}).setdefault(when, []).append(variant)
-    return variants
-
-
-def _variants_by_when_name(pkg):
-    """Adaptor to get variants keyed by { when: { name: Variant } }"""
-    # TODO: replace with pkg.variants when unified directive dicts are merged.
-    variants = {}
-    for name, (variant, whens) in pkg.variants.items():
-        for when in whens:
-            variants.setdefault(when, {})[name] = variant
-    return variants
-
-
 def _print_variants_header(pkg):
     """output variants"""
 
@@ -353,32 +344,22 @@ def _print_variants_header(pkg):
     color.cprint("")
     color.cprint(section_title("Variants:"))
 
-    variants_by_name = _variants_by_name_when(pkg)
-
     # Calculate the max length of the "name [default]" part of the variant display
     # This lets us know where to print variant values.
     max_name_default_len = max(
         color.clen(_fmt_name_and_default(variant))
-        for name, when_variants in variants_by_name.items()
-        for variants in when_variants.values()
-        for variant in variants
+        for name in pkg.variant_names()
+        for _, variant in pkg.variant_definitions(name)
     )
 
-    return max_name_default_len, variants_by_name
-
-
-def _unconstrained_ver_first(item):
-    """sort key that puts specs with open version ranges first"""
-    spec, _ = item
-    return (spack.version.any_version not in spec.versions, spec)
+    return max_name_default_len
 
 
 def print_variants_grouped_by_when(pkg):
-    max_name_default_len, _ = _print_variants_header(pkg)
+    max_name_default_len = _print_variants_header(pkg)
 
     indent = 4
-    variants = _variants_by_when_name(pkg)
-    for when, variants_by_name in sorted(variants.items(), key=_unconstrained_ver_first):
+    for when, variants_by_name in pkg.variant_items():
         padded_values = max_name_default_len + 4
         start_indent = indent
 
@@ -396,15 +377,14 @@ def print_variants_grouped_by_when(pkg):
 
 
 def print_variants_by_name(pkg):
-    max_name_default_len, variants_by_name = _print_variants_header(pkg)
+    max_name_default_len = _print_variants_header(pkg)
     max_name_default_len += 4
 
     indent = 4
-    for name, when_variants in variants_by_name.items():
-        for when, variants in sorted(when_variants.items(), key=_unconstrained_ver_first):
-            for variant in variants:
-                _fmt_variant(variant, max_name_default_len, indent, when, out=sys.stdout)
-                sys.stdout.write("\n")
+    for name in pkg.variant_names():
+        for when, variant in pkg.variant_definitions(name):
+            _fmt_variant(variant, max_name_default_len, indent, when, out=sys.stdout)
+            sys.stdout.write("\n")
 
 
 def print_variants(pkg, args):
@@ -502,7 +482,7 @@ def print_licenses(pkg, args):
 
 def info(parser, args):
     spec = spack.spec.Spec(args.package)
-    pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)
+    pkg_cls = spack.repo.PATH.get_pkg_class(spec.fullname)
     pkg = pkg_cls(spec)
 
     # Output core package information
@@ -522,6 +502,7 @@ def info(parser, args):
     # Now output optional information in expected order
     sections = [
         (args.all or args.maintainers, print_maintainers),
+        (args.all or args.namespace, print_namespace),
         (args.all or args.detectable, print_detectable),
         (args.all or args.tags, print_tags),
         (args.all or not args.no_versions, print_versions),

lib/spack/spack/cmd/install.py

@@ -13,18 +13,15 @@
 from llnl.string import plural
 from llnl.util import lang, tty
 
-import spack.build_environment
 import spack.cmd
 import spack.config
 import spack.environment as ev
-import spack.fetch_strategy
-import spack.package_base
 import spack.paths
 import spack.report
 import spack.spec
 import spack.store
 from spack.cmd.common import arguments
-from spack.error import SpackError
+from spack.error import InstallError, SpackError
 from spack.installer import PackageInstaller
 
 description = "build and install packages"
@@ -287,7 +284,7 @@ def require_user_confirmation_for_overwrite(concrete_specs, args):
         tty.die("Reinstallation aborted.")
 
 
-def _dump_log_on_error(e: spack.build_environment.InstallError):
+def _dump_log_on_error(e: InstallError):
     e.print_context()
     assert e.pkg, "Expected InstallError to include the associated package"
     if not os.path.exists(e.pkg.log_path):
@@ -352,7 +349,7 @@ def reporter_factory(specs):
             install_with_active_env(env, args, install_kwargs, reporter_factory)
         else:
             install_without_active_env(args, install_kwargs, reporter_factory)
-    except spack.build_environment.InstallError as e:
+    except InstallError as e:
         if args.show_log_on_error:
             _dump_log_on_error(e)
         raise
@@ -477,5 +474,5 @@ def install_without_active_env(args, install_kwargs, reporter_factory):
 
         installs = [s.package for s in concrete_specs]
         install_kwargs["explicit"] = [s.dag_hash() for s in concrete_specs]
-        builder = PackageInstaller(installs, install_kwargs)
+        builder = PackageInstaller(installs, **install_kwargs)
         builder.install()

lib/spack/spack/cmd/load.py

@@ -6,11 +6,10 @@
 import sys
 
 import spack.cmd
-import spack.cmd.find
+import spack.cmd.common
 import spack.environment as ev
 import spack.store
 import spack.user_environment as uenv
-import spack.util.environment
 from spack.cmd.common import arguments
 
 description = "add package to the user environment"