ginkgo: revert mpi@3.1 dependency:

Currently this breaks builds with OpenMPI - as spack is resolving this to 'openmpi@1.7.4'

.github/dependabot.yml

@@ -5,14 +5,10 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
-  # Requirements to build documentation
-  - package-ecosystem: "pip"
-    directory: "/lib/spack/docs"
-    schedule:
-      interval: "daily"
-  # Requirements to run style checks
+  # Requirements to run style checks and build documentation
   - package-ecosystem: "pip"
     directories:
-     - "/.github/workflows/requirements/*"
+     - "/.github/workflows/requirements/style/*"
+     - "/lib/spack/docs"
     schedule:
       interval: "daily"

.github/workflows/audit.yaml

@@ -29,7 +29,7 @@ jobs:
         shell: ${{ matrix.system.shell }}
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: ${{inputs.python_version}}
     - name: Install Python packages
@@ -44,6 +44,7 @@ jobs:
       run: |
           . share/spack/setup-env.sh
           coverage run $(which spack) audit packages
+          coverage run $(which spack) audit configs
           coverage run $(which spack) -d audit externals
           coverage combine
           coverage xml
@@ -52,6 +53,7 @@ jobs:
       run: |
           . share/spack/setup-env.sh          
           spack -d audit packages
+          spack -d audit configs
           spack -d audit externals
     - name: Package audits (without coverage)
       if: ${{ runner.os == 'Windows' }}
@@ -59,6 +61,8 @@ jobs:
           . share/spack/setup-env.sh          
           spack -d audit packages
           ./share/spack/qa/validate_last_exit.ps1
+          spack -d audit configs
+          ./share/spack/qa/validate_last_exit.ps1
           spack -d audit externals
           ./share/spack/qa/validate_last_exit.ps1
     - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673

.github/workflows/bootstrap.yml

@@ -63,7 +63,7 @@ jobs:
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
         with:
           python-version: "3.12"
       - name: Bootstrap clingo
@@ -112,10 +112,10 @@ jobs:
     runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        runner: ['macos-13', 'macos-14', "ubuntu-latest"]
+        runner: ['macos-13', 'macos-14', "ubuntu-latest", "windows-latest"]
     steps:
       - name: Setup macOS
-        if: ${{ matrix.runner != 'ubuntu-latest' }}
+        if: ${{ matrix.runner != 'ubuntu-latest' && matrix.runner != 'windows-latest'}}
         run: |
           brew install tree
           # Remove GnuPG since we want to bootstrap it
@@ -124,11 +124,16 @@ jobs:
         if: ${{ matrix.runner == 'ubuntu-latest' }}
         run: |
           sudo rm -rf $(which gpg) $(which gpg2) $(which patchelf)
+      - name: Setup Windows
+        if: ${{ matrix.runner == 'windows-latest' }}
+        run: |
+          Remove-Item -Path (Get-Command gpg).Path
+          Remove-Item -Path (Get-Command file).Path
       - name: Checkout
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
         with:
           python-version: |
             3.8
@@ -137,11 +142,20 @@ jobs:
             3.11
             3.12
       - name: Set bootstrap sources
+        env:
+          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
+          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
+        run: |
+          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
+          spack bootstrap disable github-actions-v0.4
+      - name: Disable from source bootstrap
+        if: ${{ matrix.runner != 'windows-latest' }}
         run: |
           source share/spack/setup-env.sh
-          spack bootstrap disable github-actions-v0.4
           spack bootstrap disable spack-install
       - name: Bootstrap clingo
+        # No binary clingo on Windows yet
+        if: ${{ matrix.runner != 'windows-latest' }}
         run: |
           set -e
           for ver in '3.8' '3.9' '3.10' '3.11' '3.12' ; do
@@ -164,7 +178,24 @@ jobs:
             fi
           done
       - name: Bootstrap GnuPG
+        env:
+          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
+          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
+          USER_SCOPE_PARENT_DIR: ${{ matrix.runner == 'windows-latest' && '$env:userprofile' || '$HOME' }}
+          VALIDATE_LAST_EXIT: ${{ matrix.runner == 'windows-latest' && './share/spack/qa/validate_last_exit.ps1' || '' }}
         run: |
-          source share/spack/setup-env.sh
+          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
           spack -d gpg list
-          tree ~/.spack/bootstrap/store/
+          ${{ env.VALIDATE_LAST_EXIT }}
+          tree ${{ env.USER_SCOPE_PARENT_DIR }}/.spack/bootstrap/store/
+      - name: Bootstrap File
+        env:
+          SETUP_SCRIPT_EXT: ${{ matrix.runner == 'windows-latest' && 'ps1' || 'sh' }}
+          SETUP_SCRIPT_SOURCE: ${{ matrix.runner == 'windows-latest' && './' || 'source ' }}
+          USER_SCOPE_PARENT_DIR: ${{ matrix.runner == 'windows-latest' && '$env:userprofile' || '$HOME' }}
+          VALIDATE_LAST_EXIT: ${{ matrix.runner == 'windows-latest' && './share/spack/qa/validate_last_exit.ps1' || '' }}
+        run: |
+          ${{ env.SETUP_SCRIPT_SOURCE }}share/spack/setup-env.${{ env.SETUP_SCRIPT_EXT }}
+          spack -d python share/spack/qa/bootstrap-file.py
+          ${{ env.VALIDATE_LAST_EXIT }}
+          tree ${{ env.USER_SCOPE_PARENT_DIR }}/.spack/bootstrap/store/

.github/workflows/build-containers.yml

@@ -87,19 +87,19 @@ jobs:
           fi
 
       - name: Upload Dockerfile
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
         with:
           name: dockerfiles_${{ matrix.dockerfile[0] }}
           path: dockerfiles
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db
 
       - name: Log in to GitHub Container Registry
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -107,13 +107,13 @@ jobs:
 
       - name: Log in to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build & Deploy ${{ matrix.dockerfile[0] }}
-        uses: docker/build-push-action@1a162644f9a7e87d8f4b053101d1d9a712edc18c
+        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85
         with:
           context: dockerfiles/${{ matrix.dockerfile[0] }}
           platforms: ${{ matrix.dockerfile[1] }}
@@ -126,7 +126,7 @@ jobs:
     needs: deploy-images
     steps:
       - name: Merge Artifacts
-        uses: actions/upload-artifact/merge@0b2256b8c012f0828dc542b3febcab082c67f72b
+        uses: actions/upload-artifact/merge@50769540e7f4bd5e21e526ee35c689e35e0d6874
         with:
           name: dockerfiles
           pattern: dockerfiles_*

.github/workflows/nightly-win-builds.yml

@@ -17,7 +17,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: 3.9
     - name: Install Python packages

.github/workflows/requirements/style/requirements.txt

@@ -1,6 +1,6 @@
-black==24.4.2
+black==24.8.0
 clingo==5.7.1
-flake8==7.1.0
+flake8==7.1.1
 isort==5.13.2
 mypy==1.8.0
 types-six==1.16.21.20240513

.github/workflows/unit_tests.yaml

@@ -16,45 +16,34 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
-        concretizer: ['clingo']
         on_develop:
         - ${{ github.ref == 'refs/heads/develop' }}
         include:
-        - python-version: '3.11'
-          os: ubuntu-latest
-          concretizer: original
-          on_develop: ${{ github.ref == 'refs/heads/develop' }}
         - python-version: '3.6'
           os: ubuntu-20.04
-          concretizer: clingo
           on_develop: ${{ github.ref == 'refs/heads/develop' }}
         exclude:
         - python-version: '3.7'
           os: ubuntu-latest
-          concretizer: 'clingo'
           on_develop: false
         - python-version: '3.8'
           os: ubuntu-latest
-          concretizer: 'clingo'
           on_develop: false
         - python-version: '3.9'
           os: ubuntu-latest
-          concretizer: 'clingo'
           on_develop: false
         - python-version: '3.10'
           os: ubuntu-latest
-          concretizer: 'clingo'
           on_develop: false
         - python-version: '3.11'
           os: ubuntu-latest
-          concretizer: 'clingo'
           on_develop: false
 
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install System packages
@@ -85,7 +74,6 @@ jobs:
     - name: Run unit tests
       env:
           SPACK_PYTHON: python
-          SPACK_TEST_SOLVER: ${{ matrix.concretizer }}
           SPACK_TEST_PARALLEL: 2
           COVERAGE: true
           UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }}
@@ -103,7 +91,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -163,7 +151,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -182,7 +170,6 @@ jobs:
     - name: Run unit tests (full suite with coverage)
       env:
           COVERAGE: true
-          SPACK_TEST_SOLVER: clingo
       run: |
           share/spack/qa/run-unit-tests
     - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
@@ -201,7 +188,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Python packages
@@ -213,7 +200,6 @@ jobs:
         brew install dash fish gcc gnupg2 kcov
     - name: Run unit tests
       env:
-        SPACK_TEST_SOLVER: clingo
         SPACK_TEST_PARALLEL: 4
       run: |
         git --version
@@ -239,7 +225,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: 3.9
     - name: Install Python packages

.github/workflows/valid-style.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: '3.11'
         cache: 'pip'
@@ -38,7 +38,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f
+    - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
       with:
         python-version: '3.11'
         cache: 'pip'

etc/spack/defaults/config.yaml

@@ -170,23 +170,6 @@ config:
   # If set to true, Spack will use ccache to cache C compiles.
   ccache: false
 
-
-  # The concretization algorithm to use in Spack. Options are:
-  #
-  #   'clingo': Uses a logic solver under the hood to solve DAGs with full
-  #       backtracking and optimization for user preferences. Spack will
-  #       try to bootstrap the logic solver, if not already available.
-  #
-  #   'original': Spack's original greedy, fixed-point concretizer. This
-  #       algorithm can make decisions too early and will not backtrack
-  #       sufficiently for many specs. This will soon be deprecated in
-  #       favor of clingo.
-  #
-  # See `concretizer.yaml` for more settings you can fine-tune when
-  # using clingo.
-  concretizer: clingo
-
-
   # How long to wait to lock the Spack installation database. This lock is used
   # when Spack needs to manage its own package metadata and all operations are
   # expected to complete within the default time limit. The timeout should

etc/spack/defaults/linux/packages.yaml

@@ -1,3 +0,0 @@
-packages:
-  iconv:
-    require: [libiconv]

etc/spack/defaults/packages.yaml

@@ -20,11 +20,14 @@ packages:
       awk: [gawk]
       armci: [armcimpi]
       blas: [openblas, amdblis]
+      c: [gcc]
+      cxx: [gcc]
       D: [ldc]
       daal: [intel-oneapi-daal]
       elf: [elfutils]
       fftw-api: [fftw, amdfftw]
       flame: [libflame, amdlibflame]
+      fortran: [gcc]
       fortran-rt: [gcc-runtime, intel-oneapi-runtime]
       fuse: [libfuse]
       gl: [glx, osmesa]
@@ -61,6 +64,7 @@ packages:
       tbb: [intel-tbb]
       unwind: [libunwind]
       uuid: [util-linux-uuid, libuuid]
+      wasi-sdk: [wasi-sdk-prebuilt]
       xxd: [xxd-standalone, vim]
       yacc: [bison, byacc]
       ziglang: [zig]
@@ -68,3 +72,13 @@ packages:
     permissions:
       read: world
       write: user
+  cray-mpich:
+    buildable: false
+  cray-mvapich2:
+    buildable: false
+  fujitsu-mpi:
+    buildable: false
+  hpcx-mpi:
+    buildable: false
+  spectrum-mpi:
+    buildable: false

etc/spack/defaults/windows/config.yaml

@@ -1,6 +1,5 @@
 config:
   locks: false
-  concretizer: clingo
   build_stage::
     - '$spack/.staging'
   stage_name: '{name}-{version}-{hash:7}'

lib/spack/docs/conf.py

@@ -206,6 +206,7 @@ def setup(sphinx):
     ("py:class", "six.moves.urllib.parse.ParseResult"),
     ("py:class", "TextIO"),
     ("py:class", "hashlib._Hash"),
+    ("py:class", "concurrent.futures._base.Executor"),
     # Spack classes that are private and we don't want to expose
     ("py:class", "spack.provider_index._IndexBase"),
     ("py:class", "spack.repo._PrependFileLoader"),

lib/spack/docs/environments.rst

@@ -863,7 +863,7 @@ named list ``compilers`` is ``['%gcc', '%clang', '%intel']`` on
    spack:
      definitions:
        - compilers: ['%gcc', '%clang']
-       - when: arch.satisfies('x86_64:')
+       - when: arch.satisfies('target=x86_64:')
          compilers: ['%intel']
 
 .. note::
@@ -893,8 +893,9 @@ The valid variables for a ``when`` clause are:
 
 #. ``env``. The user environment (usually ``os.environ`` in Python).
 
-#. ``hostname``. The hostname of the system (if ``hostname`` is an
-   executable in the user's PATH).
+#. ``hostname``. The hostname of the system.
+
+#. ``full_hostname``. The fully qualified hostname of the system.
 
 ^^^^^^^^^^^^^^^^^^^^^^^^
 SpecLists as Constraints

lib/spack/docs/packaging_guide.rst

@@ -1263,6 +1263,11 @@ Git fetching supports the following parameters to ``version``:
   option ``--depth 1`` will be used if the version of git and the specified
   transport protocol support it, and ``--single-branch`` will be used if the
   version of git supports it.
+* ``git_sparse_paths``: Use ``sparse-checkout`` to only clone these relative paths.
+  This feature requires ``git`` to be version ``2.25.0`` or later but is useful for
+  large repositories that have separate portions that can be built independently.
+  If paths provided are directories then all the subdirectories and associated files
+  will also be cloned. 
 
 Only one of ``tag``, ``branch``, or ``commit`` can be used at a time.
 
@@ -1361,6 +1366,41 @@ Submodules
   For more information about git submodules see the manpage of git: ``man
   git-submodule``.
 
+Sparse-Checkout
+  You can supply ``git_sparse_paths`` at the package or version level to utilize git's 
+  sparse-checkout feature. This will only clone the paths that are specified in the 
+  ``git_sparse_paths`` attribute for the package along with the files in the top level directory.
+  This feature allows you to only clone what you need from a large repository.
+  Note that this is a newer feature in git and requries git ``2.25.0`` or greater.
+  If ``git_sparse_paths`` is supplied and the git version is too old
+  then a warning will be issued and that package will use the standard cloning operations instead.
+  ``git_sparse_paths`` should be supplied as a list of paths, a callable function for versions,
+  or a more complex package attribute using the ``@property`` decorator. The return value should be
+  a list for a callable implementation of ``git_sparse_paths``.
+
+  .. code-block:: python
+
+    def sparse_path_function(package)
+        """a callable function that can be used in side a version"""
+        # paths can be directories or functions, all subdirectories and files are included
+        paths = ["doe", "rae", "me/file.cpp"]
+        if package.spec.version >  Version("1.2.0"):
+            paths.extend(["fae"])
+        return paths
+
+    class MyPackage(package):
+        # can also be a package attribute that will be used if not specified in versions
+        git_sparse_paths = ["doe", "rae"]
+
+        # use the package attribute
+        version("1.0.0")
+        version("1.1.0")
+        # use the function
+        version("1.1.5", git_sparse_paths=sparse_path_func)
+        version("1.2.0", git_sparse_paths=sparse_path_func)
+        version("1.2.5", git_sparse_paths=sparse_path_func)
+        version("1.1.5", git_sparse_paths=sparse_path_func)
+
 .. _github-fetch:
 
 ^^^^^^

lib/spack/docs/requirements.txt

@@ -1,13 +1,13 @@
-sphinx==7.2.6
+sphinx==7.4.7
 sphinxcontrib-programoutput==0.17
-sphinx_design==0.6.0
+sphinx_design==0.6.1
 sphinx-rtd-theme==2.0.0
 python-levenshtein==0.25.1
 docutils==0.20.1
 pygments==2.18.0
 urllib3==2.2.2
-pytest==8.2.2
+pytest==8.3.2
 isort==5.13.2
-black==24.4.2
-flake8==7.1.0
-mypy==1.10.1
+black==24.8.0
+flake8==7.1.1
+mypy==1.11.1

lib/spack/env/cc

@@ -174,6 +174,46 @@ preextend() {
     unset IFS
 }
 
+execute() {
+    # dump the full command if the caller supplies SPACK_TEST_COMMAND=dump-args
+    if [ -n "${SPACK_TEST_COMMAND=}" ]; then
+        case "$SPACK_TEST_COMMAND" in
+            dump-args)
+                IFS="$lsep"
+                for arg in $full_command_list; do
+                    echo "$arg"
+                done
+                unset IFS
+                exit
+                ;;
+            dump-env-*)
+                var=${SPACK_TEST_COMMAND#dump-env-}
+                eval "printf '%s\n' \"\$0: \$var: \$$var\""
+                ;;
+            *)
+                die "Unknown test command: '$SPACK_TEST_COMMAND'"
+                ;;
+        esac
+    fi
+
+    #
+    # Write the input and output commands to debug logs if it's asked for.
+    #
+    if [ "$SPACK_DEBUG" = TRUE ]; then
+        input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_DEBUG_LOG_ID.in.log"
+        output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_DEBUG_LOG_ID.out.log"
+        echo "[$mode] $command $input_command" >> "$input_log"
+        IFS="$lsep"
+        echo "[$mode] "$full_command_list >> "$output_log"
+        unset IFS
+    fi
+
+    # Execute the full command, preserving spaces with IFS set
+    # to the alarm bell separator.
+    IFS="$lsep"; exec $full_command_list
+    exit
+}
+
 # Fail with a clear message if the input contains any bell characters.
 if eval "[ \"\${*#*${lsep}}\" != \"\$*\" ]"; then
     die "Compiler command line contains our separator ('${lsep}'). Cannot parse."
@@ -231,12 +271,17 @@ fi
 #    ld      link
 #    ccld    compile & link
 
+# Note. SPACK_ALWAYS_XFLAGS are applied for all compiler invocations,
+# including version checks (SPACK_XFLAGS variants are not applied
+# for version checks).
 command="${0##*/}"
 comp="CC"
+vcheck_flags=""
 case "$command" in
     cpp)
         mode=cpp
         debug_flags="-g"
+        vcheck_flags="${SPACK_ALWAYS_CPPFLAGS}"
         ;;
     cc|c89|c99|gcc|clang|armclang|icc|icx|pgcc|nvc|xlc|xlc_r|fcc|amdclang|cl.exe|craycc)
         command="$SPACK_CC"
@@ -244,6 +289,7 @@ case "$command" in
         comp="CC"
         lang_flags=C
         debug_flags="-g"
+        vcheck_flags="${SPACK_ALWAYS_CFLAGS}"
         ;;
     c++|CC|g++|clang++|armclang++|icpc|icpx|pgc++|nvc++|xlc++|xlc++_r|FCC|amdclang++|crayCC)
         command="$SPACK_CXX"
@@ -251,6 +297,7 @@ case "$command" in
         comp="CXX"
         lang_flags=CXX
         debug_flags="-g"
+        vcheck_flags="${SPACK_ALWAYS_CXXFLAGS}"
         ;;
     ftn|f90|fc|f95|gfortran|flang|armflang|ifort|ifx|pgfortran|nvfortran|xlf90|xlf90_r|nagfor|frt|amdflang|crayftn)
         command="$SPACK_FC"
@@ -258,6 +305,7 @@ case "$command" in
         comp="FC"
         lang_flags=F
         debug_flags="-g"
+        vcheck_flags="${SPACK_ALWAYS_FFLAGS}"
         ;;
     f77|xlf|xlf_r|pgf77)
         command="$SPACK_F77"
@@ -265,6 +313,7 @@ case "$command" in
         comp="F77"
         lang_flags=F
         debug_flags="-g"
+        vcheck_flags="${SPACK_ALWAYS_FFLAGS}"
         ;;
     ld|ld.gold|ld.lld)
         mode=ld
@@ -365,7 +414,11 @@ unset IFS
 export PATH="$new_dirs"
 
 if [ "$mode" = vcheck ]; then
-    exec "${command}" "$@"
+    full_command_list="$command"
+    args="$@"
+    extend full_command_list vcheck_flags
+    extend full_command_list args
+    execute
 fi
 
 # Darwin's linker has a -r argument that merges object files together.
@@ -722,6 +775,7 @@ case "$mode" in
     cc|ccld)
         case $lang_flags in
             F)
+                extend spack_flags_list SPACK_ALWAYS_FFLAGS
                 extend spack_flags_list SPACK_FFLAGS
                 ;;
         esac
@@ -731,6 +785,7 @@ esac
 # C preprocessor flags come before any C/CXX flags
 case "$mode" in
     cpp|as|cc|ccld)
+        extend spack_flags_list SPACK_ALWAYS_CPPFLAGS
         extend spack_flags_list SPACK_CPPFLAGS
         ;;
 esac
@@ -741,9 +796,11 @@ case "$mode" in
     cc|ccld)
         case $lang_flags in
             C)
+                extend spack_flags_list SPACK_ALWAYS_CFLAGS
                 extend spack_flags_list SPACK_CFLAGS
                 ;;
             CXX)
+                extend spack_flags_list SPACK_ALWAYS_CXXFLAGS
                 extend spack_flags_list SPACK_CXXFLAGS
                 ;;
         esac
@@ -933,39 +990,4 @@ if [ -n "$SPACK_CCACHE_BINARY" ]; then
     esac
 fi
 
-# dump the full command if the caller supplies SPACK_TEST_COMMAND=dump-args
-if [ -n "${SPACK_TEST_COMMAND=}" ]; then
-    case "$SPACK_TEST_COMMAND" in
-        dump-args)
-            IFS="$lsep"
-            for arg in $full_command_list; do
-                echo "$arg"
-            done
-            unset IFS
-            exit
-            ;;
-        dump-env-*)
-            var=${SPACK_TEST_COMMAND#dump-env-}
-            eval "printf '%s\n' \"\$0: \$var: \$$var\""
-            ;;
-        *)
-            die "Unknown test command: '$SPACK_TEST_COMMAND'"
-            ;;
-    esac
-fi
-
-#
-# Write the input and output commands to debug logs if it's asked for.
-#
-if [ "$SPACK_DEBUG" = TRUE ]; then
-    input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_DEBUG_LOG_ID.in.log"
-    output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_DEBUG_LOG_ID.out.log"
-    echo "[$mode] $command $input_command" >> "$input_log"
-    IFS="$lsep"
-    echo "[$mode] "$full_command_list >> "$output_log"
-    unset IFS
-fi
-
-# Execute the full command, preserving spaces with IFS set
-# to the alarm bell separator.
-IFS="$lsep"; exec $full_command_list
+execute

lib/spack/external/__init__.py

@@ -18,7 +18,7 @@
 
 * Homepage: https://pypi.python.org/pypi/archspec
 * Usage: Labeling, comparison and detection of microarchitectures
-* Version: 0.2.4 (commit 48b92512b9ce203ded0ebd1ac41b42593e931f7c)
+* Version: 0.2.5-dev (commit 7e6740012b897ae4a950f0bba7e9726b767e921f)
 
 astunparse
 ----------------

lib/spack/external/_vendoring/distro/distro.py

@@ -1265,27 +1265,29 @@ def _distro_release_info(self) -> Dict[str, str]:
             match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename)
         else:
             try:
-                basenames = [
-                    basename
-                    for basename in os.listdir(self.etc_dir)
-                    if basename not in _DISTRO_RELEASE_IGNORE_BASENAMES
-                    and os.path.isfile(os.path.join(self.etc_dir, basename))
-                ]
+                with os.scandir(self.etc_dir) as it:
+                    etc_files = [
+                        p.path for p in it
+                        if p.is_file() and p.name not in _DISTRO_RELEASE_IGNORE_BASENAMES
+                    ]
                 # We sort for repeatability in cases where there are multiple
                 # distro specific files; e.g. CentOS, Oracle, Enterprise all
                 # containing `redhat-release` on top of their own.
-                basenames.sort()
+                etc_files.sort()
             except OSError:
                 # This may occur when /etc is not readable but we can't be
                 # sure about the *-release files. Check common entries of
                 # /etc for information. If they turn out to not be there the
                 # error is handled in `_parse_distro_release_file()`.
-                basenames = _DISTRO_RELEASE_BASENAMES
-            for basename in basenames:
-                match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename)
+                etc_files = [
+                    os.path.join(self.etc_dir, basename)
+                    for basename in _DISTRO_RELEASE_BASENAMES
+                ]
+
+            for filepath in etc_files:
+                match = _DISTRO_RELEASE_BASENAME_PATTERN.match(os.path.basename(filepath))
                 if match is None:
                     continue
-                filepath = os.path.join(self.etc_dir, basename)
                 distro_info = self._parse_distro_release_file(filepath)
                 # The name is always present if the pattern matches.
                 if "name" not in distro_info:

lib/spack/external/_vendoring/jsonschema/_format.py

@@ -231,96 +231,6 @@ def is_host_name(instance):
     return True
 
 
-try:
-    # The built-in `idna` codec only implements RFC 3890, so we go elsewhere.
-    import idna
-except ImportError:
-    pass
-else:
-    @_checks_drafts(draft7="idn-hostname", raises=idna.IDNAError)
-    def is_idn_host_name(instance):
-        if not isinstance(instance, str_types):
-            return True
-        idna.encode(instance)
-        return True
-
-
-try:
-    import rfc3987
-except ImportError:
-    try:
-        from rfc3986_validator import validate_rfc3986
-    except ImportError:
-        pass
-    else:
-        @_checks_drafts(name="uri")
-        def is_uri(instance):
-            if not isinstance(instance, str_types):
-                return True
-            return validate_rfc3986(instance, rule="URI")
-
-        @_checks_drafts(
-            draft6="uri-reference",
-            draft7="uri-reference",
-            raises=ValueError,
-        )
-        def is_uri_reference(instance):
-            if not isinstance(instance, str_types):
-                return True
-            return validate_rfc3986(instance, rule="URI_reference")
-
-else:
-    @_checks_drafts(draft7="iri", raises=ValueError)
-    def is_iri(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return rfc3987.parse(instance, rule="IRI")
-
-    @_checks_drafts(draft7="iri-reference", raises=ValueError)
-    def is_iri_reference(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return rfc3987.parse(instance, rule="IRI_reference")
-
-    @_checks_drafts(name="uri", raises=ValueError)
-    def is_uri(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return rfc3987.parse(instance, rule="URI")
-
-    @_checks_drafts(
-        draft6="uri-reference",
-        draft7="uri-reference",
-        raises=ValueError,
-    )
-    def is_uri_reference(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return rfc3987.parse(instance, rule="URI_reference")
-
-
-try:
-    from strict_rfc3339 import validate_rfc3339
-except ImportError:
-    try:
-        from rfc3339_validator import validate_rfc3339
-    except ImportError:
-        validate_rfc3339 = None
-
-if validate_rfc3339:
-    @_checks_drafts(name="date-time")
-    def is_datetime(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return validate_rfc3339(instance)
-
-    @_checks_drafts(draft7="time")
-    def is_time(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return is_datetime("1970-01-01T" + instance)
-
-
 @_checks_drafts(name="regex", raises=re.error)
 def is_regex(instance):
     if not isinstance(instance, str_types):
@@ -340,86 +250,3 @@ def is_draft3_time(instance):
     if not isinstance(instance, str_types):
         return True
     return datetime.datetime.strptime(instance, "%H:%M:%S")
-
-
-try:
-    import webcolors
-except ImportError:
-    pass
-else:
-    def is_css_color_code(instance):
-        return webcolors.normalize_hex(instance)
-
-    @_checks_drafts(draft3="color", raises=(ValueError, TypeError))
-    def is_css21_color(instance):
-        if (
-            not isinstance(instance, str_types) or
-            instance.lower() in webcolors.css21_names_to_hex
-        ):
-            return True
-        return is_css_color_code(instance)
-
-    def is_css3_color(instance):
-        if instance.lower() in webcolors.css3_names_to_hex:
-            return True
-        return is_css_color_code(instance)
-
-
-try:
-    import jsonpointer
-except ImportError:
-    pass
-else:
-    @_checks_drafts(
-        draft6="json-pointer",
-        draft7="json-pointer",
-        raises=jsonpointer.JsonPointerException,
-    )
-    def is_json_pointer(instance):
-        if not isinstance(instance, str_types):
-            return True
-        return jsonpointer.JsonPointer(instance)
-
-    # TODO: I don't want to maintain this, so it
-    #       needs to go either into jsonpointer (pending
-    #       https://github.com/stefankoegl/python-json-pointer/issues/34) or
-    #       into a new external library.
-    @_checks_drafts(
-        draft7="relative-json-pointer",
-        raises=jsonpointer.JsonPointerException,
-    )
-    def is_relative_json_pointer(instance):
-        # Definition taken from:
-        # https://tools.ietf.org/html/draft-handrews-relative-json-pointer-01#section-3
-        if not isinstance(instance, str_types):
-            return True
-        non_negative_integer, rest = [], ""
-        for i, character in enumerate(instance):
-            if character.isdigit():
-                non_negative_integer.append(character)
-                continue
-
-            if not non_negative_integer:
-                return False
-
-            rest = instance[i:]
-            break
-        return (rest == "#") or jsonpointer.JsonPointer(rest)
-
-
-try:
-    import uritemplate.exceptions
-except ImportError:
-    pass
-else:
-    @_checks_drafts(
-        draft6="uri-template",
-        draft7="uri-template",
-        raises=uritemplate.exceptions.InvalidTemplate,
-    )
-    def is_uri_template(
-        instance,
-        template_validator=uritemplate.Validator().force_balanced_braces(),
-    ):
-        template = uritemplate.URITemplate(instance)
-        return template_validator.validate(template)

lib/spack/external/archspec/cpu/detect.py

@@ -47,7 +47,11 @@ def decorator(factory):
 
 
 def partial_uarch(
-    name: str = "", vendor: str = "", features: Optional[Set[str]] = None, generation: int = 0
+    name: str = "",
+    vendor: str = "",
+    features: Optional[Set[str]] = None,
+    generation: int = 0,
+    cpu_part: str = "",
 ) -> Microarchitecture:
     """Construct a partial microarchitecture, from information gathered during system scan."""
     return Microarchitecture(
@@ -57,6 +61,7 @@ def partial_uarch(
         features=features or set(),
         compilers={},
         generation=generation,
+        cpu_part=cpu_part,
     )
 
 
@@ -90,6 +95,7 @@ def proc_cpuinfo() -> Microarchitecture:
         return partial_uarch(
             vendor=_canonicalize_aarch64_vendor(data),
             features=_feature_set(data, key="Features"),
+            cpu_part=data.get("CPU part", ""),
         )
 
     if architecture in (PPC64LE, PPC64):
@@ -345,6 +351,10 @@ def sorting_fn(item):
     generic_candidates = [c for c in candidates if c.vendor == "generic"]
     best_generic = max(generic_candidates, key=sorting_fn)
 
+    # Relevant for AArch64. Filter on "cpu_part" if we have any match
+    if info.cpu_part != "" and any(c for c in candidates if info.cpu_part == c.cpu_part):
+        candidates = [c for c in candidates if info.cpu_part == c.cpu_part]
+
     # Filter the candidates to be descendant of the best generic candidate.
     # This is to avoid that the lack of a niche feature that can be disabled
     # from e.g. BIOS prevents detection of a reasonably performant architecture

lib/spack/external/archspec/cpu/microarchitecture.py

@@ -2,9 +2,7 @@
 # Archspec Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-"""Types and functions to manage information
-on CPU microarchitectures.
-"""
+"""Types and functions to manage information on CPU microarchitectures."""
 import functools
 import platform
 import re
@@ -65,21 +63,24 @@ class Microarchitecture:
                 passed in as argument above.
             * versions: versions that support this micro-architecture.
 
-        generation (int): generation of the micro-architecture, if
-            relevant.
+        generation (int): generation of the micro-architecture, if relevant.
+        cpu_part (str): cpu part of the architecture, if relevant.
     """
 
-    # pylint: disable=too-many-arguments
+    # pylint: disable=too-many-arguments,too-many-instance-attributes
     #: Aliases for micro-architecture's features
     feature_aliases = FEATURE_ALIASES
 
-    def __init__(self, name, parents, vendor, features, compilers, generation=0):
+    def __init__(self, name, parents, vendor, features, compilers, generation=0, cpu_part=""):
         self.name = name
         self.parents = parents
         self.vendor = vendor
         self.features = features
         self.compilers = compilers
+        # Only relevant for PowerPC
         self.generation = generation
+        # Only relevant for AArch64
+        self.cpu_part = cpu_part
         # Cache the ancestor computation
         self._ancestors = None
 
@@ -111,6 +112,7 @@ def __eq__(self, other):
             and self.parents == other.parents  # avoid ancestors here
             and self.compilers == other.compilers
             and self.generation == other.generation
+            and self.cpu_part == other.cpu_part
         )
 
     @coerce_target_names
@@ -143,7 +145,8 @@ def __repr__(self):
         cls_name = self.__class__.__name__
         fmt = (
             cls_name + "({0.name!r}, {0.parents!r}, {0.vendor!r}, "
-            "{0.features!r}, {0.compilers!r}, {0.generation!r})"
+            "{0.features!r}, {0.compilers!r}, generation={0.generation!r}, "
+            "cpu_part={0.cpu_part!r})"
         )
         return fmt.format(self)
 
@@ -190,6 +193,7 @@ def to_dict(self):
             "generation": self.generation,
             "parents": [str(x) for x in self.parents],
             "compilers": self.compilers,
+            "cpupart": self.cpu_part,
         }
 
     @staticmethod
@@ -202,6 +206,7 @@ def from_dict(data) -> "Microarchitecture":
             features=set(data["features"]),
             compilers=data.get("compilers", {}),
             generation=data.get("generation", 0),
+            cpu_part=data.get("cpupart", ""),
         )
 
     def optimization_flags(self, compiler, version):
@@ -360,8 +365,11 @@ def fill_target_from_dict(name, data, targets):
         features = set(values["features"])
         compilers = values.get("compilers", {})
         generation = values.get("generation", 0)
+        cpu_part = values.get("cpupart", "")
 
-        targets[name] = Microarchitecture(name, parents, vendor, features, compilers, generation)
+        targets[name] = Microarchitecture(
+            name, parents, vendor, features, compilers, generation=generation, cpu_part=cpu_part
+        )
 
     known_targets = {}
     data = archspec.cpu.schema.TARGETS_JSON["microarchitectures"]

lib/spack/external/archspec/json/cpu/microarchitectures.json

@@ -2225,10 +2225,14 @@
         ],
         "nvhpc": [
           {
-            "versions": "21.11:",
+            "versions": "21.11:23.8",
 	    "name": "zen3",
             "flags": "-tp {name}",
-	    "warnings": "zen4 is not fully supported by nvhpc yet, falling back to zen3"
+	    "warnings": "zen4 is not fully supported by nvhpc versions < 23.9, falling back to zen3"
+          },
+          {
+            "versions": "23.9:",
+            "flags": "-tp {name}"
           }
 	]
       }
@@ -2711,7 +2715,8 @@
             "flags": "-mcpu=thunderx2t99"
           }
         ]
-      }
+      },
+      "cpupart": "0x0af"
     },
     "a64fx": {
       "from": ["armv8.2a"],
@@ -2779,7 +2784,8 @@
             "flags": "-march=armv8.2-a+crc+crypto+fp16+sve"
           }
         ]
-      }
+      },
+      "cpupart": "0x001"
     },
     "cortex_a72": {
       "from": ["aarch64"],
@@ -2816,7 +2822,8 @@
                   "flags" : "-mcpu=cortex-a72"
               }
           ]
-      }
+      },
+      "cpupart": "0xd08"
     },
     "neoverse_n1": {
       "from": ["cortex_a72", "armv8.2a"],
@@ -2902,7 +2909,8 @@
                   "flags": "-tp {name}"
               }
           ]
-      }
+      },
+      "cpupart": "0xd0c"
     },
     "neoverse_v1": {
       "from": ["neoverse_n1", "armv8.4a"],
@@ -2926,8 +2934,6 @@
           "lrcpc",
           "dcpop",
           "sha3",
-          "sm3",
-          "sm4",
           "asimddp",
           "sha512",
           "sve",
@@ -3028,7 +3034,8 @@
                   "flags": "-tp {name}"
               }
           ]
-      }
+      },
+      "cpupart": "0xd40"
     },
     "neoverse_v2": {
       "from": ["neoverse_n1", "armv9.0a"],
@@ -3052,13 +3059,10 @@
 	  "lrcpc",
 	  "dcpop",
 	  "sha3",
-	  "sm3",
-	  "sm4",
 	  "asimddp",
 	  "sha512",
 	  "sve",
 	  "asimdfhm",
-	  "dit",
 	  "uscat",
 	  "ilrcpc",
 	  "flagm",
@@ -3066,18 +3070,12 @@
 	  "sb",
 	  "dcpodp",
 	  "sve2",
-	  "sveaes",
-	  "svepmull",
-	  "svebitperm",
-	  "svesha3",
-	  "svesm4",
 	  "flagm2",
 	  "frint",
 	  "svei8mm",
 	  "svebf16",
 	  "i8mm",
-	  "bf16",
-	  "dgh"
+	  "bf16"
       ],
       "compilers" : {
           "gcc": [
@@ -3102,15 +3100,19 @@
                   "flags" : "-march=armv8.5-a+sve -mtune=cortex-a76"
               },
               {
-                  "versions": "10.0:11.99",
+                  "versions": "10.0:11.3.99",
                   "flags" : "-march=armv8.5-a+sve+sve2+i8mm+bf16 -mtune=cortex-a77"
               },
+	      {
+                  "versions": "11.4:11.99",
+                  "flags" : "-mcpu=neoverse-v2"
+              },
               {
-                  "versions": "12.0:12.99",
+                  "versions": "12.0:12.2.99",
                   "flags" : "-march=armv9-a+i8mm+bf16 -mtune=cortex-a710"
               },
 	      {
-                  "versions": "13.0:",
+                  "versions": "12.3:",
                   "flags" : "-mcpu=neoverse-v2"
               }
           ],
@@ -3145,7 +3147,113 @@
                   "flags": "-tp {name}"
               }
           ]
-      }
+      },
+      "cpupart": "0xd4f"
+    },
+    "neoverse_n2": {
+      "from": ["neoverse_n1", "armv9.0a"],
+      "vendor": "ARM",
+      "features": [
+          "fp",
+	  "asimd",
+	  "evtstrm",
+	  "aes",
+	  "pmull",
+	  "sha1",
+	  "sha2",
+	  "crc32",
+	  "atomics",
+	  "fphp",
+	  "asimdhp",
+	  "cpuid",
+	  "asimdrdm",
+	  "jscvt",
+	  "fcma",
+	  "lrcpc",
+	  "dcpop",
+	  "sha3",
+	  "asimddp",
+	  "sha512",
+	  "sve",
+	  "asimdfhm",
+	  "uscat",
+	  "ilrcpc",
+	  "flagm",
+	  "ssbs",
+	  "sb",
+	  "dcpodp",
+	  "sve2",
+	  "flagm2",
+	  "frint",
+	  "svei8mm",
+	  "svebf16",
+	  "i8mm",
+	  "bf16"
+      ],
+      "compilers" : {
+          "gcc": [
+              {
+                  "versions": "4.8:5.99",
+                  "flags": "-march=armv8-a"
+              },
+              {
+                  "versions": "6:6.99",
+                  "flags" : "-march=armv8.1-a"
+              },
+              {
+                  "versions": "7.0:7.99",
+                  "flags" : "-march=armv8.2-a -mtune=cortex-a72"
+              },
+              {
+                  "versions": "8.0:8.99",
+                  "flags" : "-march=armv8.4-a+sve -mtune=cortex-a72"
+              },
+              {
+                  "versions": "9.0:9.99",
+                  "flags" : "-march=armv8.5-a+sve -mtune=cortex-a76"
+              },
+              {
+                  "versions": "10.0:10.99",
+                  "flags" : "-march=armv8.5-a+sve+sve2+i8mm+bf16 -mtune=cortex-a77"
+              },
+              {
+                  "versions": "11.0:",
+                  "flags" : "-mcpu=neoverse-n2"
+              }
+          ],
+          "clang" : [
+              {
+                  "versions": "9.0:10.99",
+                  "flags" : "-march=armv8.5-a+sve"
+              },
+              {
+                  "versions": "11.0:13.99",
+                  "flags" : "-march=armv8.5-a+sve+sve2+i8mm+bf16"
+              },
+              {
+                  "versions": "14.0:15.99",
+                  "flags" : "-march=armv9-a+i8mm+bf16"
+              },
+              {
+                  "versions": "16.0:",
+                  "flags" : "-mcpu=neoverse-n2"
+              }
+          ],
+          "arm" : [
+              {
+                  "versions": "23.04.0:",
+                  "flags" : "-mcpu=neoverse-n2"
+              }
+          ],
+          "nvhpc" : [
+              {
+                  "versions": "23.3:",
+                  "name": "neoverse-n1",
+                  "flags": "-tp {name}"
+              }
+          ]
+      },
+      "cpupart": "0xd49"
     },
     "m1": {
       "from": ["armv8.4a"],
@@ -3211,7 +3319,8 @@
             "flags" : "-mcpu=apple-m1"
           }
         ]
-      }
+      },
+      "cpupart": "0x022"
     },
     "m2": {
       "from": ["m1", "armv8.5a"],
@@ -3289,7 +3398,8 @@
             "flags" : "-mcpu=apple-m2"
           }
         ]
-      }
+      },
+      "cpupart": "0x032"
     },
     "arm": {
       "from": [],

lib/spack/external/archspec/json/cpu/microarchitectures_schema.json

@@ -52,6 +52,9 @@
                   }
                 }
               }
+            },
+            "cpupart": {
+              "type": "string"
             }
           },
           "required": [
@@ -107,4 +110,4 @@
       "additionalProperties": false
     }
   }
-}
+}

lib/spack/external/patches/distro.patch

@@ -0,0 +1,45 @@
+diff --git a/lib/spack/external/_vendoring/distro/distro.py b/lib/spack/external/_vendoring/distro/distro.py
+index 89e1868047..50c3b18d4d 100644
+--- a/lib/spack/external/_vendoring/distro/distro.py
++++ b/lib/spack/external/_vendoring/distro/distro.py
+@@ -1265,27 +1265,29 @@ def _distro_release_info(self) -> Dict[str, str]:
+             match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename)
+         else:
+             try:
+-                basenames = [
+-                    basename
+-                    for basename in os.listdir(self.etc_dir)
+-                    if basename not in _DISTRO_RELEASE_IGNORE_BASENAMES
+-                    and os.path.isfile(os.path.join(self.etc_dir, basename))
+-                ]
++                with os.scandir(self.etc_dir) as it:
++                    etc_files = [
++                        p.path for p in it
++                        if p.is_file() and p.name not in _DISTRO_RELEASE_IGNORE_BASENAMES
++                    ]
+                 # We sort for repeatability in cases where there are multiple
+                 # distro specific files; e.g. CentOS, Oracle, Enterprise all
+                 # containing `redhat-release` on top of their own.
+-                basenames.sort()
++                etc_files.sort()
+             except OSError:
+                 # This may occur when /etc is not readable but we can't be
+                 # sure about the *-release files. Check common entries of
+                 # /etc for information. If they turn out to not be there the
+                 # error is handled in `_parse_distro_release_file()`.
+-                basenames = _DISTRO_RELEASE_BASENAMES
+-            for basename in basenames:
+-                match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename)
++                etc_files = [
++                    os.path.join(self.etc_dir, basename)
++                    for basename in _DISTRO_RELEASE_BASENAMES
++                ]
++
++            for filepath in etc_files:
++                match = _DISTRO_RELEASE_BASENAME_PATTERN.match(os.path.basename(filepath))
+                 if match is None:
+                     continue
+-                filepath = os.path.join(self.etc_dir, basename)
+                 distro_info = self._parse_distro_release_file(filepath)
+                 # The name is always present if the pattern matches.
+                 if "name" not in distro_info:

lib/spack/external/patches/jsonschema.patch

@@ -13,3 +13,191 @@ index 6b630cdfbb..1791fe7fbf 100644
 -__version__ = metadata.version("jsonschema")
 +
 +__version__ = "3.2.0"
+diff --git a/lib/spack/external/_vendoring/jsonschema/_format.py b/lib/spack/external/_vendoring/jsonschema/_format.py
+index 281a7cfcff..29061e3661 100644
+--- a/lib/spack/external/_vendoring/jsonschema/_format.py
++++ b/lib/spack/external/_vendoring/jsonschema/_format.py
+@@ -231,96 +231,6 @@ def is_host_name(instance):
+     return True
+ 
+ 
+-try:
+-    # The built-in `idna` codec only implements RFC 3890, so we go elsewhere.
+-    import idna
+-except ImportError:
+-    pass
+-else:
+-    @_checks_drafts(draft7="idn-hostname", raises=idna.IDNAError)
+-    def is_idn_host_name(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        idna.encode(instance)
+-        return True
+-
+-
+-try:
+-    import rfc3987
+-except ImportError:
+-    try:
+-        from rfc3986_validator import validate_rfc3986
+-    except ImportError:
+-        pass
+-    else:
+-        @_checks_drafts(name="uri")
+-        def is_uri(instance):
+-            if not isinstance(instance, str_types):
+-                return True
+-            return validate_rfc3986(instance, rule="URI")
+-
+-        @_checks_drafts(
+-            draft6="uri-reference",
+-            draft7="uri-reference",
+-            raises=ValueError,
+-        )
+-        def is_uri_reference(instance):
+-            if not isinstance(instance, str_types):
+-                return True
+-            return validate_rfc3986(instance, rule="URI_reference")
+-
+-else:
+-    @_checks_drafts(draft7="iri", raises=ValueError)
+-    def is_iri(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return rfc3987.parse(instance, rule="IRI")
+-
+-    @_checks_drafts(draft7="iri-reference", raises=ValueError)
+-    def is_iri_reference(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return rfc3987.parse(instance, rule="IRI_reference")
+-
+-    @_checks_drafts(name="uri", raises=ValueError)
+-    def is_uri(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return rfc3987.parse(instance, rule="URI")
+-
+-    @_checks_drafts(
+-        draft6="uri-reference",
+-        draft7="uri-reference",
+-        raises=ValueError,
+-    )
+-    def is_uri_reference(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return rfc3987.parse(instance, rule="URI_reference")
+-
+-
+-try:
+-    from strict_rfc3339 import validate_rfc3339
+-except ImportError:
+-    try:
+-        from rfc3339_validator import validate_rfc3339
+-    except ImportError:
+-        validate_rfc3339 = None
+-
+-if validate_rfc3339:
+-    @_checks_drafts(name="date-time")
+-    def is_datetime(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return validate_rfc3339(instance)
+-
+-    @_checks_drafts(draft7="time")
+-    def is_time(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return is_datetime("1970-01-01T" + instance)
+-
+-
+ @_checks_drafts(name="regex", raises=re.error)
+ def is_regex(instance):
+     if not isinstance(instance, str_types):
+@@ -340,86 +250,3 @@ def is_draft3_time(instance):
+     if not isinstance(instance, str_types):
+         return True
+     return datetime.datetime.strptime(instance, "%H:%M:%S")
+-
+-
+-try:
+-    import webcolors
+-except ImportError:
+-    pass
+-else:
+-    def is_css_color_code(instance):
+-        return webcolors.normalize_hex(instance)
+-
+-    @_checks_drafts(draft3="color", raises=(ValueError, TypeError))
+-    def is_css21_color(instance):
+-        if (
+-            not isinstance(instance, str_types) or
+-            instance.lower() in webcolors.css21_names_to_hex
+-        ):
+-            return True
+-        return is_css_color_code(instance)
+-
+-    def is_css3_color(instance):
+-        if instance.lower() in webcolors.css3_names_to_hex:
+-            return True
+-        return is_css_color_code(instance)
+-
+-
+-try:
+-    import jsonpointer
+-except ImportError:
+-    pass
+-else:
+-    @_checks_drafts(
+-        draft6="json-pointer",
+-        draft7="json-pointer",
+-        raises=jsonpointer.JsonPointerException,
+-    )
+-    def is_json_pointer(instance):
+-        if not isinstance(instance, str_types):
+-            return True
+-        return jsonpointer.JsonPointer(instance)
+-
+-    # TODO: I don't want to maintain this, so it
+-    #       needs to go either into jsonpointer (pending
+-    #       https://github.com/stefankoegl/python-json-pointer/issues/34) or
+-    #       into a new external library.
+-    @_checks_drafts(
+-        draft7="relative-json-pointer",
+-        raises=jsonpointer.JsonPointerException,
+-    )
+-    def is_relative_json_pointer(instance):
+-        # Definition taken from:
+-        # https://tools.ietf.org/html/draft-handrews-relative-json-pointer-01#section-3
+-        if not isinstance(instance, str_types):
+-            return True
+-        non_negative_integer, rest = [], ""
+-        for i, character in enumerate(instance):
+-            if character.isdigit():
+-                non_negative_integer.append(character)
+-                continue
+-
+-            if not non_negative_integer:
+-                return False
+-
+-            rest = instance[i:]
+-            break
+-        return (rest == "#") or jsonpointer.JsonPointer(rest)
+-
+-
+-try:
+-    import uritemplate.exceptions
+-except ImportError:
+-    pass
+-else:
+-    @_checks_drafts(
+-        draft6="uri-template",
+-        draft7="uri-template",
+-        raises=uritemplate.exceptions.InvalidTemplate,
+-    )
+-    def is_uri_template(
+-        instance,
+-        template_validator=uritemplate.Validator().force_balanced_braces(),
+-    ):
+-        template = uritemplate.URITemplate(instance)
+-        return template_validator.validate(template)

lib/spack/llnl/util/filesystem.py

@@ -27,8 +27,6 @@
 from llnl.util.lang import dedupe, memoized
 from llnl.util.symlink import islink, readlink, resolve_link_target_relative_to_the_link, symlink
 
-from spack.util.executable import Executable, which
-
 from ..path import path_to_os_path, system_path_filter
 
 if sys.platform != "win32":
@@ -53,7 +51,6 @@
     "find_all_headers",
     "find_libraries",
     "find_system_libraries",
-    "fix_darwin_install_name",
     "force_remove",
     "force_symlink",
     "getuid",
@@ -248,42 +245,6 @@ def path_contains_subdirectory(path, root):
     return norm_path.startswith(norm_root)
 
 
-@memoized
-def file_command(*args):
-    """Creates entry point to `file` system command with provided arguments"""
-    file_cmd = which("file", required=True)
-    for arg in args:
-        file_cmd.add_default_arg(arg)
-    return file_cmd
-
-
-@memoized
-def _get_mime_type():
-    """Generate method to call `file` system command to aquire mime type
-    for a specified path
-    """
-    if sys.platform == "win32":
-        # -h option (no-dereference) does not exist in Windows
-        return file_command("-b", "--mime-type")
-    else:
-        return file_command("-b", "-h", "--mime-type")
-
-
-def mime_type(filename):
-    """Returns the mime type and subtype of a file.
-
-    Args:
-        filename: file to be analyzed
-
-    Returns:
-        Tuple containing the MIME type and subtype
-    """
-    output = _get_mime_type()(filename, output=str, error=str).strip()
-    tty.debug("==> " + output)
-    type, _, subtype = output.partition("/")
-    return type, subtype
-
-
 #: This generates the library filenames that may appear on any OS.
 library_extensions = ["a", "la", "so", "tbd", "dylib"]
 
@@ -1624,6 +1585,12 @@ def remove_linked_tree(path):
             shutil.rmtree(os.path.realpath(path), **kwargs)
             os.unlink(path)
         else:
+            if sys.platform == "win32":
+                # Adding this prefix allows shutil to remove long paths on windows
+                # https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry
+                long_path_pfx = "\\\\?\\"
+                if not path.startswith(long_path_pfx):
+                    path = long_path_pfx + path
             shutil.rmtree(path, **kwargs)
 
 
@@ -1673,41 +1640,6 @@ def safe_remove(*files_or_dirs):
         raise
 
 
-@system_path_filter
-def fix_darwin_install_name(path):
-    """Fix install name of dynamic libraries on Darwin to have full path.
-
-    There are two parts of this task:
-
-    1. Use ``install_name('-id', ...)`` to change install name of a single lib
-    2. Use ``install_name('-change', ...)`` to change the cross linking between
-       libs. The function assumes that all libraries are in one folder and
-       currently won't follow subfolders.
-
-    Parameters:
-        path (str): directory in which .dylib files are located
-    """
-    libs = glob.glob(join_path(path, "*.dylib"))
-    for lib in libs:
-        # fix install name first:
-        install_name_tool = Executable("install_name_tool")
-        install_name_tool("-id", lib, lib)
-        otool = Executable("otool")
-        long_deps = otool("-L", lib, output=str).split("\n")
-        deps = [dep.partition(" ")[0][1::] for dep in long_deps[2:-1]]
-        # fix all dependencies:
-        for dep in deps:
-            for loc in libs:
-                # We really want to check for either
-                #     dep == os.path.basename(loc)   or
-                #     dep == join_path(builddir, os.path.basename(loc)),
-                # but we don't know builddir (nor how symbolic links look
-                # in builddir). We thus only compare the basenames.
-                if os.path.basename(dep) == os.path.basename(loc):
-                    install_name_tool("-change", dep, loc, lib)
-                    break
-
-
 def find_first(root: str, files: Union[Iterable[str], str], bfs_depth: int = 2) -> Optional[str]:
     """Find the first file matching a pattern.
 

lib/spack/llnl/util/lang.py

@@ -6,7 +6,6 @@
 import collections.abc
 import contextlib
 import functools
-import inspect
 import itertools
 import os
 import re
@@ -16,7 +15,7 @@
 from typing import Any, Callable, Iterable, List, Tuple
 
 # Ignore emacs backups when listing modules
-ignore_modules = [r"^\.#", "~$"]
+ignore_modules = r"^\.#|~$"
 
 
 def index_by(objects, *funcs):
@@ -84,20 +83,6 @@ def index_by(objects, *funcs):
     return result
 
 
-def caller_locals():
-    """This will return the locals of the *parent* of the caller.
-    This allows a function to insert variables into its caller's
-    scope.  Yes, this is some black magic, and yes it's useful
-    for implementing things like depends_on and provides.
-    """
-    # Passing zero here skips line context for speed.
-    stack = inspect.stack(0)
-    try:
-        return stack[2][0].f_locals
-    finally:
-        del stack
-
-
 def attr_setdefault(obj, name, value):
     """Like dict.setdefault, but for objects."""
     if not hasattr(obj, name):
@@ -105,15 +90,6 @@ def attr_setdefault(obj, name, value):
     return getattr(obj, name)
 
 
-def has_method(cls, name):
-    for base in inspect.getmro(cls):
-        if base is object:
-            continue
-        if name in base.__dict__:
-            return True
-    return False
-
-
 def union_dicts(*dicts):
     """Use update() to combine all dicts into one.
 
@@ -178,19 +154,22 @@ def list_modules(directory, **kwargs):
     order."""
     list_directories = kwargs.setdefault("directories", True)
 
-    for name in os.listdir(directory):
-        if name == "__init__.py":
-            continue
+    ignore = re.compile(ignore_modules)
 
-        path = os.path.join(directory, name)
-        if list_directories and os.path.isdir(path):
-            init_py = os.path.join(path, "__init__.py")
-            if os.path.isfile(init_py):
-                yield name
+    with os.scandir(directory) as it:
+        for entry in it:
+            if entry.name == "__init__.py" or entry.name == "__pycache__":
+                continue
 
-        elif name.endswith(".py"):
-            if not any(re.search(pattern, name) for pattern in ignore_modules):
-                yield re.sub(".py$", "", name)
+            if (
+                list_directories
+                and entry.is_dir()
+                and os.path.isfile(os.path.join(entry.path, "__init__.py"))
+            ):
+                yield entry.name
+
+            elif entry.name.endswith(".py") and entry.is_file() and not ignore.search(entry.name):
+                yield entry.name[:-3]  # strip .py
 
 
 def decorator_with_or_without_args(decorator):
@@ -237,8 +216,8 @@ def setter(name, value):
         value.__name__ = name
         setattr(cls, name, value)
 
-    if not has_method(cls, "_cmp_key"):
-        raise TypeError("'%s' doesn't define _cmp_key()." % cls.__name__)
+    if not hasattr(cls, "_cmp_key"):
+        raise TypeError(f"'{cls.__name__}' doesn't define _cmp_key().")
 
     setter("__eq__", lambda s, o: (s is o) or (o is not None and s._cmp_key() == o._cmp_key()))
     setter("__lt__", lambda s, o: o is not None and s._cmp_key() < o._cmp_key())
@@ -388,8 +367,8 @@ def cd_fun():
         TypeError: If the class does not have a ``_cmp_iter`` method
 
     """
-    if not has_method(cls, "_cmp_iter"):
-        raise TypeError("'%s' doesn't define _cmp_iter()." % cls.__name__)
+    if not hasattr(cls, "_cmp_iter"):
+        raise TypeError(f"'{cls.__name__}' doesn't define _cmp_iter().")
 
     # comparison operators are implemented in terms of lazy_eq and lazy_lt
     def eq(self, other):
@@ -864,20 +843,19 @@ def uniq(sequence):
     return uniq_list
 
 
-def elide_list(line_list, max_num=10):
+def elide_list(line_list: List[str], max_num: int = 10) -> List[str]:
     """Takes a long list and limits it to a smaller number of elements,
     replacing intervening elements with '...'.  For example::
 
-        elide_list([1,2,3,4,5,6], 4)
+        elide_list(["1", "2", "3", "4", "5", "6"], 4)
 
     gives::
 
-        [1, 2, 3, '...', 6]
+        ["1", "2", "3", "...", "6"]
     """
     if len(line_list) > max_num:
-        return line_list[: max_num - 1] + ["..."] + line_list[-1:]
-    else:
-        return line_list
+        return [*line_list[: max_num - 1], "...", line_list[-1]]
+    return line_list
 
 
 @contextlib.contextmanager

lib/spack/llnl/util/tty/log.py

@@ -10,6 +10,7 @@
 import errno
 import io
 import multiprocessing
+import multiprocessing.connection
 import os
 import re
 import select

lib/spack/spack/abi.py

@@ -1,131 +0,0 @@
-# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-import os
-
-from llnl.util.lang import memoized
-
-import spack.spec
-import spack.version
-from spack.compilers.clang import Clang
-from spack.util.executable import Executable, ProcessError
-
-
-class ABI:
-    """This class provides methods to test ABI compatibility between specs.
-    The current implementation is rather rough and could be improved."""
-
-    def architecture_compatible(
-        self, target: spack.spec.Spec, constraint: spack.spec.Spec
-    ) -> bool:
-        """Return true if architecture of target spec is ABI compatible
-        to the architecture of constraint spec. If either the target
-        or constraint specs have no architecture, target is also defined
-        as architecture ABI compatible to constraint."""
-        return (
-            not target.architecture
-            or not constraint.architecture
-            or target.architecture.intersects(constraint.architecture)
-        )
-
-    @memoized
-    def _gcc_get_libstdcxx_version(self, version):
-        """Returns gcc ABI compatibility info by getting the library version of
-        a compiler's libstdc++ or libgcc_s"""
-        from spack.build_environment import dso_suffix
-
-        spec = spack.spec.CompilerSpec("gcc", version)
-        compilers = spack.compilers.compilers_for_spec(spec)
-        if not compilers:
-            return None
-        compiler = compilers[0]
-        rungcc = None
-        libname = None
-        output = None
-        if compiler.cxx:
-            rungcc = Executable(compiler.cxx)
-            libname = "libstdc++." + dso_suffix
-        elif compiler.cc:
-            rungcc = Executable(compiler.cc)
-            libname = "libgcc_s." + dso_suffix
-        else:
-            return None
-        try:
-            # Some gcc's are actually clang and don't respond properly to
-            # --print-file-name (they just print the filename, not the
-            # full path).  Ignore these and expect them to be handled as clang.
-            if Clang.default_version(rungcc.exe[0]) != "unknown":
-                return None
-
-            output = rungcc("--print-file-name=%s" % libname, output=str)
-        except ProcessError:
-            return None
-        if not output:
-            return None
-        libpath = os.path.realpath(output.strip())
-        if not libpath:
-            return None
-        return os.path.basename(libpath)
-
-    @memoized
-    def _gcc_compiler_compare(self, pversion, cversion):
-        """Returns true iff the gcc version pversion and cversion
-        are ABI compatible."""
-        plib = self._gcc_get_libstdcxx_version(pversion)
-        clib = self._gcc_get_libstdcxx_version(cversion)
-        if not plib or not clib:
-            return False
-        return plib == clib
-
-    def _intel_compiler_compare(
-        self, pversion: spack.version.ClosedOpenRange, cversion: spack.version.ClosedOpenRange
-    ) -> bool:
-        """Returns true iff the intel version pversion and cversion
-        are ABI compatible"""
-
-        # Test major and minor versions.  Ignore build version.
-        pv = pversion.lo
-        cv = cversion.lo
-        return pv.up_to(2) == cv.up_to(2)
-
-    def compiler_compatible(
-        self, parent: spack.spec.Spec, child: spack.spec.Spec, loose: bool = False
-    ) -> bool:
-        """Return true if compilers for parent and child are ABI compatible."""
-        if not parent.compiler or not child.compiler:
-            return True
-
-        if parent.compiler.name != child.compiler.name:
-            # Different compiler families are assumed ABI incompatible
-            return False
-
-        if loose:
-            return True
-
-        # TODO: Can we move the specialized ABI matching stuff
-        # TODO: into compiler classes?
-        for pversion in parent.compiler.versions:
-            for cversion in child.compiler.versions:
-                # For a few compilers use specialized comparisons.
-                # Otherwise match on version match.
-                if pversion.intersects(cversion):
-                    return True
-                elif parent.compiler.name == "gcc" and self._gcc_compiler_compare(
-                    pversion, cversion
-                ):
-                    return True
-                elif parent.compiler.name == "intel" and self._intel_compiler_compare(
-                    pversion, cversion
-                ):
-                    return True
-        return False
-
-    def compatible(
-        self, target: spack.spec.Spec, constraint: spack.spec.Spec, loose: bool = False
-    ) -> bool:
-        """Returns true if target spec is ABI compatible to constraint spec"""
-        return self.architecture_compatible(target, constraint) and self.compiler_compatible(
-            target, constraint, loose=loose
-        )

lib/spack/spack/audit.py

@@ -39,9 +39,9 @@ def _search_duplicate_compilers(error_cls):
 import collections
 import collections.abc
 import glob
-import inspect
 import io
 import itertools
+import os
 import pathlib
 import pickle
 import re
@@ -210,6 +210,11 @@ def _search_duplicate_compilers(error_cls):
     group="configs", tag="CFG-PACKAGES", description="Sanity checks on packages.yaml", kwargs=()
 )
 
+#: Sanity checks on packages.yaml
+config_repos = AuditClass(
+    group="configs", tag="CFG-REPOS", description="Sanity checks on repositories", kwargs=()
+)
+
 
 @config_packages
 def _search_duplicate_specs_in_externals(error_cls):
@@ -252,40 +257,6 @@ def _search_duplicate_specs_in_externals(error_cls):
     return errors
 
 
-@config_packages
-def _deprecated_preferences(error_cls):
-    """Search package preferences deprecated in v0.21 (and slated for removal in v0.23)"""
-    # TODO (v0.23): remove this audit as the attributes will not be allowed in config
-    errors = []
-    packages_yaml = spack.config.CONFIG.get_config("packages")
-
-    def make_error(attribute_name, config_data, summary):
-        s = io.StringIO()
-        s.write("Occurring in the following file:\n")
-        dict_view = syaml.syaml_dict((k, v) for k, v in config_data.items() if k == attribute_name)
-        syaml.dump_config(dict_view, stream=s, blame=True)
-        return error_cls(summary=summary, details=[s.getvalue()])
-
-    if "all" in packages_yaml and "version" in packages_yaml["all"]:
-        summary = "Using the deprecated 'version' attribute under 'packages:all'"
-        errors.append(make_error("version", packages_yaml["all"], summary))
-
-    for package_name in packages_yaml:
-        if package_name == "all":
-            continue
-
-        package_conf = packages_yaml[package_name]
-        for attribute in ("compiler", "providers", "target"):
-            if attribute not in package_conf:
-                continue
-            summary = (
-                f"Using the deprecated '{attribute}' attribute " f"under 'packages:{package_name}'"
-            )
-            errors.append(make_error(attribute, package_conf, summary))
-
-    return errors
-
-
 @config_packages
 def _avoid_mismatched_variants(error_cls):
     """Warns if variant preferences have mismatched types or names."""
@@ -351,6 +322,43 @@ def _wrongly_named_spec(error_cls):
     return errors
 
 
+@config_packages
+def _ensure_all_virtual_packages_have_default_providers(error_cls):
+    """All virtual packages must have a default provider explicitly set."""
+    configuration = spack.config.create()
+    defaults = configuration.get("packages", scope="defaults")
+    default_providers = defaults["all"]["providers"]
+    virtuals = spack.repo.PATH.provider_index.providers
+    default_providers_filename = configuration.scopes["defaults"].get_section_filename("packages")
+
+    return [
+        error_cls(f"'{virtual}' must have a default provider in {default_providers_filename}", [])
+        for virtual in virtuals
+        if virtual not in default_providers
+    ]
+
+
+@config_repos
+def _ensure_no_folders_without_package_py(error_cls):
+    """Check that we don't leave any folder without a package.py in repos"""
+    errors = []
+    for repository in spack.repo.PATH.repos:
+        missing = []
+        for entry in os.scandir(repository.packages_path):
+            if not entry.is_dir():
+                continue
+            package_py = pathlib.Path(entry.path) / spack.repo.package_file_name
+            if not package_py.exists():
+                missing.append(entry.path)
+        if missing:
+            summary = (
+                f"The '{repository.namespace}' repository misses a package.py file"
+                f" in the following folders"
+            )
+            errors.append(error_cls(summary=summary, details=[f"{x}" for x in missing]))
+    return errors
+
+
 def _make_config_error(config_data, summary, error_cls):
     s = io.StringIO()
     s.write("Occurring in the following file:\n")
@@ -482,7 +490,7 @@ def _search_for_reserved_attributes_names_in_packages(pkgs, error_cls):
         name_definitions = collections.defaultdict(list)
         pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
 
-        for cls_item in inspect.getmro(pkg_cls):
+        for cls_item in pkg_cls.__mro__:
             for name in RESERVED_NAMES:
                 current_value = cls_item.__dict__.get(name)
                 if current_value is None:
@@ -511,7 +519,7 @@ def _ensure_all_package_names_are_lowercase(pkgs, error_cls):
     badname_regex, errors = re.compile(r"[_A-Z]"), []
     for pkg_name in pkgs:
         if badname_regex.search(pkg_name):
-            error_msg = "Package name '{}' is either lowercase or conatine '_'".format(pkg_name)
+            error_msg = f"Package name '{pkg_name}' should be lowercase and must not contain '_'"
             errors.append(error_cls(error_msg, []))
     return errors
 

lib/spack/spack/bootstrap/__init__.py

@@ -9,6 +9,7 @@
     all_core_root_specs,
     ensure_clingo_importable_or_raise,
     ensure_core_dependencies,
+    ensure_file_in_path_or_raise,
     ensure_gpg_in_path_or_raise,
     ensure_patchelf_in_path_or_raise,
 )
@@ -19,6 +20,7 @@
     "is_bootstrapping",
     "ensure_bootstrap_configuration",
     "ensure_core_dependencies",
+    "ensure_file_in_path_or_raise",
     "ensure_gpg_in_path_or_raise",
     "ensure_clingo_importable_or_raise",
     "ensure_patchelf_in_path_or_raise",

lib/spack/spack/bootstrap/_common.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Common basic functions used through the spack.bootstrap package"""
 import fnmatch
+import importlib
 import os.path
 import re
 import sys
@@ -28,7 +29,7 @@
 
 def _python_import(module: str) -> bool:
     try:
-        __import__(module)
+        importlib.import_module(module)
     except ImportError:
         return False
     return True

lib/spack/spack/bootstrap/clingo.py

@@ -0,0 +1,154 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Bootstrap concrete specs for clingo
+
+Spack uses clingo to concretize specs. When clingo itself needs to be bootstrapped from sources,
+we need to rely on another mechanism to get a concrete spec that fits the current host.
+
+This module contains the logic to get a concrete spec for clingo, starting from a prototype
+JSON file for a similar platform.
+"""
+import pathlib
+import sys
+from typing import Dict, Optional, Tuple
+
+import archspec.cpu
+
+import spack.compiler
+import spack.compilers
+import spack.platforms
+import spack.spec
+import spack.traverse
+
+from .config import spec_for_current_python
+
+
+class ClingoBootstrapConcretizer:
+    def __init__(self, configuration):
+        self.host_platform = spack.platforms.host()
+        self.host_os = self.host_platform.operating_system("frontend")
+        self.host_target = archspec.cpu.host().family
+        self.host_architecture = spack.spec.ArchSpec.frontend_arch()
+        self.host_architecture.target = str(self.host_target)
+        self.host_compiler = self._valid_compiler_or_raise()
+        self.host_python = self.python_external_spec()
+        if str(self.host_platform) == "linux":
+            self.host_libc = self.libc_external_spec()
+
+        self.external_cmake, self.external_bison = self._externals_from_yaml(configuration)
+
+    def _valid_compiler_or_raise(self) -> "spack.compiler.Compiler":
+        if str(self.host_platform) == "linux":
+            compiler_name = "gcc"
+        elif str(self.host_platform) == "darwin":
+            compiler_name = "apple-clang"
+        elif str(self.host_platform) == "windows":
+            compiler_name = "msvc"
+        elif str(self.host_platform) == "freebsd":
+            compiler_name = "clang"
+        else:
+            raise RuntimeError(f"Cannot bootstrap clingo from sources on {self.host_platform}")
+        candidates = spack.compilers.compilers_for_spec(
+            compiler_name, arch_spec=self.host_architecture
+        )
+        if not candidates:
+            raise RuntimeError(
+                f"Cannot find any version of {compiler_name} to bootstrap clingo from sources"
+            )
+        candidates.sort(key=lambda x: x.spec.version, reverse=True)
+        return candidates[0]
+
+    def _externals_from_yaml(
+        self, configuration: "spack.config.Configuration"
+    ) -> Tuple[Optional["spack.spec.Spec"], Optional["spack.spec.Spec"]]:
+        packages_yaml = configuration.get("packages")
+        requirements = {"cmake": "@3.20:", "bison": "@2.5:"}
+        selected: Dict[str, Optional["spack.spec.Spec"]] = {"cmake": None, "bison": None}
+        for pkg_name in ["cmake", "bison"]:
+            if pkg_name not in packages_yaml:
+                continue
+
+            candidates = packages_yaml[pkg_name].get("externals", [])
+            for candidate in candidates:
+                s = spack.spec.Spec(candidate["spec"], external_path=candidate["prefix"])
+                if not s.satisfies(requirements[pkg_name]):
+                    continue
+
+                if not s.intersects(f"%{self.host_compiler.spec}"):
+                    continue
+
+                if not s.intersects(f"arch={self.host_architecture}"):
+                    continue
+
+                selected[pkg_name] = self._external_spec(s)
+                break
+        return selected["cmake"], selected["bison"]
+
+    def prototype_path(self) -> pathlib.Path:
+        """Path to a prototype concrete specfile for clingo"""
+        parent_dir = pathlib.Path(__file__).parent
+        result = parent_dir / "prototypes" / f"clingo-{self.host_platform}-{self.host_target}.json"
+        if str(self.host_platform) == "linux":
+            # Using aarch64 as a fallback, since it has gnuconfig (x86_64 doesn't have it)
+            if not result.exists():
+                result = parent_dir / "prototypes" / f"clingo-{self.host_platform}-aarch64.json"
+
+        elif str(self.host_platform) == "freebsd":
+            result = parent_dir / "prototypes" / f"clingo-{self.host_platform}-amd64.json"
+
+        elif not result.exists():
+            raise RuntimeError(f"Cannot bootstrap clingo from sources on {self.host_platform}")
+
+        return result
+
+    def concretize(self) -> "spack.spec.Spec":
+        # Read the prototype and mark it NOT concrete
+        s = spack.spec.Spec.from_specfile(str(self.prototype_path()))
+        s._mark_concrete(False)
+
+        # Tweak it to conform to the host architecture
+        for node in s.traverse():
+            node.architecture.os = str(self.host_os)
+            node.compiler = self.host_compiler.spec
+            node.architecture = self.host_architecture
+
+            if node.name == "gcc-runtime":
+                node.versions = self.host_compiler.spec.versions
+
+        for edge in spack.traverse.traverse_edges([s], cover="edges"):
+            if edge.spec.name == "python":
+                edge.spec = self.host_python
+
+            if edge.spec.name == "bison" and self.external_bison:
+                edge.spec = self.external_bison
+
+            if edge.spec.name == "cmake" and self.external_cmake:
+                edge.spec = self.external_cmake
+
+            if "libc" in edge.virtuals:
+                edge.spec = self.host_libc
+
+        s._finalize_concretization()
+
+        # Work around the fact that the installer calls Spec.dependents() and
+        # we modified edges inconsistently
+        return s.copy()
+
+    def python_external_spec(self) -> "spack.spec.Spec":
+        """Python external spec corresponding to the current running interpreter"""
+        result = spack.spec.Spec(spec_for_current_python(), external_path=sys.exec_prefix)
+        return self._external_spec(result)
+
+    def libc_external_spec(self) -> "spack.spec.Spec":
+        result = self.host_compiler.default_libc
+        return self._external_spec(result)
+
+    def _external_spec(self, initial_spec) -> "spack.spec.Spec":
+        initial_spec.namespace = "builtin"
+        initial_spec.compiler = self.host_compiler.spec
+        initial_spec.architecture = self.host_architecture
+        for flag_type in spack.spec.FlagMap.valid_compiler_flags():
+            initial_spec.compiler_flags[flag_type] = []
+        return spack.spec.parse_with_version_concrete(initial_spec)

lib/spack/spack/bootstrap/config.py

@@ -143,11 +143,7 @@ def _bootstrap_config_scopes() -> Sequence["spack.config.ConfigScope"]:
 def _add_compilers_if_missing() -> None:
     arch = spack.spec.ArchSpec.frontend_arch()
     if not spack.compilers.compilers_for_arch(arch):
-        new_compilers = spack.compilers.find_new_compilers(
-            mixed_toolchain=sys.platform == "darwin"
-        )
-        if new_compilers:
-            spack.compilers.add_compilers_to_config(new_compilers)
+        spack.compilers.find_compilers()
 
 
 @contextlib.contextmanager
@@ -156,7 +152,7 @@ def _ensure_bootstrap_configuration() -> Generator:
     bootstrap_store_path = store_path()
     user_configuration = _read_and_sanitize_configuration()
     with spack.environment.no_active_environment():
-        with spack.platforms.prevent_cray_detection(), spack.platforms.use_platform(
+        with spack.platforms.use_platform(
             spack.platforms.real_host()
         ), spack.repo.use_repositories(spack.paths.packages_path):
             # Default configuration scopes excluding command line

lib/spack/spack/bootstrap/core.py

@@ -54,6 +54,7 @@
 import spack.version
 
 from ._common import _executables_in_store, _python_import, _root_spec, _try_import_from_store
+from .clingo import ClingoBootstrapConcretizer
 from .config import spack_python_interpreter, spec_for_current_python
 
 #: Name of the file containing metadata about the bootstrapping source
@@ -268,15 +269,13 @@ def try_import(self, module: str, abstract_spec_str: str) -> bool:
 
         # Try to build and install from sources
         with spack_python_interpreter():
-            # Add hint to use frontend operating system on Cray
-            concrete_spec = spack.spec.Spec(abstract_spec_str + " ^" + spec_for_current_python())
-
             if module == "clingo":
-                # TODO: remove when the old concretizer is deprecated  # pylint: disable=fixme
-                concrete_spec._old_concretize(  # pylint: disable=protected-access
-                    deprecation_warning=False
-                )
+                bootstrapper = ClingoBootstrapConcretizer(configuration=spack.config.CONFIG)
+                concrete_spec = bootstrapper.concretize()
             else:
+                concrete_spec = spack.spec.Spec(
+                    abstract_spec_str + " ^" + spec_for_current_python()
+                )
                 concrete_spec.concretize()
 
         msg = "[BOOTSTRAP MODULE {0}] Try installing '{1}' from sources"
@@ -303,14 +302,7 @@ def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bo
         # might reduce compilation time by a fair amount
         _add_externals_if_missing()
 
-        concrete_spec = spack.spec.Spec(abstract_spec_str)
-        if concrete_spec.name == "patchelf":
-            concrete_spec._old_concretize(  # pylint: disable=protected-access
-                deprecation_warning=False
-            )
-        else:
-            concrete_spec.concretize()
-
+        concrete_spec = spack.spec.Spec(abstract_spec_str).concretized()
         msg = "[BOOTSTRAP] Try installing '{0}' from sources"
         tty.debug(msg.format(abstract_spec_str))
         with spack.config.override(self.mirror_scope):
@@ -480,7 +472,8 @@ def ensure_clingo_importable_or_raise() -> None:
 
 def gnupg_root_spec() -> str:
     """Return the root spec used to bootstrap GnuPG"""
-    return _root_spec("gnupg@2.3:")
+    root_spec_name = "win-gpg" if IS_WINDOWS else "gnupg"
+    return _root_spec(f"{root_spec_name}@2.3:")
 
 
 def ensure_gpg_in_path_or_raise() -> None:
@@ -490,6 +483,19 @@ def ensure_gpg_in_path_or_raise() -> None:
     )
 
 
+def file_root_spec() -> str:
+    """Return the root spec used to bootstrap file"""
+    root_spec_name = "win-file" if IS_WINDOWS else "file"
+    return _root_spec(root_spec_name)
+
+
+def ensure_file_in_path_or_raise() -> None:
+    """Ensure file is in the PATH or raise"""
+    return ensure_executables_in_path_or_raise(
+        executables=["file"], abstract_spec=file_root_spec()
+    )
+
+
 def patchelf_root_spec() -> str:
     """Return the root spec used to bootstrap patchelf"""
     # 0.13.1 is the last version not to require C++17.
@@ -573,14 +579,15 @@ def ensure_core_dependencies() -> None:
     """Ensure the presence of all the core dependencies."""
     if sys.platform.lower() == "linux":
         ensure_patchelf_in_path_or_raise()
-    if not IS_WINDOWS:
-        ensure_gpg_in_path_or_raise()
+    elif sys.platform == "win32":
+        ensure_file_in_path_or_raise()
+    ensure_gpg_in_path_or_raise()
     ensure_clingo_importable_or_raise()
 
 
 def all_core_root_specs() -> List[str]:
     """Return a list of all the core root specs that may be used to bootstrap Spack"""
-    return [clingo_root_spec(), gnupg_root_spec(), patchelf_root_spec()]
+    return [clingo_root_spec(), gnupg_root_spec(), patchelf_root_spec(), file_root_spec()]
 
 
 def bootstrapping_sources(scope: Optional[str] = None):

lib/spack/spack/bootstrap/status.py

@@ -88,7 +88,7 @@ def _core_requirements() -> List[RequiredResponseType]:
 
 def _buildcache_requirements() -> List[RequiredResponseType]:
     _buildcache_exes = {
-        "file": _missing("file", "required to analyze files for buildcaches"),
+        "file": _missing("file", "required to analyze files for buildcaches", system_only=False),
         ("gpg2", "gpg"): _missing("gpg2", "required to sign/verify buildcaches", False),
     }
     if platform.system().lower() == "darwin":
@@ -124,7 +124,7 @@ def _development_requirements() -> List[RequiredResponseType]:
     # Ensure we trigger environment modifications if we have an environment
     if BootstrapEnvironment.spack_yaml().exists():
         with BootstrapEnvironment() as env:
-            env.update_syspath_and_environ()
+            env.load()
 
     return [
         _required_executable(

lib/spack/spack/build_environment.py

@@ -457,9 +457,12 @@ def set_wrapper_variables(pkg, env):
     env.set(SPACK_DEBUG_LOG_ID, pkg.spec.format("{name}-{hash:7}"))
     env.set(SPACK_DEBUG_LOG_DIR, spack.main.spack_working_dir)
 
-    # Find ccache binary and hand it to build environment
     if spack.config.get("config:ccache"):
+        # Enable ccache in the compiler wrapper
         env.set(SPACK_CCACHE_BINARY, spack.util.executable.which_string("ccache", required=True))
+    else:
+        # Avoid cache pollution if a build system forces `ccache <compiler wrapper invocation>`.
+        env.set("CCACHE_DISABLE", "1")
 
     # Gather information about various types of dependencies
     link_deps = set(pkg.spec.traverse(root=False, deptype=("link")))
@@ -1000,7 +1003,6 @@ def set_all_package_py_globals(self):
         """Set the globals in modules of package.py files."""
         for dspec, flag in chain(self.external, self.nonexternal):
             pkg = dspec.package
-
             if self.should_set_package_py_globals & flag:
                 if self.context == Context.BUILD and self.needs_build_context & flag:
                     set_package_py_globals(pkg, context=Context.BUILD)
@@ -1008,6 +1010,12 @@ def set_all_package_py_globals(self):
                     # This includes runtime dependencies, also runtime deps of direct build deps.
                     set_package_py_globals(pkg, context=Context.RUN)
 
+        # Looping over the set of packages a second time
+        # ensures all globals are loaded into the module space prior to
+        # any package setup. This guarantees package setup methods have
+        # access to expected module level definitions such as "spack_cc"
+        for dspec, flag in chain(self.external, self.nonexternal):
+            pkg = dspec.package
             for spec in dspec.dependents():
                 # Note: some specs have dependents that are unreachable from the root, so avoid
                 # setting globals for those.
@@ -1473,7 +1481,7 @@ def long_message(self):
             out.write("  {0}\n".format(self.log_name))
 
         # Also output the test log path IF it exists
-        if self.context != "test":
+        if self.context != "test" and have_log:
             test_log = join_path(os.path.dirname(self.log_name), spack_install_test_log)
             if os.path.isfile(test_log):
                 out.write("\nSee test log for details:\n")
@@ -1550,21 +1558,21 @@ class ModuleChangePropagator:
 
     _PROTECTED_NAMES = ("package", "current_module", "modules_in_mro", "_set_attributes")
 
-    def __init__(self, package):
+    def __init__(self, package: spack.package_base.PackageBase) -> None:
         self._set_self_attributes("package", package)
         self._set_self_attributes("current_module", package.module)
 
         #: Modules for the classes in the MRO up to PackageBase
         modules_in_mro = []
-        for cls in inspect.getmro(type(package)):
-            module = cls.module
+        for cls in package.__class__.__mro__:
+            module = getattr(cls, "module", None)
 
-            if module == self.current_module:
-                continue
-
-            if module == spack.package_base:
+            if module is None or module is spack.package_base:
                 break
 
+            if module is self.current_module:
+                continue
+
             modules_in_mro.append(module)
         self._set_self_attributes("modules_in_mro", modules_in_mro)
         self._set_self_attributes("_set_attributes", {})

lib/spack/spack/build_systems/autotools.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 import os
 import os.path
 import stat
@@ -549,13 +548,12 @@ def autoreconf(self, pkg, spec, prefix):
         tty.warn("*        a custom AUTORECONF phase in the package       *")
         tty.warn("*********************************************************")
         with fs.working_dir(self.configure_directory):
-            m = inspect.getmodule(self.pkg)
             # This line is what is needed most of the time
             # --install, --verbose, --force
             autoreconf_args = ["-ivf"]
             autoreconf_args += self.autoreconf_search_path_args
             autoreconf_args += self.autoreconf_extra_args
-            m.autoreconf(*autoreconf_args)
+            self.pkg.module.autoreconf(*autoreconf_args)
 
     @property
     def autoreconf_search_path_args(self):
@@ -579,7 +577,9 @@ def set_configure_or_die(self):
             raise RuntimeError(msg.format(self.configure_directory))
 
         # Monkey-patch the configure script in the corresponding module
-        inspect.getmodule(self.pkg).configure = Executable(self.configure_abs_path)
+        globals_for_pkg = spack.build_environment.ModuleChangePropagator(self.pkg)
+        globals_for_pkg.configure = Executable(self.configure_abs_path)
+        globals_for_pkg.propagate_changes_to_mro()
 
     def configure_args(self):
         """Return the list of all the arguments that must be passed to configure,
@@ -596,7 +596,7 @@ def configure(self, pkg, spec, prefix):
         options += self.configure_args()
 
         with fs.working_dir(self.build_directory, create=True):
-            inspect.getmodule(self.pkg).configure(*options)
+            pkg.module.configure(*options)
 
     def build(self, pkg, spec, prefix):
         """Run "make" on the build targets specified by the builder."""
@@ -604,12 +604,12 @@ def build(self, pkg, spec, prefix):
         params = ["V=1"]
         params += self.build_targets
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make(*params)
+            pkg.module.make(*params)
 
     def install(self, pkg, spec, prefix):
         """Run "make" on the install targets specified by the builder."""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make(*self.install_targets)
+            pkg.module.make(*self.install_targets)
 
     spack.builder.run_after("build")(execute_build_time_tests)
 

lib/spack/spack/build_systems/cargo.py

@@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import inspect
-
 import llnl.util.filesystem as fs
 
 import spack.builder
@@ -72,9 +70,7 @@ def check_args(self):
     def build(self, pkg, spec, prefix):
         """Runs ``cargo install`` in the source directory"""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(pkg).cargo(
-                "install", "--root", "out", "--path", ".", *self.build_args
-            )
+            pkg.module.cargo("install", "--root", "out", "--path", ".", *self.build_args)
 
     def install(self, pkg, spec, prefix):
         """Copy build files into package prefix."""
@@ -86,4 +82,4 @@ def install(self, pkg, spec, prefix):
     def check(self):
         """Run "cargo test"."""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).cargo("test", *self.check_args)
+            self.pkg.module.cargo("test", *self.check_args)

lib/spack/spack/build_systems/cmake.py

@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import collections.abc
-import inspect
 import os
 import pathlib
 import platform
@@ -108,6 +107,11 @@ def _conditional_cmake_defaults(pkg: spack.package_base.PackageBase, args: List[
     if _supports_compilation_databases(pkg):
         args.append(CMakeBuilder.define("CMAKE_EXPORT_COMPILE_COMMANDS", True))
 
+    # Enable MACOSX_RPATH by default when cmake_minimum_required < 3
+    # https://cmake.org/cmake/help/latest/policy/CMP0042.html
+    if pkg.spec.satisfies("platform=darwin") and cmake.satisfies("@3:"):
+        args.append(CMakeBuilder.define("CMAKE_POLICY_DEFAULT_CMP0042", "NEW"))
+
 
 def generator(*names: str, default: Optional[str] = None):
     """The build system generator to use.
@@ -539,24 +543,24 @@ def cmake(self, pkg, spec, prefix):
         options += self.cmake_args()
         options.append(os.path.abspath(self.root_cmakelists_dir))
         with fs.working_dir(self.build_directory, create=True):
-            inspect.getmodule(self.pkg).cmake(*options)
+            pkg.module.cmake(*options)
 
     def build(self, pkg, spec, prefix):
         """Make the build targets"""
         with fs.working_dir(self.build_directory):
             if self.generator == "Unix Makefiles":
-                inspect.getmodule(self.pkg).make(*self.build_targets)
+                pkg.module.make(*self.build_targets)
             elif self.generator == "Ninja":
                 self.build_targets.append("-v")
-                inspect.getmodule(self.pkg).ninja(*self.build_targets)
+                pkg.module.ninja(*self.build_targets)
 
     def install(self, pkg, spec, prefix):
         """Make the install targets"""
         with fs.working_dir(self.build_directory):
             if self.generator == "Unix Makefiles":
-                inspect.getmodule(self.pkg).make(*self.install_targets)
+                pkg.module.make(*self.install_targets)
             elif self.generator == "Ninja":
-                inspect.getmodule(self.pkg).ninja(*self.install_targets)
+                pkg.module.ninja(*self.install_targets)
 
     spack.builder.run_after("build")(execute_build_time_tests)
 

lib/spack/spack/build_systems/cuda.py

@@ -124,6 +124,8 @@ def cuda_flags(arch_list):
         # minimum supported versions
         conflicts("%gcc@:4", when="+cuda ^cuda@11.0:")
         conflicts("%gcc@:5", when="+cuda ^cuda@11.4:")
+        conflicts("%gcc@:7.2", when="+cuda ^cuda@12.4:")
+        conflicts("%clang@:6", when="+cuda ^cuda@12.2:")
 
         # maximum supported version
         # NOTE:
@@ -136,7 +138,7 @@ def cuda_flags(arch_list):
         conflicts("%gcc@11.2:", when="+cuda ^cuda@:11.5")
         conflicts("%gcc@12:", when="+cuda ^cuda@:11.8")
         conflicts("%gcc@13:", when="+cuda ^cuda@:12.3")
-        conflicts("%gcc@14:", when="+cuda ^cuda@:12.5")
+        conflicts("%gcc@14:", when="+cuda ^cuda@:12.6")
         conflicts("%clang@12:", when="+cuda ^cuda@:11.4.0")
         conflicts("%clang@13:", when="+cuda ^cuda@:11.5")
         conflicts("%clang@14:", when="+cuda ^cuda@:11.7")
@@ -144,6 +146,7 @@ def cuda_flags(arch_list):
         conflicts("%clang@16:", when="+cuda ^cuda@:12.1")
         conflicts("%clang@17:", when="+cuda ^cuda@:12.3")
         conflicts("%clang@18:", when="+cuda ^cuda@:12.5")
+        conflicts("%clang@19:", when="+cuda ^cuda@:12.6")
 
         # https://gist.github.com/ax3l/9489132#gistcomment-3860114
         conflicts("%gcc@10", when="+cuda ^cuda@:11.4.0")
@@ -211,12 +214,16 @@ def cuda_flags(arch_list):
         conflicts("%intel@19.0:", when="+cuda ^cuda@:10.0")
         conflicts("%intel@19.1:", when="+cuda ^cuda@:10.1")
         conflicts("%intel@19.2:", when="+cuda ^cuda@:11.1.0")
+        conflicts("%intel@2021:", when="+cuda ^cuda@:11.4.0")
 
         # XL is mostly relevant for ppc64le Linux
         conflicts("%xl@:12,14:", when="+cuda ^cuda@:9.1")
         conflicts("%xl@:12,14:15,17:", when="+cuda ^cuda@9.2")
         conflicts("%xl@:12,17:", when="+cuda ^cuda@:11.1.0")
 
+        # PowerPC.
+        conflicts("target=ppc64le", when="+cuda ^cuda@12.5:")
+
         # Darwin.
         # TODO: add missing conflicts for %apple-clang cuda@:10
-        conflicts("platform=darwin", when="+cuda ^cuda@11.0.2: ")
+        conflicts("platform=darwin", when="+cuda ^cuda@11.0.2:")

lib/spack/spack/build_systems/go.py

@@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import inspect
-
 import llnl.util.filesystem as fs
 
 import spack.builder
@@ -72,7 +70,7 @@ def build_directory(self):
     def build_args(self):
         """Arguments for ``go build``."""
         # Pass ldflags -s = --strip-all and -w = --no-warnings by default
-        return ["-ldflags", "-s -w", "-o", f"{self.pkg.name}"]
+        return ["-modcacherw", "-ldflags", "-s -w", "-o", f"{self.pkg.name}"]
 
     @property
     def check_args(self):
@@ -82,7 +80,7 @@ def check_args(self):
     def build(self, pkg, spec, prefix):
         """Runs ``go build`` in the source directory"""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(pkg).go("build", *self.build_args)
+            pkg.module.go("build", *self.build_args)
 
     def install(self, pkg, spec, prefix):
         """Install built binaries into prefix bin."""
@@ -95,4 +93,4 @@ def install(self, pkg, spec, prefix):
     def check(self):
         """Run ``go test .`` in the source directory"""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).go("test", *self.check_args)
+            self.pkg.module.go("test", *self.check_args)

lib/spack/spack/build_systems/makefile.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 from typing import List
 
 import llnl.util.filesystem as fs
@@ -103,12 +102,12 @@ def edit(self, pkg, spec, prefix):
     def build(self, pkg, spec, prefix):
         """Run "make" on the build targets specified by the builder."""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make(*self.build_targets)
+            pkg.module.make(*self.build_targets)
 
     def install(self, pkg, spec, prefix):
         """Run "make" on the install targets specified by the builder."""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make(*self.install_targets)
+            pkg.module.make(*self.install_targets)
 
     spack.builder.run_after("build")(execute_build_time_tests)
 

lib/spack/spack/build_systems/meson.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 import os
 from typing import List
 
@@ -195,19 +194,19 @@ def meson(self, pkg, spec, prefix):
         options += self.std_meson_args
         options += self.meson_args()
         with fs.working_dir(self.build_directory, create=True):
-            inspect.getmodule(self.pkg).meson(*options)
+            pkg.module.meson(*options)
 
     def build(self, pkg, spec, prefix):
         """Make the build targets"""
         options = ["-v"]
         options += self.build_targets
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).ninja(*options)
+            pkg.module.ninja(*options)
 
     def install(self, pkg, spec, prefix):
         """Make the install targets"""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).ninja(*self.install_targets)
+            pkg.module.ninja(*self.install_targets)
 
     spack.builder.run_after("build")(execute_build_time_tests)
 

lib/spack/spack/build_systems/msbuild.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 from typing import List  # novm
 
 import llnl.util.filesystem as fs
@@ -104,7 +103,7 @@ def msbuild_install_args(self):
     def build(self, pkg, spec, prefix):
         """Run "msbuild" on the build targets specified by the builder."""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).msbuild(
+            pkg.module.msbuild(
                 *self.std_msbuild_args,
                 *self.msbuild_args(),
                 self.define_targets(*self.build_targets),
@@ -114,6 +113,6 @@ def install(self, pkg, spec, prefix):
         """Run "msbuild" on the install targets specified by the builder.
         This is INSTALL by default"""
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).msbuild(
+            pkg.module.msbuild(
                 *self.msbuild_install_args(), self.define_targets(*self.install_targets)
             )

lib/spack/spack/build_systems/nmake.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 from typing import List  # novm
 
 import llnl.util.filesystem as fs
@@ -132,9 +131,7 @@ def build(self, pkg, spec, prefix):
         if self.makefile_name:
             opts.append("/F{}".format(self.makefile_name))
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).nmake(
-                *opts, *self.build_targets, ignore_quotes=self.ignore_quotes
-            )
+            pkg.module.nmake(*opts, *self.build_targets, ignore_quotes=self.ignore_quotes)
 
     def install(self, pkg, spec, prefix):
         """Run "nmake" on the install targets specified by the builder.
@@ -146,6 +143,4 @@ def install(self, pkg, spec, prefix):
             opts.append("/F{}".format(self.makefile_name))
         opts.append(self.define("PREFIX", fs.windows_sfn(prefix)))
         with fs.working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).nmake(
-                *opts, *self.install_targets, ignore_quotes=self.ignore_quotes
-            )
+            pkg.module.nmake(*opts, *self.install_targets, ignore_quotes=self.ignore_quotes)

lib/spack/spack/build_systems/octave.py

@@ -2,8 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
-
 import spack.builder
 import spack.package_base
 from spack.directives import build_system, extends
@@ -47,7 +45,7 @@ class OctaveBuilder(BaseBuilder):
 
     def install(self, pkg, spec, prefix):
         """Install the package from the archive file"""
-        inspect.getmodule(self.pkg).octave(
+        pkg.module.octave(
             "--quiet",
             "--norc",
             "--built-in-docstrings-file=/dev/null",

lib/spack/spack/build_systems/oneapi.py

@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Common utilities for managing intel oneapi packages."""
-import getpass
 import os
 import platform
 import shutil
@@ -13,6 +12,7 @@
 from llnl.util.filesystem import HeaderList, LibraryList, find_libraries, join_path, mkdirp
 from llnl.util.link_tree import LinkTree
 
+import spack.util.path
 from spack.build_environment import dso_suffix
 from spack.directives import conflicts, license, redistribute, variant
 from spack.package_base import InstallError
@@ -99,7 +99,7 @@ def install_component(self, installer_path):
             # with other install depends on the userid. For root, we
             # delete the installercache before and after install. For
             # non root we redefine the HOME environment variable.
-            if getpass.getuser() == "root":
+            if spack.util.path.get_user() == "root":
                 shutil.rmtree("/var/intel/installercache", ignore_errors=True)
 
             bash = Executable("bash")
@@ -122,7 +122,7 @@ def install_component(self, installer_path):
                 self.prefix,
             )
 
-            if getpass.getuser() == "root":
+            if spack.util.path.get_user() == "root":
                 shutil.rmtree("/var/intel/installercache", ignore_errors=True)
 
         # Some installers have a bug and do not return an error code when failing

lib/spack/spack/build_systems/perl.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 import os
 from typing import Iterable
 
@@ -134,7 +133,7 @@ def build_method(self):
     def build_executable(self):
         """Returns the executable method to build the perl package"""
         if self.build_method == "Makefile.PL":
-            build_executable = inspect.getmodule(self.pkg).make
+            build_executable = self.pkg.module.make
         elif self.build_method == "Build.PL":
             build_executable = Executable(os.path.join(self.pkg.stage.source_path, "Build"))
         return build_executable
@@ -158,7 +157,7 @@ def configure(self, pkg, spec, prefix):
             options = ["Build.PL", "--install_base", prefix]
         options += self.configure_args()
 
-        inspect.getmodule(self.pkg).perl(*options)
+        pkg.module.perl(*options)
 
     # It is possible that the shebang in the Build script that is created from
     # Build.PL may be too long causing the build to fail. Patching the shebang

lib/spack/spack/build_systems/python.py

@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import functools
-import inspect
 import operator
 import os
 import re
@@ -17,7 +16,7 @@
 import llnl.util.filesystem as fs
 import llnl.util.lang as lang
 import llnl.util.tty as tty
-from llnl.util.filesystem import HeaderList, LibraryList
+from llnl.util.filesystem import HeaderList, LibraryList, join_path
 
 import spack.builder
 import spack.config
@@ -120,6 +119,12 @@ def skip_modules(self) -> Iterable[str]:
         """
         return []
 
+    @property
+    def bindir(self) -> str:
+        """Path to Python package's bindir, bin on unix like OS's Scripts on Windows"""
+        windows = self.spec.satisfies("platform=windows")
+        return join_path(self.spec.prefix, "Scripts" if windows else "bin")
+
     def view_file_conflicts(self, view, merge_map):
         """Report all file conflicts, excepting special cases for python.
         Specifically, this does not report errors for duplicate
@@ -222,7 +227,7 @@ def test_imports(self) -> None:
 
         # Make sure we are importing the installed modules,
         # not the ones in the source directory
-        python = inspect.getmodule(self).python  # type: ignore[union-attr]
+        python = self.module.python
         for module in self.import_modules:
             with test_part(
                 self,
@@ -309,9 +314,9 @@ def get_external_python_for_prefix(self):
         )
 
         python_externals_detected = [
-            d.spec
-            for d in python_externals_detection.get("python", [])
-            if d.prefix == self.spec.external_path
+            spec
+            for spec in python_externals_detection.get("python", [])
+            if spec.external_path == self.spec.external_path
         ]
         if python_externals_detected:
             return python_externals_detected[0]

lib/spack/spack/build_systems/qmake.py

@@ -2,8 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
-
 from llnl.util.filesystem import working_dir
 
 import spack.builder
@@ -66,17 +64,17 @@ def qmake_args(self):
     def qmake(self, pkg, spec, prefix):
         """Run ``qmake`` to configure the project and generate a Makefile."""
         with working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).qmake(*self.qmake_args())
+            pkg.module.qmake(*self.qmake_args())
 
     def build(self, pkg, spec, prefix):
         """Make the build targets"""
         with working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make()
+            pkg.module.make()
 
     def install(self, pkg, spec, prefix):
         """Make the install targets"""
         with working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make("install")
+            pkg.module.make("install")
 
     def check(self):
         """Search the Makefile for a ``check:`` target and runs it if found."""

lib/spack/spack/build_systems/r.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 from typing import Optional, Tuple
 
 import llnl.util.lang as lang
@@ -51,7 +50,7 @@ def install(self, pkg, spec, prefix):
 
         args.extend(["--library={0}".format(self.pkg.module.r_lib_dir), self.stage.source_path])
 
-        inspect.getmodule(self.pkg).R(*args)
+        pkg.module.R(*args)
 
 
 class RPackage(Package):

lib/spack/spack/build_systems/ruby.py

@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import glob
-import inspect
 
 import spack.builder
 import spack.package_base
@@ -52,10 +51,10 @@ def build(self, pkg, spec, prefix):
         gemspecs = glob.glob("*.gemspec")
         rakefiles = glob.glob("Rakefile")
         if gemspecs:
-            inspect.getmodule(self.pkg).gem("build", "--norc", gemspecs[0])
+            pkg.module.gem("build", "--norc", gemspecs[0])
         elif rakefiles:
-            jobs = inspect.getmodule(self.pkg).make_jobs
-            inspect.getmodule(self.pkg).rake("package", "-j{0}".format(jobs))
+            jobs = pkg.module.make_jobs
+            pkg.module.rake("package", "-j{0}".format(jobs))
         else:
             # Some Ruby packages only ship `*.gem` files, so nothing to build
             pass
@@ -70,6 +69,6 @@ def install(self, pkg, spec, prefix):
             # if --install-dir is not used, GEM_PATH is deleted from the
             # environement, and Gems required to build native extensions will
             # not be found. Those extensions are built during `gem install`.
-            inspect.getmodule(self.pkg).gem(
+            pkg.module.gem(
                 "install", "--norc", "--ignore-dependencies", "--install-dir", prefix, gems[0]
             )

lib/spack/spack/build_systems/scons.py

@@ -2,8 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
-
 import spack.builder
 import spack.package_base
 from spack.directives import build_system, depends_on
@@ -63,8 +61,7 @@ def build_args(self, spec, prefix):
 
     def build(self, pkg, spec, prefix):
         """Build the package."""
-        args = self.build_args(spec, prefix)
-        inspect.getmodule(self.pkg).scons(*args)
+        pkg.module.scons(*self.build_args(spec, prefix))
 
     def install_args(self, spec, prefix):
         """Arguments to pass to install."""
@@ -72,9 +69,7 @@ def install_args(self, spec, prefix):
 
     def install(self, pkg, spec, prefix):
         """Install the package."""
-        args = self.install_args(spec, prefix)
-
-        inspect.getmodule(self.pkg).scons("install", *args)
+        pkg.module.scons("install", *self.install_args(spec, prefix))
 
     def build_test(self):
         """Run unit tests after build.

lib/spack/spack/build_systems/sip.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
 import os
 import re
 
@@ -86,14 +85,13 @@ def import_modules(self):
 
     def python(self, *args, **kwargs):
         """The python ``Executable``."""
-        inspect.getmodule(self).python(*args, **kwargs)
+        self.pkg.module.python(*args, **kwargs)
 
     def test_imports(self):
         """Attempts to import modules of the installed package."""
 
         # Make sure we are importing the installed modules,
         # not the ones in the source directory
-        python = inspect.getmodule(self).python
         for module in self.import_modules:
             with spack.install_test.test_part(
                 self,
@@ -101,7 +99,7 @@ def test_imports(self):
                 purpose="checking import of {0}".format(module),
                 work_dir="spack-test",
             ):
-                python("-c", "import {0}".format(module))
+                self.python("-c", "import {0}".format(module))
 
 
 @spack.builder.builder("sip")
@@ -136,9 +134,13 @@ def configure(self, pkg, spec, prefix):
         """Configure the package."""
 
         # https://www.riverbankcomputing.com/static/Docs/sip/command_line_tools.html
-        args = ["--verbose", "--target-dir", inspect.getmodule(self.pkg).python_platlib]
+        args = ["--verbose", "--target-dir", pkg.module.python_platlib]
         args.extend(self.configure_args())
 
+        # https://github.com/Python-SIP/sip/commit/cb0be6cb6e9b756b8b0db3136efb014f6fb9b766
+        if spec["py-sip"].satisfies("@6.1.0:"):
+            args.extend(["--scripts-dir", pkg.prefix.bin])
+
         sip_build = Executable(spec["py-sip"].prefix.bin.join("sip-build"))
         sip_build(*args)
 
@@ -151,7 +153,7 @@ def build(self, pkg, spec, prefix):
         args = self.build_args()
 
         with working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make(*args)
+            pkg.module.make(*args)
 
     def build_args(self):
         """Arguments to pass to build."""
@@ -162,7 +164,7 @@ def install(self, pkg, spec, prefix):
         args = self.install_args()
 
         with working_dir(self.build_directory):
-            inspect.getmodule(self.pkg).make("install", *args)
+            pkg.module.make("install", *args)
 
     def install_args(self):
         """Arguments to pass to install."""

lib/spack/spack/build_systems/waf.py

@@ -2,8 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import inspect
-
 from llnl.util.filesystem import working_dir
 
 import spack.builder
@@ -90,11 +88,11 @@ def build_directory(self):
 
     def python(self, *args, **kwargs):
         """The python ``Executable``."""
-        inspect.getmodule(self.pkg).python(*args, **kwargs)
+        self.pkg.module.python(*args, **kwargs)
 
     def waf(self, *args, **kwargs):
         """Runs the waf ``Executable``."""
-        jobs = inspect.getmodule(self.pkg).make_jobs
+        jobs = self.pkg.module.make_jobs
 
         with working_dir(self.build_directory):
             self.python("waf", "-j{0}".format(jobs), *args, **kwargs)

lib/spack/spack/builder.py

@@ -6,12 +6,12 @@
 import collections.abc
 import copy
 import functools
-import inspect
 from typing import List, Optional, Tuple
 
 from llnl.util import lang
 
 import spack.build_environment
+import spack.multimethod
 
 #: Builder classes, as registered by the "builder" decorator
 BUILDER_CLS = {}
@@ -96,11 +96,10 @@ class hierarchy (look at AspellDictPackage for an example of that)
     Args:
         pkg (spack.package_base.PackageBase): package object for which we need a builder
     """
-    package_module = inspect.getmodule(pkg)
     package_buildsystem = buildsystem_name(pkg)
     default_builder_cls = BUILDER_CLS[package_buildsystem]
     builder_cls_name = default_builder_cls.__name__
-    builder_cls = getattr(package_module, builder_cls_name, None)
+    builder_cls = getattr(pkg.module, builder_cls_name, None)
     if builder_cls:
         return builder_cls(pkg)
 
@@ -295,7 +294,11 @@ def _decorator(fn):
         return _decorator
 
 
-class BuilderMeta(PhaseCallbacksMeta, type(collections.abc.Sequence)):  # type: ignore
+class BuilderMeta(
+    PhaseCallbacksMeta,
+    spack.multimethod.MultiMethodMeta,
+    type(collections.abc.Sequence),  # type: ignore
+):
     pass
 
 

lib/spack/spack/caches.py

@@ -9,11 +9,11 @@
 
 import llnl.util.lang
 from llnl.util.filesystem import mkdirp
-from llnl.util.symlink import symlink
 
 import spack.config
 import spack.error
 import spack.fetch_strategy
+import spack.mirror
 import spack.paths
 import spack.util.file_cache
 import spack.util.path
@@ -74,23 +74,6 @@ def store(self, fetcher, relative_dest):
         mkdirp(os.path.dirname(dst))
         fetcher.archive(dst)
 
-    def symlink(self, mirror_ref):
-        """Symlink a human readible path in our mirror to the actual
-        storage location."""
-
-        cosmetic_path = os.path.join(self.root, mirror_ref.cosmetic_path)
-        storage_path = os.path.join(self.root, mirror_ref.storage_path)
-        relative_dst = os.path.relpath(storage_path, start=os.path.dirname(cosmetic_path))
-
-        if not os.path.exists(cosmetic_path):
-            if os.path.lexists(cosmetic_path):
-                # In this case the link itself exists but it is broken: remove
-                # it and recreate it (in order to fix any symlinks broken prior
-                # to https://github.com/spack/spack/pull/13908)
-                os.unlink(cosmetic_path)
-            mkdirp(os.path.dirname(cosmetic_path))
-            symlink(relative_dst, cosmetic_path)
-
 
 #: Spack's local cache for downloaded source archives
 FETCH_CACHE: Union[spack.fetch_strategy.FsCache, llnl.util.lang.Singleton] = (

lib/spack/spack/ci.py

@@ -38,6 +38,7 @@
 import spack.paths
 import spack.repo
 import spack.spec
+import spack.stage
 import spack.util.git
 import spack.util.gpg as gpg_util
 import spack.util.spack_yaml as syaml
@@ -71,7 +72,7 @@
 # TODO: Remove this in Spack 0.23
 SHARED_PR_MIRROR_URL = "s3://spack-binaries-prs/shared_pr_mirror"
 JOB_NAME_FORMAT = (
-    "{name}{@version} {/hash:7} {%compiler.name}{@compiler.version}{arch=architecture}"
+    "{name}{@version} {/hash:7} {%compiler.name}{@compiler.version}{ arch=architecture}"
 )
 IS_WINDOWS = sys.platform == "win32"
 spack_gpg = spack.main.SpackCommand("gpg")
@@ -1107,9 +1108,10 @@ def main_script_replacements(cmd):
     if cdash_handler and cdash_handler.auth_token:
         try:
             cdash_handler.populate_buildgroup(all_job_names)
-        except (SpackError, HTTPError, URLError) as err:
+        except (SpackError, HTTPError, URLError, TimeoutError) as err:
             tty.warn(f"Problem populating buildgroup: {err}")
-    else:
+    elif cdash_config:
+        # warn only if there was actually a CDash configuration.
         tty.warn("Unable to populate buildgroup without CDash credentials")
 
     service_job_retries = {
@@ -1370,15 +1372,6 @@ def can_verify_binaries():
     return len(gpg_util.public_keys()) >= 1
 
 
-def _push_to_build_cache(spec: spack.spec.Spec, sign_binaries: bool, mirror_url: str) -> None:
-    """Unchecked version of the public API, for easier mocking"""
-    bindist.push_or_raise(
-        spec,
-        spack.mirror.Mirror.from_url(mirror_url).push_url,
-        bindist.PushOptions(force=True, unsigned=not sign_binaries),
-    )
-
-
 def push_to_build_cache(spec: spack.spec.Spec, mirror_url: str, sign_binaries: bool) -> bool:
     """Push one or more binary packages to the mirror.
 
@@ -1389,20 +1382,15 @@ def push_to_build_cache(spec: spack.spec.Spec, mirror_url: str, sign_binaries: b
         sign_binaries: If True, spack will attempt to sign binary package before pushing.
     """
     tty.debug(f"Pushing to build cache ({'signed' if sign_binaries else 'unsigned'})")
+    signing_key = bindist.select_signing_key() if sign_binaries else None
+    mirror = spack.mirror.Mirror.from_url(mirror_url)
     try:
-        _push_to_build_cache(spec, sign_binaries, mirror_url)
+        with bindist.make_uploader(mirror, signing_key=signing_key) as uploader:
+            uploader.push_or_raise([spec])
         return True
     except bindist.PushToBuildCacheError as e:
-        tty.error(str(e))
+        tty.error(f"Problem writing to {mirror_url}: {e}")
         return False
-    except Exception as e:
-        # TODO (zackgalbreath): write an adapter for boto3 exceptions so we can catch a specific
-        # exception instead of parsing str(e)...
-        msg = str(e)
-        if any(x in msg for x in ["Access Denied", "InvalidAccessKeyId"]):
-            tty.error(f"Permission problem writing to {mirror_url}: {msg}")
-            return False
-        raise
 
 
 def remove_other_mirrors(mirrors_to_keep, scope=None):
@@ -1448,10 +1436,6 @@ def copy_stage_logs_to_artifacts(job_spec: spack.spec.Spec, job_log_dir: str) ->
         job_log_dir: path into which build log should be copied
     """
     tty.debug(f"job spec: {job_spec}")
-    if not job_spec:
-        msg = f"Cannot copy stage logs: job spec ({job_spec}) is required"
-        tty.error(msg)
-        return
 
     try:
         pkg_cls = spack.repo.PATH.get_pkg_class(job_spec.name)
@@ -2083,7 +2067,7 @@ def read_broken_spec(broken_spec_url):
     """
     try:
         _, _, fs = web_util.read_from_url(broken_spec_url)
-    except (URLError, web_util.SpackWebError, HTTPError):
+    except web_util.SpackWebError:
         tty.warn(f"Unable to read broken spec from {broken_spec_url}")
         return None
 

lib/spack/spack/cmd/__init__.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import argparse
+import importlib
 import os
 import re
 import sys
@@ -114,8 +115,8 @@ def get_module(cmd_name):
 
     try:
         # Try to import the command from the built-in directory
-        module_name = "%s.%s" % (__name__, pname)
-        module = __import__(module_name, fromlist=[pname, SETUP_PARSER, DESCRIPTION], level=0)
+        module_name = f"{__name__}.{pname}"
+        module = importlib.import_module(module_name)
         tty.debug("Imported {0} from built-in commands".format(pname))
     except ImportError:
         module = spack.extensions.get_module(cmd_name)
@@ -237,7 +238,7 @@ def ensure_single_spec_or_die(spec, matching_specs):
     if len(matching_specs) <= 1:
         return
 
-    format_string = "{name}{@version}{%compiler.name}{@compiler.version}{arch=architecture}"
+    format_string = "{name}{@version}{%compiler.name}{@compiler.version}{ arch=architecture}"
     args = ["%s matches multiple packages." % spec, "Matching packages:"]
     args += [
         colorize("  @K{%s} " % s.dag_hash(7)) + s.cformat(format_string) for s in matching_specs

lib/spack/spack/cmd/buildcache.py

@@ -3,28 +3,24 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import argparse
-import copy
 import glob
-import hashlib
 import json
-import multiprocessing
-import multiprocessing.pool
 import os
 import shutil
 import sys
 import tempfile
-from typing import Dict, List, Optional, Tuple, Union
+from typing import List, Tuple
 
 import llnl.util.tty as tty
 from llnl.string import plural
-from llnl.util.lang import elide_list
+from llnl.util.lang import elide_list, stable_partition
 
 import spack.binary_distribution as bindist
 import spack.cmd
 import spack.config
+import spack.deptypes as dt
 import spack.environment as ev
 import spack.error
-import spack.hash_types as ht
 import spack.mirror
 import spack.oci.oci
 import spack.oci.opener
@@ -35,28 +31,12 @@
 import spack.store
 import spack.user_environment
 import spack.util.crypto
+import spack.util.parallel
 import spack.util.url as url_util
 import spack.util.web as web_util
 from spack import traverse
-from spack.build_environment import determine_number_of_jobs
 from spack.cmd import display_specs
 from spack.cmd.common import arguments
-from spack.oci.image import (
-    Digest,
-    ImageReference,
-    default_config,
-    default_index_tag,
-    default_manifest,
-    default_tag,
-    tag_is_spec,
-)
-from spack.oci.oci import (
-    copy_missing_layers_with_retry,
-    get_manifest_and_config_with_retry,
-    list_tags,
-    upload_blob_with_retry,
-    upload_manifest_with_retry,
-)
 from spack.spec import Spec, save_dependency_specfiles
 
 description = "create, download and install binary packages"
@@ -112,6 +92,17 @@ def setup_parser(subparser: argparse.ArgumentParser):
         "Alternatively, one can decide to build a cache for only the package or only the "
         "dependencies",
     )
+    with_or_without_build_deps = push.add_mutually_exclusive_group()
+    with_or_without_build_deps.add_argument(
+        "--with-build-dependencies",
+        action="store_true",
+        help="include build dependencies in the buildcache",
+    )
+    with_or_without_build_deps.add_argument(
+        "--without-build-dependencies",
+        action="store_true",
+        help="exclude build dependencies from the buildcache",
+    )
     push.add_argument(
         "--fail-fast",
         action="store_true",
@@ -329,39 +320,6 @@ def _format_spec(spec: Spec) -> str:
     return spec.cformat("{name}{@version}{/hash:7}")
 
 
-def _progress(i: int, total: int):
-    if total > 1:
-        digits = len(str(total))
-        return f"[{i+1:{digits}}/{total}] "
-    return ""
-
-
-class NoPool:
-    def map(self, func, args):
-        return [func(a) for a in args]
-
-    def starmap(self, func, args):
-        return [func(*a) for a in args]
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        pass
-
-
-MaybePool = Union[multiprocessing.pool.Pool, NoPool]
-
-
-def _make_pool() -> MaybePool:
-    """Can't use threading because it's unsafe, and can't use spawned processes because of globals.
-    That leaves only forking"""
-    if multiprocessing.get_start_method() == "fork":
-        return multiprocessing.pool.Pool(determine_number_of_jobs(parallel=True))
-    else:
-        return NoPool()
-
-
 def _skip_no_redistribute_for_public(specs):
     remaining_specs = list()
     removed_specs = list()
@@ -381,6 +339,45 @@ def _skip_no_redistribute_for_public(specs):
     return remaining_specs
 
 
+class PackagesAreNotInstalledError(spack.error.SpackError):
+    """Raised when a list of specs is not installed but picked to be packaged."""
+
+    def __init__(self, specs: List[Spec]):
+        super().__init__(
+            "Cannot push non-installed packages",
+            ", ".join(elide_list([_format_spec(s) for s in specs], 5)),
+        )
+
+
+class PackageNotInstalledError(spack.error.SpackError):
+    """Raised when a spec is not installed but picked to be packaged."""
+
+
+def _specs_to_be_packaged(
+    requested: List[Spec], things_to_install: str, build_deps: bool
+) -> List[Spec]:
+    """Collect all non-external with or without roots and dependencies"""
+    if "dependencies" not in things_to_install:
+        deptype = dt.NONE
+    elif build_deps:
+        deptype = dt.ALL
+    else:
+        deptype = dt.RUN | dt.LINK | dt.TEST
+    specs = [
+        s
+        for s in traverse.traverse_nodes(
+            requested,
+            root="package" in things_to_install,
+            deptype=deptype,
+            order="breadth",
+            key=traverse.by_dag_hash,
+        )
+        if not s.external
+    ]
+    specs.reverse()
+    return specs
+
+
 def push_fn(args):
     """create a binary package and push it to a mirror"""
     if args.spec_file:
@@ -394,13 +391,8 @@ def push_fn(args):
     else:
         roots = spack.cmd.require_active_env(cmd_name="buildcache push").concrete_roots()
 
-    mirror: spack.mirror.Mirror = args.mirror
-
-    # Check if this is an OCI image.
-    try:
-        target_image = spack.oci.oci.image_from_mirror(mirror)
-    except ValueError:
-        target_image = None
+    mirror = args.mirror
+    assert isinstance(mirror, spack.mirror.Mirror)
 
     push_url = mirror.push_url
 
@@ -411,92 +403,52 @@ def push_fn(args):
         unsigned = not (args.key or args.signed)
 
     # For OCI images, we require dependencies to be pushed for now.
-    if target_image:
-        if "dependencies" not in args.things_to_install:
-            tty.die("Dependencies must be pushed for OCI images.")
-        if not unsigned:
-            tty.warn(
-                "Code signing is currently not supported for OCI images. "
-                "Use --unsigned to silence this warning."
-            )
+    if mirror.push_url.startswith("oci://") and not unsigned:
+        tty.warn(
+            "Code signing is currently not supported for OCI images. "
+            "Use --unsigned to silence this warning."
+        )
+        unsigned = True
 
-    # This is a list of installed, non-external specs.
-    specs = bindist.specs_to_be_packaged(
+    # Select a signing key, or None if unsigned.
+    signing_key = None if unsigned else (args.key or bindist.select_signing_key())
+
+    specs = _specs_to_be_packaged(
         roots,
-        root="package" in args.things_to_install,
-        dependencies="dependencies" in args.things_to_install,
+        things_to_install=args.things_to_install,
+        build_deps=args.with_build_dependencies or not args.without_build_dependencies,
     )
+
     if not args.private:
         specs = _skip_no_redistribute_for_public(specs)
 
-    # When pushing multiple specs, print the url once ahead of time, as well as how
-    # many specs are being pushed.
     if len(specs) > 1:
         tty.info(f"Selected {len(specs)} specs to push to {push_url}")
 
-    failed = []
-
-    # TODO: unify this logic in the future.
-    if target_image:
-        base_image = ImageReference.from_string(args.base_image) if args.base_image else None
-        with tempfile.TemporaryDirectory(
-            dir=spack.stage.get_stage_root()
-        ) as tmpdir, _make_pool() as pool:
-            skipped, base_images, checksums = _push_oci(
-                target_image=target_image,
-                base_image=base_image,
-                installed_specs_with_deps=specs,
-                force=args.force,
-                tmpdir=tmpdir,
-                pool=pool,
-            )
-
-            # Apart from creating manifests for each individual spec, we allow users to create a
-            # separate image tag for all root specs and their runtime dependencies.
-            if args.tag:
-                tagged_image = target_image.with_tag(args.tag)
-                # _push_oci may not populate base_images if binaries were already in the registry
-                for spec in roots:
-                    _update_base_images(
-                        base_image=base_image,
-                        target_image=target_image,
-                        spec=spec,
-                        base_image_cache=base_images,
-                    )
-                _put_manifest(base_images, checksums, tagged_image, tmpdir, None, None, *roots)
-                tty.info(f"Tagged {tagged_image}")
-
-    else:
-        skipped = []
-
-        for i, spec in enumerate(specs):
-            try:
-                bindist.push_or_raise(
-                    spec,
-                    push_url,
-                    bindist.PushOptions(
-                        force=args.force,
-                        unsigned=unsigned,
-                        key=args.key,
-                        regenerate_index=args.update_index,
-                    ),
+    # Pushing not installed specs is an error. Either fail fast or populate the error list and
+    # push installed package in best effort mode.
+    failed: List[Tuple[Spec, BaseException]] = []
+    with spack.store.STORE.db.read_transaction():
+        if any(not s.installed for s in specs):
+            specs, not_installed = stable_partition(specs, lambda s: s.installed)
+            if args.fail_fast:
+                raise PackagesAreNotInstalledError(not_installed)
+            else:
+                failed.extend(
+                    (s, PackageNotInstalledError("package not installed")) for s in not_installed
                 )
 
-                msg = f"{_progress(i, len(specs))}Pushed {_format_spec(spec)}"
-                if len(specs) == 1:
-                    msg += f" to {push_url}"
-                tty.info(msg)
-
-            except bindist.NoOverwriteException:
-                skipped.append(_format_spec(spec))
-
-            # Catch any other exception unless the fail fast option is set
-            except Exception as e:
-                if args.fail_fast or isinstance(
-                    e, (bindist.PickKeyException, bindist.NoKeyException)
-                ):
-                    raise
-                failed.append((_format_spec(spec), e))
+    with bindist.make_uploader(
+        mirror=mirror,
+        force=args.force,
+        update_index=args.update_index,
+        signing_key=signing_key,
+        base_image=args.base_image,
+    ) as uploader:
+        skipped, upload_errors = uploader.push(specs=specs)
+        failed.extend(upload_errors)
+        if not upload_errors and args.tag:
+            uploader.tag(args.tag, roots)
 
     if skipped:
         if len(specs) == 1:
@@ -508,7 +460,7 @@ def push_fn(args):
                 "The following {} specs were skipped as they already exist in the buildcache:\n"
                 "    {}\n"
                 "    Use --force to overwrite them.".format(
-                    len(skipped), ", ".join(elide_list(skipped, 5))
+                    len(skipped), ", ".join(elide_list([_format_spec(s) for s in skipped], 5))
                 )
             )
 
@@ -519,389 +471,15 @@ def push_fn(args):
         raise spack.error.SpackError(
             f"The following {len(failed)} errors occurred while pushing specs to the buildcache",
             "\n".join(
-                elide_list([f"    {spec}: {e.__class__.__name__}: {e}" for spec, e in failed], 5)
-            ),
-        )
-
-    # Update the index if requested
-    # TODO: remove update index logic out of bindist; should be once after all specs are pushed
-    # not once per spec.
-    if target_image and len(skipped) < len(specs) and args.update_index:
-        with tempfile.TemporaryDirectory(
-            dir=spack.stage.get_stage_root()
-        ) as tmpdir, _make_pool() as pool:
-            _update_index_oci(target_image, tmpdir, pool)
-
-
-def _get_spack_binary_blob(image_ref: ImageReference) -> Optional[spack.oci.oci.Blob]:
-    """Get the spack tarball layer digests and size if it exists"""
-    try:
-        manifest, config = get_manifest_and_config_with_retry(image_ref)
-
-        return spack.oci.oci.Blob(
-            compressed_digest=Digest.from_string(manifest["layers"][-1]["digest"]),
-            uncompressed_digest=Digest.from_string(config["rootfs"]["diff_ids"][-1]),
-            size=manifest["layers"][-1]["size"],
-        )
-    except Exception:
-        return None
-
-
-def _push_single_spack_binary_blob(image_ref: ImageReference, spec: spack.spec.Spec, tmpdir: str):
-    filename = os.path.join(tmpdir, f"{spec.dag_hash()}.tar.gz")
-
-    # Create an oci.image.layer aka tarball of the package
-    compressed_tarfile_checksum, tarfile_checksum = spack.oci.oci.create_tarball(spec, filename)
-
-    blob = spack.oci.oci.Blob(
-        Digest.from_sha256(compressed_tarfile_checksum),
-        Digest.from_sha256(tarfile_checksum),
-        os.path.getsize(filename),
-    )
-
-    # Upload the blob
-    upload_blob_with_retry(image_ref, file=filename, digest=blob.compressed_digest)
-
-    # delete the file
-    os.unlink(filename)
-
-    return blob
-
-
-def _retrieve_env_dict_from_config(config: dict) -> dict:
-    """Retrieve the environment variables from the image config file.
-    Sets a default value for PATH if it is not present.
-
-    Args:
-        config (dict): The image config file.
-
-    Returns:
-        dict: The environment variables.
-    """
-    env = {"PATH": "/bin:/usr/bin"}
-
-    if "Env" in config.get("config", {}):
-        for entry in config["config"]["Env"]:
-            key, value = entry.split("=", 1)
-            env[key] = value
-    return env
-
-
-def _archspec_to_gooarch(spec: spack.spec.Spec) -> str:
-    name = spec.target.family.name
-    name_map = {"aarch64": "arm64", "x86_64": "amd64"}
-    return name_map.get(name, name)
-
-
-def _put_manifest(
-    base_images: Dict[str, Tuple[dict, dict]],
-    checksums: Dict[str, spack.oci.oci.Blob],
-    image_ref: ImageReference,
-    tmpdir: str,
-    extra_config: Optional[dict],
-    annotations: Optional[dict],
-    *specs: spack.spec.Spec,
-):
-    architecture = _archspec_to_gooarch(specs[0])
-
-    dependencies = list(
-        reversed(
-            list(
-                s
-                for s in traverse.traverse_nodes(
-                    specs, order="topo", deptype=("link", "run"), root=True
+                elide_list(
+                    [
+                        f"    {_format_spec(spec)}: {e.__class__.__name__}: {e}"
+                        for spec, e in failed
+                    ],
+                    5,
                 )
-                if not s.external
-            )
-        )
-    )
-
-    base_manifest, base_config = base_images[architecture]
-    env = _retrieve_env_dict_from_config(base_config)
-
-    # If the base image uses `vnd.docker.distribution.manifest.v2+json`, then we use that too.
-    # This is because Singularity / Apptainer is very strict about not mixing them.
-    base_manifest_mediaType = base_manifest.get(
-        "mediaType", "application/vnd.oci.image.manifest.v1+json"
-    )
-    use_docker_format = (
-        base_manifest_mediaType == "application/vnd.docker.distribution.manifest.v2+json"
-    )
-
-    spack.user_environment.environment_modifications_for_specs(*specs).apply_modifications(env)
-
-    # Create an oci.image.config file
-    config = copy.deepcopy(base_config)
-
-    # Add the diff ids of the dependencies
-    for s in dependencies:
-        config["rootfs"]["diff_ids"].append(str(checksums[s.dag_hash()].uncompressed_digest))
-
-    # Set the environment variables
-    config["config"]["Env"] = [f"{k}={v}" for k, v in env.items()]
-
-    if extra_config:
-        # From the OCI v1.0 spec:
-        # > Any extra fields in the Image JSON struct are considered implementation
-        # > specific and MUST be ignored by any implementations which are unable to
-        # > interpret them.
-        config.update(extra_config)
-
-    config_file = os.path.join(tmpdir, f"{specs[0].dag_hash()}.config.json")
-
-    with open(config_file, "w") as f:
-        json.dump(config, f, separators=(",", ":"))
-
-    config_file_checksum = Digest.from_sha256(
-        spack.util.crypto.checksum(hashlib.sha256, config_file)
-    )
-
-    # Upload the config file
-    upload_blob_with_retry(image_ref, file=config_file, digest=config_file_checksum)
-
-    manifest = {
-        "mediaType": base_manifest_mediaType,
-        "schemaVersion": 2,
-        "config": {
-            "mediaType": base_manifest["config"]["mediaType"],
-            "digest": str(config_file_checksum),
-            "size": os.path.getsize(config_file),
-        },
-        "layers": [
-            *(layer for layer in base_manifest["layers"]),
-            *(
-                {
-                    "mediaType": (
-                        "application/vnd.docker.image.rootfs.diff.tar.gzip"
-                        if use_docker_format
-                        else "application/vnd.oci.image.layer.v1.tar+gzip"
-                    ),
-                    "digest": str(checksums[s.dag_hash()].compressed_digest),
-                    "size": checksums[s.dag_hash()].size,
-                }
-                for s in dependencies
             ),
-        ],
-    }
-
-    if not use_docker_format and annotations:
-        manifest["annotations"] = annotations
-
-    # Finally upload the manifest
-    upload_manifest_with_retry(image_ref, manifest=manifest)
-
-    # delete the config file
-    os.unlink(config_file)
-
-
-def _update_base_images(
-    *,
-    base_image: Optional[ImageReference],
-    target_image: ImageReference,
-    spec: spack.spec.Spec,
-    base_image_cache: Dict[str, Tuple[dict, dict]],
-):
-    """For a given spec and base image, copy the missing layers of the base image with matching
-    arch to the registry of the target image. If no base image is specified, create a dummy
-    manifest and config file."""
-    architecture = _archspec_to_gooarch(spec)
-    if architecture in base_image_cache:
-        return
-    if base_image is None:
-        base_image_cache[architecture] = (
-            default_manifest(),
-            default_config(architecture, "linux"),
         )
-    else:
-        base_image_cache[architecture] = copy_missing_layers_with_retry(
-            base_image, target_image, architecture
-        )
-
-
-def _push_oci(
-    *,
-    target_image: ImageReference,
-    base_image: Optional[ImageReference],
-    installed_specs_with_deps: List[Spec],
-    tmpdir: str,
-    pool: MaybePool,
-    force: bool = False,
-) -> Tuple[List[str], Dict[str, Tuple[dict, dict]], Dict[str, spack.oci.oci.Blob]]:
-    """Push specs to an OCI registry
-
-    Args:
-        image_ref: The target OCI image
-        base_image: Optional base image, which will be copied to the target registry.
-        installed_specs_with_deps: The installed specs to push, excluding externals,
-            including deps, ordered from roots to leaves.
-        force: Whether to overwrite existing layers and manifests in the buildcache.
-
-    Returns:
-        A tuple consisting of the list of skipped specs already in the build cache,
-        a dictionary mapping architectures to base image manifests and configs,
-        and a dictionary mapping each spec's dag hash to a blob.
-    """
-
-    # Reverse the order
-    installed_specs_with_deps = list(reversed(installed_specs_with_deps))
-
-    # Spec dag hash -> blob
-    checksums: Dict[str, spack.oci.oci.Blob] = {}
-
-    # arch -> (manifest, config)
-    base_images: Dict[str, Tuple[dict, dict]] = {}
-
-    # Specs not uploaded because they already exist
-    skipped = []
-
-    if not force:
-        tty.info("Checking for existing specs in the buildcache")
-        to_be_uploaded = []
-
-        tags_to_check = (target_image.with_tag(default_tag(s)) for s in installed_specs_with_deps)
-        available_blobs = pool.map(_get_spack_binary_blob, tags_to_check)
-
-        for spec, maybe_blob in zip(installed_specs_with_deps, available_blobs):
-            if maybe_blob is not None:
-                checksums[spec.dag_hash()] = maybe_blob
-                skipped.append(_format_spec(spec))
-            else:
-                to_be_uploaded.append(spec)
-    else:
-        to_be_uploaded = installed_specs_with_deps
-
-    if not to_be_uploaded:
-        return skipped, base_images, checksums
-
-    tty.info(
-        f"{len(to_be_uploaded)} specs need to be pushed to "
-        f"{target_image.domain}/{target_image.name}"
-    )
-
-    # Upload blobs
-    new_blobs = pool.starmap(
-        _push_single_spack_binary_blob, ((target_image, spec, tmpdir) for spec in to_be_uploaded)
-    )
-
-    # And update the spec to blob mapping
-    for spec, blob in zip(to_be_uploaded, new_blobs):
-        checksums[spec.dag_hash()] = blob
-
-    # Copy base images if necessary
-    for spec in to_be_uploaded:
-        _update_base_images(
-            base_image=base_image,
-            target_image=target_image,
-            spec=spec,
-            base_image_cache=base_images,
-        )
-
-    def extra_config(spec: Spec):
-        spec_dict = spec.to_dict(hash=ht.dag_hash)
-        spec_dict["buildcache_layout_version"] = 1
-        spec_dict["binary_cache_checksum"] = {
-            "hash_algorithm": "sha256",
-            "hash": checksums[spec.dag_hash()].compressed_digest.digest,
-        }
-        return spec_dict
-
-    # Upload manifests
-    tty.info("Uploading manifests")
-    pool.starmap(
-        _put_manifest,
-        (
-            (
-                base_images,
-                checksums,
-                target_image.with_tag(default_tag(spec)),
-                tmpdir,
-                extra_config(spec),
-                {"org.opencontainers.image.description": spec.format()},
-                spec,
-            )
-            for spec in to_be_uploaded
-        ),
-    )
-
-    # Print the image names of the top-level specs
-    for spec in to_be_uploaded:
-        tty.info(f"Pushed {_format_spec(spec)} to {target_image.with_tag(default_tag(spec))}")
-
-    return skipped, base_images, checksums
-
-
-def _config_from_tag(image_ref: ImageReference, tag: str) -> Optional[dict]:
-    # Don't allow recursion here, since Spack itself always uploads
-    # vnd.oci.image.manifest.v1+json, not vnd.oci.image.index.v1+json
-    _, config = get_manifest_and_config_with_retry(image_ref.with_tag(tag), tag, recurse=0)
-
-    # Do very basic validation: if "spec" is a key in the config, it
-    # must be a Spec object too.
-    return config if "spec" in config else None
-
-
-def _update_index_oci(image_ref: ImageReference, tmpdir: str, pool: MaybePool) -> None:
-    tags = list_tags(image_ref)
-
-    # Fetch all image config files in parallel
-    spec_dicts = pool.starmap(
-        _config_from_tag, ((image_ref, tag) for tag in tags if tag_is_spec(tag))
-    )
-
-    # Populate the database
-    db_root_dir = os.path.join(tmpdir, "db_root")
-    db = bindist.BuildCacheDatabase(db_root_dir)
-
-    for spec_dict in spec_dicts:
-        spec = Spec.from_dict(spec_dict)
-        db.add(spec, directory_layout=None)
-        db.mark(spec, "in_buildcache", True)
-
-    # Create the index.json file
-    index_json_path = os.path.join(tmpdir, "index.json")
-    with open(index_json_path, "w") as f:
-        db._write_to_file(f)
-
-    # Create an empty config.json file
-    empty_config_json_path = os.path.join(tmpdir, "config.json")
-    with open(empty_config_json_path, "wb") as f:
-        f.write(b"{}")
-
-    # Upload the index.json file
-    index_shasum = Digest.from_sha256(spack.util.crypto.checksum(hashlib.sha256, index_json_path))
-    upload_blob_with_retry(image_ref, file=index_json_path, digest=index_shasum)
-
-    # Upload the config.json file
-    empty_config_digest = Digest.from_sha256(
-        spack.util.crypto.checksum(hashlib.sha256, empty_config_json_path)
-    )
-    upload_blob_with_retry(image_ref, file=empty_config_json_path, digest=empty_config_digest)
-
-    # Push a manifest file that references the index.json file as a layer
-    # Notice that we push this as if it is an image, which it of course is not.
-    # When the ORAS spec becomes official, we can use that instead of a fake image.
-    # For now we just use the OCI image spec, so that we don't run into issues with
-    # automatic garbage collection of blobs that are not referenced by any image manifest.
-    oci_manifest = {
-        "mediaType": "application/vnd.oci.image.manifest.v1+json",
-        "schemaVersion": 2,
-        # Config is just an empty {} file for now, and irrelevant
-        "config": {
-            "mediaType": "application/vnd.oci.image.config.v1+json",
-            "digest": str(empty_config_digest),
-            "size": os.path.getsize(empty_config_json_path),
-        },
-        # The buildcache index is the only layer, and is not a tarball, we lie here.
-        "layers": [
-            {
-                "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
-                "digest": str(index_shasum),
-                "size": os.path.getsize(index_json_path),
-            }
-        ],
-    }
-
-    upload_manifest_with_retry(image_ref.with_tag(default_index_tag), oci_manifest)
 
 
 def install_fn(args):
@@ -1182,14 +760,15 @@ def update_index(mirror: spack.mirror.Mirror, update_keys=False):
     if image_ref:
         with tempfile.TemporaryDirectory(
             dir=spack.stage.get_stage_root()
-        ) as tmpdir, _make_pool() as pool:
-            _update_index_oci(image_ref, tmpdir, pool)
+        ) as tmpdir, spack.util.parallel.make_concurrent_executor() as executor:
+            bindist._oci_update_index(image_ref, tmpdir, executor)
         return
 
     # Otherwise, assume a normal mirror.
     url = mirror.push_url
 
-    bindist.generate_package_index(url_util.join(url, bindist.build_cache_relative_path()))
+    with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:
+        bindist._url_generate_package_index(url, tmpdir)
 
     if update_keys:
         keys_url = url_util.join(
@@ -1197,7 +776,8 @@ def update_index(mirror: spack.mirror.Mirror, update_keys=False):
         )
 
         try:
-            bindist.generate_key_index(keys_url)
+            with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:
+                bindist.generate_key_index(keys_url, tmpdir)
         except bindist.CannotListKeys as e:
             # Do not error out if listing keys went wrong. This usually means that the _gpg path
             # does not exist. TODO: distinguish between this and other errors.

lib/spack/spack/cmd/commands.py

@@ -7,11 +7,11 @@
 import copy
 import os
 import re
+import shlex
 import sys
 from argparse import ArgumentParser, Namespace
 from typing import IO, Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
 
-import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 from llnl.util.argparsewriter import ArgparseRstWriter, ArgparseWriter, Command
 from llnl.util.tty.colify import colify
@@ -19,6 +19,7 @@
 import spack.cmd
 import spack.main
 import spack.paths
+import spack.platforms
 from spack.main import section_descriptions
 
 description = "list available spack commands"
@@ -140,7 +141,7 @@ def usage(self, usage: str) -> str:
 
         cmd = self.parser.prog.replace(" ", "-")
         if cmd in self.documented:
-            string += "\n:ref:`More documentation <cmd-{0}>`\n".format(cmd)
+            string = f"{string}\n:ref:`More documentation <cmd-{cmd}>`\n"
 
         return string
 
@@ -250,33 +251,27 @@ def body(
             Function body.
         """
         if positionals:
-            return """
+            return f"""
     if $list_options
     then
-        {0}
+        {self.optionals(optionals)}
     else
-        {1}
+        {self.positionals(positionals)}
     fi
-""".format(
-                self.optionals(optionals), self.positionals(positionals)
-            )
+"""
         elif subcommands:
-            return """
+            return f"""
     if $list_options
     then
-        {0}
+        {self.optionals(optionals)}
     else
-        {1}
+        {self.subcommands(subcommands)}
     fi
-""".format(
-                self.optionals(optionals), self.subcommands(subcommands)
-            )
+"""
         else:
-            return """
-    {0}
-""".format(
-                self.optionals(optionals)
-            )
+            return f"""
+    {self.optionals(optionals)}
+"""
 
     def positionals(self, positionals: Sequence[str]) -> str:
         """Return the syntax for reporting positional arguments.
@@ -305,7 +300,7 @@ def optionals(self, optionals: Sequence[str]) -> str:
         Returns:
             Syntax for optional flags.
         """
-        return 'SPACK_COMPREPLY="{0}"'.format(" ".join(optionals))
+        return f'SPACK_COMPREPLY="{" ".join(optionals)}"'
 
     def subcommands(self, subcommands: Sequence[str]) -> str:
         """Return the syntax for reporting subcommands.
@@ -316,7 +311,7 @@ def subcommands(self, subcommands: Sequence[str]) -> str:
         Returns:
             Syntax for subcommand parsers
         """
-        return 'SPACK_COMPREPLY="{0}"'.format(" ".join(subcommands))
+        return f'SPACK_COMPREPLY="{" ".join(subcommands)}"'
 
 
 # Map argument destination names to their complete commands
@@ -396,7 +391,7 @@ def _fish_dest_get_complete(prog: str, dest: str) -> Optional[str]:
     subcmd = s[1] if len(s) == 2 else ""
 
     for (prog_key, pos_key), value in _dest_to_fish_complete.items():
-        if subcmd.startswith(prog_key) and re.match("^" + pos_key + "$", dest):
+        if subcmd.startswith(prog_key) and re.match(f"^{pos_key}$", dest):
             return value
     return None
 
@@ -428,24 +423,6 @@ def format(self, cmd: Command) -> str:
             + self.complete(cmd.prog, positionals, optionals, subcommands)
         )
 
-    def _quote(self, string: str) -> str:
-        """Quote string and escape special characters if necessary.
-
-        Args:
-            string: Input string.
-
-        Returns:
-            Quoted string.
-        """
-        # Goal here is to match fish_indent behavior
-
-        # Strings without spaces (or other special characters) do not need to be escaped
-        if not any([sub in string for sub in [" ", "'", '"']]):
-            return string
-
-        string = string.replace("'", r"\'")
-        return f"'{string}'"
-
     def optspecs(
         self,
         prog: str,
@@ -464,7 +441,7 @@ def optspecs(
         optspec_var = "__fish_spack_optspecs_" + prog.replace(" ", "_").replace("-", "_")
 
         if optionals is None:
-            return "set -g %s\n" % optspec_var
+            return f"set -g {optspec_var}\n"
 
         # Build optspec by iterating over options
         args = []
@@ -491,11 +468,11 @@ def optspecs(
             long = [f[2:] for f in flags if f.startswith("--")]
 
             while len(short) > 0 and len(long) > 0:
-                arg = "%s/%s%s" % (short.pop(), long.pop(), required)
+                arg = f"{short.pop()}/{long.pop()}{required}"
             while len(short) > 0:
-                arg = "%s/%s" % (short.pop(), required)
+                arg = f"{short.pop()}/{required}"
             while len(long) > 0:
-                arg = "%s%s" % (long.pop(), required)
+                arg = f"{long.pop()}{required}"
 
             args.append(arg)
 
@@ -504,7 +481,7 @@ def optspecs(
         # indicate that such subcommand exists.
         args = " ".join(args)
 
-        return "set -g %s %s\n" % (optspec_var, args)
+        return f"set -g {optspec_var} {args}\n"
 
     @staticmethod
     def complete_head(
@@ -525,12 +502,14 @@ def complete_head(
         subcmd = s[1] if len(s) == 2 else ""
 
         if index is None:
-            return "complete -c %s -n '__fish_spack_using_command %s'" % (s[0], subcmd)
+            return f"complete -c {s[0]} -n '__fish_spack_using_command {subcmd}'"
         elif nargs in [argparse.ZERO_OR_MORE, argparse.ONE_OR_MORE, argparse.REMAINDER]:
-            head = "complete -c %s -n '__fish_spack_using_command_pos_remainder %d %s'"
+            return (
+                f"complete -c {s[0]} -n '__fish_spack_using_command_pos_remainder "
+                f"{index} {subcmd}'"
+            )
         else:
-            head = "complete -c %s -n '__fish_spack_using_command_pos %d %s'"
-        return head % (s[0], index, subcmd)
+            return f"complete -c {s[0]} -n '__fish_spack_using_command_pos {index} {subcmd}'"
 
     def complete(
         self,
@@ -598,25 +577,18 @@ def positionals(
 
             if choices is not None:
                 # If there are choices, we provide a completion for all possible values.
-                commands.append(head + " -f -a %s" % self._quote(" ".join(choices)))
+                commands.append(f"{head} -f -a {shlex.quote(' '.join(choices))}")
             else:
                 # Otherwise, we try to find a predefined completion for it
                 value = _fish_dest_get_complete(prog, args)
                 if value is not None:
-                    commands.append(head + " " + value)
+                    commands.append(f"{head} {value}")
 
         return "\n".join(commands) + "\n"
 
     def prog_comment(self, prog: str) -> str:
-        """Return a comment line for the command.
-
-        Args:
-            prog: Program name.
-
-        Returns:
-            Comment line.
-        """
-        return "\n# %s\n" % prog
+        """Return a comment line for the command."""
+        return f"\n# {prog}\n"
 
     def optionals(
         self,
@@ -659,28 +631,28 @@ def optionals(
             for f in flags:
                 if f.startswith("--"):
                     long = f[2:]
-                    prefix += " -l %s" % long
+                    prefix = f"{prefix} -l {long}"
                 elif f.startswith("-"):
                     short = f[1:]
                     assert len(short) == 1
-                    prefix += " -s %s" % short
+                    prefix = f"{prefix} -s {short}"
 
             # Check if option require argument.
             # Currently multi-argument options are not supported, so we treat it like one argument.
             if nargs != 0:
-                prefix += " -r"
+                prefix = f"{prefix} -r"
 
             if dest is not None:
                 # If there are choices, we provide a completion for all possible values.
-                commands.append(prefix + " -f -a %s" % self._quote(" ".join(dest)))
+                commands.append(f"{prefix} -f -a {shlex.quote(' '.join(dest))}")
             else:
                 # Otherwise, we try to find a predefined completion for it
                 value = _fish_dest_get_complete(prog, dest)
                 if value is not None:
-                    commands.append(prefix + " " + value)
+                    commands.append(f"{prefix} {value}")
 
             if help:
-                commands.append(prefix + " -d %s" % self._quote(help))
+                commands.append(f"{prefix} -d {shlex.quote(help)}")
 
         return "\n".join(commands) + "\n"
 
@@ -698,11 +670,11 @@ def subcommands(self, prog: str, subcommands: List[Tuple[ArgumentParser, str, st
         head = self.complete_head(prog, 0)
 
         for _, subcommand, help in subcommands:
-            command = head + " -f -a %s" % self._quote(subcommand)
+            command = f"{head} -f -a {shlex.quote(subcommand)}"
 
             if help is not None and len(help) > 0:
                 help = help.split("\n")[0]
-                command += " -d %s" % self._quote(help)
+                command = f"{command} -d {shlex.quote(help)}"
 
             commands.append(command)
 
@@ -748,7 +720,7 @@ def rst_index(out: IO) -> None:
 
         for i, cmd in enumerate(sorted(commands)):
             description = description.capitalize() if i == 0 else ""
-            ref = ":ref:`%s <spack-%s>`" % (cmd, cmd)
+            ref = f":ref:`{cmd} <spack-{cmd}>`"
             comma = "," if i != len(commands) - 1 else ""
             bar = "| " if i % 8 == 0 else "  "
             out.write(line % (description, bar + ref + comma))
@@ -859,17 +831,14 @@ def _commands(parser: ArgumentParser, args: Namespace) -> None:
 
     # check header first so we don't open out files unnecessarily
     if args.header and not os.path.exists(args.header):
-        tty.die("No such file: '%s'" % args.header)
+        tty.die(f"No such file: '{args.header}'")
 
     if args.update:
-        tty.msg("Updating file: %s" % args.update)
+        tty.msg(f"Updating file: {args.update}")
         with open(args.update, "w") as f:
             prepend_header(args, f)
             formatter(args, f)
 
-        if args.update_completion:
-            fs.set_executable(args.update)
-
     else:
         prepend_header(args, sys.stdout)
         formatter(args, sys.stdout)

lib/spack/spack/cmd/compiler.py

@@ -50,6 +50,7 @@ def setup_parser(subparser):
         default=lambda: spack.config.default_modify_scope("compilers"),
         help="configuration scope to modify",
     )
+    arguments.add_common_arguments(find_parser, ["jobs"])
 
     # Remove
     remove_parser = sp.add_parser("remove", aliases=["rm"], help="remove compiler by spec")
@@ -78,25 +79,21 @@ def setup_parser(subparser):
 def compiler_find(args):
     """Search either $PATH or a list of paths OR MODULES for compilers and
     add them to Spack's configuration.
-
     """
-    # None signals spack.compiler.find_compilers to use its default logic
     paths = args.add_paths or None
-
-    # Below scope=None because we want new compilers that don't appear
-    # in any other configuration.
-    new_compilers = spack.compilers.find_new_compilers(
-        paths, scope=None, mixed_toolchain=args.mixed_toolchain
+    new_compilers = spack.compilers.find_compilers(
+        path_hints=paths,
+        scope=args.scope,
+        mixed_toolchain=args.mixed_toolchain,
+        max_workers=args.jobs,
     )
     if new_compilers:
-        spack.compilers.add_compilers_to_config(new_compilers, scope=args.scope)
         n = len(new_compilers)
         s = "s" if n > 1 else ""
-
-        config = spack.config.CONFIG
-        filename = config.get_config_filename(args.scope, "compilers")
-        tty.msg("Added %d new compiler%s to %s" % (n, s, filename))
-        colify(reversed(sorted(c.spec.display_str for c in new_compilers)), indent=4)
+        filename = spack.config.CONFIG.get_config_filename(args.scope, "compilers")
+        tty.msg(f"Added {n:d} new compiler{s} to {filename}")
+        compiler_strs = sorted(f"{c.spec.name}@{c.spec.version}" for c in new_compilers)
+        colify(reversed(compiler_strs), indent=4)
     else:
         tty.msg("Found no new compilers")
     tty.msg("Compilers are defined in the following files:")

lib/spack/spack/cmd/create.py

@@ -6,6 +6,7 @@
 import re
 import sys
 import urllib.parse
+from typing import List
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import mkdirp
@@ -14,9 +15,15 @@
 import spack.stage
 import spack.util.web
 from spack.spec import Spec
-from spack.url import UndetectableNameError, UndetectableVersionError, parse_name, parse_version
+from spack.url import (
+    UndetectableNameError,
+    UndetectableVersionError,
+    find_versions_of_archive,
+    parse_name,
+    parse_version,
+)
 from spack.util.editor import editor
-from spack.util.executable import ProcessError, which
+from spack.util.executable import which
 from spack.util.format import get_version_lines
 from spack.util.naming import mod_to_class, simplify_name, valid_fully_qualified_module_name
 
@@ -89,14 +96,20 @@ class BundlePackageTemplate:
     url_def = "    # There is no URL since there is no code to download."
     body_def = "    # There is no need for install() since there is no code."
 
-    def __init__(self, name, versions):
+    def __init__(self, name: str, versions, languages: List[str]):
         self.name = name
         self.class_name = mod_to_class(name)
         self.versions = versions
+        self.languages = languages
 
     def write(self, pkg_path):
         """Writes the new package file."""
 
+        all_deps = [f'    depends_on("{lang}", type="build")' for lang in self.languages]
+        if all_deps and self.dependencies:
+            all_deps.append("")
+        all_deps.append(self.dependencies)
+
         # Write out a template for the file
         with open(pkg_path, "w") as pkg_file:
             pkg_file.write(
@@ -106,7 +119,7 @@ def write(self, pkg_path):
                     base_class_name=self.base_class_name,
                     url_def=self.url_def,
                     versions=self.versions,
-                    dependencies=self.dependencies,
+                    dependencies="\n".join(all_deps),
                     body_def=self.body_def,
                 )
             )
@@ -125,8 +138,8 @@ def install(self, spec, prefix):
 
     url_line = '    url = "{url}"'
 
-    def __init__(self, name, url, versions):
-        super().__init__(name, versions)
+    def __init__(self, name, url, versions, languages: List[str]):
+        super().__init__(name, versions, languages)
 
         self.url_def = self.url_line.format(url=url)
 
@@ -214,13 +227,13 @@ def luarocks_args(self):
         args = []
         return args"""
 
-    def __init__(self, name, url, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name lua-lpeg`, don't rename it lua-lua-lpeg
         if not name.startswith("lua-"):
             # Make it more obvious that we are renaming the package
             tty.msg("Changing package name from {0} to lua-{0}".format(name))
             name = "lua-{0}".format(name)
-        super().__init__(name, url, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class MesonPackageTemplate(PackageTemplate):
@@ -321,14 +334,14 @@ class RacketPackageTemplate(PackageTemplate):
     # subdirectory = None
     """
 
-    def __init__(self, name, url, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name rkt-scribble`, don't rename it rkt-rkt-scribble
         if not name.startswith("rkt-"):
             # Make it more obvious that we are renaming the package
             tty.msg("Changing package name from {0} to rkt-{0}".format(name))
             name = "rkt-{0}".format(name)
         self.body_def = self.body_def.format(name[4:])
-        super().__init__(name, url, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class PythonPackageTemplate(PackageTemplate):
@@ -361,7 +374,7 @@ def config_settings(self, spec, prefix):
         settings = {}
         return settings"""
 
-    def __init__(self, name, url, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name py-numpy`, don't rename it py-py-numpy
         if not name.startswith("py-"):
             # Make it more obvious that we are renaming the package
@@ -415,7 +428,7 @@ def __init__(self, name, url, *args, **kwargs):
                 + self.url_line
             )
 
-        super().__init__(name, url, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class RPackageTemplate(PackageTemplate):
@@ -434,7 +447,7 @@ def configure_args(self):
         args = []
         return args"""
 
-    def __init__(self, name, url, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name r-rcpp`, don't rename it r-r-rcpp
         if not name.startswith("r-"):
             # Make it more obvious that we are renaming the package
@@ -454,7 +467,7 @@ def __init__(self, name, url, *args, **kwargs):
         if bioc:
             self.url_line = '    url = "{0}"\n' '    bioc = "{1}"'.format(url, r_name)
 
-        super().__init__(name, url, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class PerlmakePackageTemplate(PackageTemplate):
@@ -474,14 +487,14 @@ def configure_args(self):
         args = []
         return args"""
 
-    def __init__(self, name, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name perl-cpp`, don't rename it perl-perl-cpp
         if not name.startswith("perl-"):
             # Make it more obvious that we are renaming the package
             tty.msg("Changing package name from {0} to perl-{0}".format(name))
             name = "perl-{0}".format(name)
 
-        super().__init__(name, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class PerlbuildPackageTemplate(PerlmakePackageTemplate):
@@ -506,7 +519,7 @@ class OctavePackageTemplate(PackageTemplate):
     # FIXME: Add additional dependencies if required.
     # depends_on("octave-foo", type=("build", "run"))"""
 
-    def __init__(self, name, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name octave-splines`, don't rename it
         # octave-octave-splines
         if not name.startswith("octave-"):
@@ -514,7 +527,7 @@ def __init__(self, name, *args, **kwargs):
             tty.msg("Changing package name from {0} to octave-{0}".format(name))
             name = "octave-{0}".format(name)
 
-        super().__init__(name, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class RubyPackageTemplate(PackageTemplate):
@@ -534,7 +547,7 @@ def build(self, spec, prefix):
         # FIXME: If not needed delete this function
         pass"""
 
-    def __init__(self, name, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name ruby-numpy`, don't rename it
         # ruby-ruby-numpy
         if not name.startswith("ruby-"):
@@ -542,7 +555,7 @@ def __init__(self, name, *args, **kwargs):
             tty.msg("Changing package name from {0} to ruby-{0}".format(name))
             name = "ruby-{0}".format(name)
 
-        super().__init__(name, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 class MakefilePackageTemplate(PackageTemplate):
@@ -580,14 +593,14 @@ def configure_args(self, spec, prefix):
         args = []
         return args"""
 
-    def __init__(self, name, *args, **kwargs):
+    def __init__(self, name, url, versions, languages: List[str]):
         # If the user provided `--name py-pyqt4`, don't rename it py-py-pyqt4
         if not name.startswith("py-"):
             # Make it more obvious that we are renaming the package
             tty.msg("Changing package name from {0} to py-{0}".format(name))
             name = "py-{0}".format(name)
 
-        super().__init__(name, *args, **kwargs)
+        super().__init__(name, url, versions, languages)
 
 
 templates = {
@@ -658,8 +671,48 @@ def setup_parser(subparser):
     )
 
 
-class BuildSystemGuesser:
-    """An instance of BuildSystemGuesser provides a callable object to be used
+#: C file extensions
+C_EXT = {".c"}
+
+#: C++ file extensions
+CXX_EXT = {
+    ".C",
+    ".c++",
+    ".cc",
+    ".ccm",
+    ".cpp",
+    ".CPP",
+    ".cxx",
+    ".h++",
+    ".hh",
+    ".hpp",
+    ".hxx",
+    ".inl",
+    ".ipp",
+    ".ixx",
+    ".tcc",
+    ".tpp",
+}
+
+#: Fortran file extensions
+FORTRAN_EXT = {
+    ".f77",
+    ".F77",
+    ".f90",
+    ".F90",
+    ".f95",
+    ".F95",
+    ".f",
+    ".F",
+    ".for",
+    ".FOR",
+    ".ftn",
+    ".FTN",
+}
+
+
+class BuildSystemAndLanguageGuesser:
+    """An instance of BuildSystemAndLanguageGuesser provides a callable object to be used
     during ``spack create``. By passing this object to ``spack checksum``, we
     can take a peek at the fetched tarball and discern the build system it uses
     """
@@ -667,81 +720,119 @@ class BuildSystemGuesser:
     def __init__(self):
         """Sets the default build system."""
         self.build_system = "generic"
+        self._c = False
+        self._cxx = False
+        self._fortran = False
 
-    def __call__(self, stage, url):
+        # List of files in the archive ordered by their depth in the directory tree.
+        self._file_entries: List[str] = []
+
+    def __call__(self, archive: str, url: str) -> None:
         """Try to guess the type of build system used by a project based on
         the contents of its archive or the URL it was downloaded from."""
 
-        if url is not None:
-            # Most octave extensions are hosted on Octave-Forge:
-            #     https://octave.sourceforge.net/index.html
-            # They all have the same base URL.
-            if "downloads.sourceforge.net/octave/" in url:
-                self.build_system = "octave"
-                return
-            if url.endswith(".gem"):
-                self.build_system = "ruby"
-                return
-            if url.endswith(".whl") or ".whl#" in url:
-                self.build_system = "python"
-                return
-            if url.endswith(".rock"):
-                self.build_system = "lua"
-                return
-
-        # A list of clues that give us an idea of the build system a package
-        # uses. If the regular expression matches a file contained in the
-        # archive, the corresponding build system is assumed.
-        # NOTE: Order is important here. If a package supports multiple
-        # build systems, we choose the first match in this list.
-        clues = [
-            (r"/CMakeLists\.txt$", "cmake"),
-            (r"/NAMESPACE$", "r"),
-            (r"/Cargo\.toml$", "cargo"),
-            (r"/go\.mod$", "go"),
-            (r"/configure$", "autotools"),
-            (r"/configure\.(in|ac)$", "autoreconf"),
-            (r"/Makefile\.am$", "autoreconf"),
-            (r"/pom\.xml$", "maven"),
-            (r"/SConstruct$", "scons"),
-            (r"/waf$", "waf"),
-            (r"/pyproject.toml", "python"),
-            (r"/setup\.(py|cfg)$", "python"),
-            (r"/WORKSPACE$", "bazel"),
-            (r"/Build\.PL$", "perlbuild"),
-            (r"/Makefile\.PL$", "perlmake"),
-            (r"/.*\.gemspec$", "ruby"),
-            (r"/Rakefile$", "ruby"),
-            (r"/setup\.rb$", "ruby"),
-            (r"/.*\.pro$", "qmake"),
-            (r"/.*\.rockspec$", "lua"),
-            (r"/(GNU)?[Mm]akefile$", "makefile"),
-            (r"/DESCRIPTION$", "octave"),
-            (r"/meson\.build$", "meson"),
-            (r"/configure\.py$", "sip"),
-        ]
-
         # Peek inside the compressed file.
-        if stage.archive_file.endswith(".zip") or ".zip#" in stage.archive_file:
+        if archive.endswith(".zip") or ".zip#" in archive:
             try:
                 unzip = which("unzip")
-                output = unzip("-lq", stage.archive_file, output=str)
-            except ProcessError:
+                assert unzip is not None
+                output = unzip("-lq", archive, output=str)
+            except Exception:
                 output = ""
         else:
             try:
                 tar = which("tar")
-                output = tar("--exclude=*/*/*", "-tf", stage.archive_file, output=str)
-            except ProcessError:
+                assert tar is not None
+                output = tar("tf", archive, output=str)
+            except Exception:
                 output = ""
-        lines = output.splitlines()
+        self._file_entries[:] = output.splitlines()
 
-        # Determine the build system based on the files contained
-        # in the archive.
-        for pattern, bs in clues:
-            if any(re.search(pattern, line) for line in lines):
-                self.build_system = bs
-                break
+        # Files closest to the root should be considered first when determining build system.
+        self._file_entries.sort(key=lambda p: p.count("/"))
+
+        self._determine_build_system(url)
+        self._determine_language()
+
+    def _determine_build_system(self, url: str) -> None:
+        # Most octave extensions are hosted on Octave-Forge:
+        #     https://octave.sourceforge.net/index.html
+        # They all have the same base URL.
+        if "downloads.sourceforge.net/octave/" in url:
+            self.build_system = "octave"
+        elif url.endswith(".gem"):
+            self.build_system = "ruby"
+        elif url.endswith(".whl") or ".whl#" in url:
+            self.build_system = "python"
+        elif url.endswith(".rock"):
+            self.build_system = "lua"
+        elif self._file_entries:
+            # A list of clues that give us an idea of the build system a package
+            # uses. If the regular expression matches a file contained in the
+            # archive, the corresponding build system is assumed.
+            # NOTE: Order is important here. If a package supports multiple
+            # build systems, we choose the first match in this list.
+            clues = [
+                (re.compile(pattern), build_system)
+                for pattern, build_system in (
+                    (r"/CMakeLists\.txt$", "cmake"),
+                    (r"/NAMESPACE$", "r"),
+                    (r"/Cargo\.toml$", "cargo"),
+                    (r"/go\.mod$", "go"),
+                    (r"/configure$", "autotools"),
+                    (r"/configure\.(in|ac)$", "autoreconf"),
+                    (r"/Makefile\.am$", "autoreconf"),
+                    (r"/pom\.xml$", "maven"),
+                    (r"/SConstruct$", "scons"),
+                    (r"/waf$", "waf"),
+                    (r"/pyproject.toml", "python"),
+                    (r"/setup\.(py|cfg)$", "python"),
+                    (r"/WORKSPACE$", "bazel"),
+                    (r"/Build\.PL$", "perlbuild"),
+                    (r"/Makefile\.PL$", "perlmake"),
+                    (r"/.*\.gemspec$", "ruby"),
+                    (r"/Rakefile$", "ruby"),
+                    (r"/setup\.rb$", "ruby"),
+                    (r"/.*\.pro$", "qmake"),
+                    (r"/.*\.rockspec$", "lua"),
+                    (r"/(GNU)?[Mm]akefile$", "makefile"),
+                    (r"/DESCRIPTION$", "octave"),
+                    (r"/meson\.build$", "meson"),
+                    (r"/configure\.py$", "sip"),
+                )
+            ]
+
+            # Determine the build system based on the files contained in the archive.
+            for file in self._file_entries:
+                for pattern, build_system in clues:
+                    if pattern.search(file):
+                        self.build_system = build_system
+                        return
+
+    def _determine_language(self):
+        for entry in self._file_entries:
+            _, ext = os.path.splitext(entry)
+
+            if not self._c and ext in C_EXT:
+                self._c = True
+            elif not self._cxx and ext in CXX_EXT:
+                self._cxx = True
+            elif not self._fortran and ext in FORTRAN_EXT:
+                self._fortran = True
+
+            if self._c and self._cxx and self._fortran:
+                return
+
+    @property
+    def languages(self) -> List[str]:
+        langs: List[str] = []
+        if self._c:
+            langs.append("c")
+        if self._cxx:
+            langs.append("cxx")
+        if self._fortran:
+            langs.append("fortran")
+        return langs
 
 
 def get_name(name, url):
@@ -811,7 +902,7 @@ def get_url(url):
 def get_versions(args, name):
     """Returns a list of versions and hashes for a package.
 
-    Also returns a BuildSystemGuesser object.
+    Also returns a BuildSystemAndLanguageGuesser object.
 
     Returns default values if no URL is provided.
 
@@ -820,7 +911,7 @@ def get_versions(args, name):
         name (str): The name of the package
 
     Returns:
-        tuple: versions and hashes, and a BuildSystemGuesser object
+        tuple: versions and hashes, and a BuildSystemAndLanguageGuesser object
     """
 
     # Default version with hash
@@ -834,7 +925,7 @@ def get_versions(args, name):
     # version("1.2.4")"""
 
     # Default guesser
-    guesser = BuildSystemGuesser()
+    guesser = BuildSystemAndLanguageGuesser()
 
     valid_url = True
     try:
@@ -847,7 +938,7 @@ def get_versions(args, name):
     if args.url is not None and args.template != "bundle" and valid_url:
         # Find available versions
         try:
-            url_dict = spack.url.find_versions_of_archive(args.url)
+            url_dict = find_versions_of_archive(args.url)
             if len(url_dict) > 1 and not args.batch and sys.stdin.isatty():
                 url_dict_filtered = spack.stage.interactive_version_filter(url_dict)
                 if url_dict_filtered is None:
@@ -874,7 +965,7 @@ def get_versions(args, name):
     return versions, guesser
 
 
-def get_build_system(template, url, guesser):
+def get_build_system(template: str, url: str, guesser: BuildSystemAndLanguageGuesser) -> str:
     """Determine the build system template.
 
     If a template is specified, always use that. Otherwise, if a URL
@@ -882,11 +973,10 @@ def get_build_system(template, url, guesser):
     build system it uses. Otherwise, use a generic template by default.
 
     Args:
-        template (str): ``--template`` argument given to ``spack create``
-        url (str): ``url`` argument given to ``spack create``
-        args (argparse.Namespace): The arguments given to ``spack create``
-        guesser (BuildSystemGuesser): The first_stage_function given to
-            ``spack checksum`` which records the build system it detects
+        template: ``--template`` argument given to ``spack create``
+        url: ``url`` argument given to ``spack create``
+        guesser: The first_stage_function given to ``spack checksum`` which records the build
+            system it detects
 
     Returns:
         str: The name of the build system template to use
@@ -960,7 +1050,7 @@ def create(parser, args):
     build_system = get_build_system(args.template, url, guesser)
 
     # Create the package template object
-    constr_args = {"name": name, "versions": versions}
+    constr_args = {"name": name, "versions": versions, "languages": guesser.languages}
     package_class = templates[build_system]
     if package_class != BundlePackageTemplate:
         constr_args["url"] = url

lib/spack/spack/cmd/debug.py

@@ -6,6 +6,7 @@
 import os
 import platform
 import re
+import sys
 from datetime import datetime
 from glob import glob
 
@@ -62,9 +63,10 @@ def create_db_tarball(args):
 
     base = os.path.basename(str(spack.store.STORE.root))
     transform_args = []
+    # Currently --transform and -s are not supported by Windows native tar
     if "GNU" in tar("--version", output=str):
         transform_args = ["--transform", "s/^%s/%s/" % (base, tarball_name)]
-    else:
+    elif sys.platform != "win32":
         transform_args = ["-s", "/^%s/%s/" % (base, tarball_name)]
 
     wd = os.path.dirname(str(spack.store.STORE.root))
@@ -90,7 +92,6 @@ def report(args):
     print("* **Spack:**", get_version())
     print("* **Python:**", platform.python_version())
     print("* **Platform:**", architecture)
-    print("* **Concretizer:**", spack.config.get("config:concretizer"))
 
 
 def debug(parser, args):

lib/spack/spack/cmd/dependents.py

@@ -47,16 +47,6 @@ def inverted_dependencies():
     dependents of, e.g., `mpi`, but virtuals are not included as
     actual dependents.
     """
-    dag = {}
-    for pkg_cls in spack.repo.PATH.all_package_classes():
-        dag.setdefault(pkg_cls.name, set())
-        for dep in pkg_cls.dependencies_by_name():
-            deps = [dep]
-
-            # expand virtuals if necessary
-            if spack.repo.PATH.is_virtual(dep):
-                deps += [s.name for s in spack.repo.PATH.providers_for(dep)]
-
     dag = collections.defaultdict(set)
     for pkg_cls in spack.repo.PATH.all_package_classes():
         for _, deps_by_name in pkg_cls.dependencies.items():

lib/spack/spack/cmd/deprecate.py

@@ -14,7 +14,6 @@
 installation and its deprecator.
 """
 import argparse
-import os
 
 import llnl.util.tty as tty
 from llnl.util.symlink import symlink
@@ -76,12 +75,7 @@ def setup_parser(sp):
     )
 
     sp.add_argument(
-        "-l",
-        "--link-type",
-        type=str,
-        default="soft",
-        choices=["soft", "hard"],
-        help="type of filesystem link to use for deprecation (default soft)",
+        "-l", "--link-type", type=str, default=None, choices=["soft", "hard"], help="(deprecated)"
     )
 
     sp.add_argument(
@@ -91,6 +85,9 @@ def setup_parser(sp):
 
 def deprecate(parser, args):
     """Deprecate one spec in favor of another"""
+    if args.link_type is not None:
+        tty.warn("The --link-type option is deprecated and will be removed in a future release.")
+
     env = ev.active_environment()
     specs = spack.cmd.parse_specs(args.specs)
 
@@ -144,7 +141,5 @@ def deprecate(parser, args):
         if not answer:
             tty.die("Will not deprecate any packages.")
 
-    link_fn = os.link if args.link_type == "hard" else symlink
-
     for dcate, dcator in zip(all_deprecate, all_deprecators):
-        dcate.package.do_deprecate(dcator, link_fn)
+        dcate.package.do_deprecate(dcator, symlink)

lib/spack/spack/cmd/develop.py

@@ -10,8 +10,10 @@
 import spack.cmd
 import spack.config
 import spack.fetch_strategy
+import spack.package_base
 import spack.repo
 import spack.spec
+import spack.stage
 import spack.util.path
 import spack.version
 from spack.cmd.common import arguments
@@ -62,7 +64,7 @@ def change_fn(section):
     spack.config.change_or_add("develop", find_fn, change_fn)
 
 
-def _retrieve_develop_source(spec, abspath):
+def _retrieve_develop_source(spec: spack.spec.Spec, abspath: str) -> None:
     # "steal" the source code via staging API. We ask for a stage
     # to be created, then copy it afterwards somewhere else. It would be
     # better if we can create the `source_path` directly into its final
@@ -71,13 +73,13 @@ def _retrieve_develop_source(spec, abspath):
     # We construct a package class ourselves, rather than asking for
     # Spec.package, since Spec only allows this when it is concrete
     package = pkg_cls(spec)
-    source_stage = package.stage[0]
+    source_stage: spack.stage.Stage = package.stage[0]
     if isinstance(source_stage.fetcher, spack.fetch_strategy.GitFetchStrategy):
         source_stage.fetcher.get_full_repo = True
         # If we retrieved this version before and cached it, we may have
         # done so without cloning the full git repo; likewise, any
         # mirror might store an instance with truncated history.
-        source_stage.disable_mirrors()
+        source_stage.default_fetcher_only = True
 
     source_stage.fetcher.set_package(package)
     package.stage.steal_source(abspath)

lib/spack/spack/cmd/env.py

@@ -468,32 +468,30 @@ def env_remove(args):
     This removes an environment managed by Spack. Directory environments
     and manifests embedded in repositories should be removed manually.
     """
-    read_envs = []
+    remove_envs = []
     valid_envs = []
     bad_envs = []
-    invalid_envs = []
 
     for env_name in ev.all_environment_names():
         try:
             env = ev.read(env_name)
-            valid_envs.append(env_name)
+            valid_envs.append(env)
 
             if env_name in args.rm_env:
-                read_envs.append(env)
+                remove_envs.append(env)
         except (spack.config.ConfigFormatError, ev.SpackEnvironmentConfigError):
-            invalid_envs.append(env_name)
-
             if env_name in args.rm_env:
                 bad_envs.append(env_name)
 
-        # Check if env is linked to another before trying to remove
-        for name in valid_envs:
+    # Check if remove_env is included from another env before trying to remove
+    for env in valid_envs:
+        for remove_env in remove_envs:
             # don't check if environment is included to itself
-            if name == env_name:
+            if env.name == remove_env.name:
                 continue
-            environ = ev.Environment(ev.root(name))
-            if ev.root(env_name) in environ.included_concrete_envs:
-                msg = f'Environment "{env_name}" is being used by environment "{name}"'
+
+            if remove_env.path in env.included_concrete_envs:
+                msg = f'Environment "{remove_env.name}" is being used by environment "{env.name}"'
                 if args.force:
                     tty.warn(msg)
                 else:
@@ -506,7 +504,7 @@ def env_remove(args):
         if not answer:
             tty.die("Will not remove any environments")
 
-    for env in read_envs:
+    for env in remove_envs:
         name = env.name
         if env.active:
             tty.die(f"Environment {name} can't be removed while activated.")

lib/spack/spack/cmd/gpg.py

@@ -5,10 +5,12 @@
 
 import argparse
 import os
+import tempfile
 
 import spack.binary_distribution
 import spack.mirror
 import spack.paths
+import spack.stage
 import spack.util.gpg
 import spack.util.url
 from spack.cmd.common import arguments
@@ -115,6 +117,7 @@ def setup_parser(subparser):
         help="URL of the mirror where keys will be published",
     )
     publish.add_argument(
+        "--update-index",
         "--rebuild-index",
         action="store_true",
         default=False,
@@ -220,9 +223,10 @@ def gpg_publish(args):
     elif args.mirror_url:
         mirror = spack.mirror.Mirror(args.mirror_url, args.mirror_url)
 
-    spack.binary_distribution.push_keys(
-        mirror, keys=args.keys, regenerate_index=args.rebuild_index
-    )
+    with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:
+        spack.binary_distribution._url_push_keys(
+            mirror, keys=args.keys, tmpdir=tmpdir, update_index=args.update_index
+        )
 
 
 def gpg(parser, args):

lib/spack/spack/cmd/info.py

@@ -502,7 +502,7 @@ def print_licenses(pkg, args):
 
 def info(parser, args):
     spec = spack.spec.Spec(args.package)
-    pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)
+    pkg_cls = spack.repo.PATH.get_pkg_class(spec.fullname)
     pkg = pkg_cls(spec)
 
     # Output core package information

lib/spack/spack/cmd/pkg.py

@@ -169,7 +169,9 @@ def pkg_hash(args):
 
 def get_grep(required=False):
     """Get a grep command to use with ``spack pkg grep``."""
-    return exe.which(os.environ.get("SPACK_GREP") or "grep", required=required)
+    grep = exe.which(os.environ.get("SPACK_GREP") or "grep", required=required)
+    grep.ignore_quotes = True  # allow `spack pkg grep '"quoted string"'` without warning
+    return grep
 
 
 def pkg_grep(args, unknown_args):

lib/spack/spack/cmd/url.py

@@ -339,7 +339,7 @@ def add(self, pkg_name, fetcher):
     for pkg_cls in spack.repo.PATH.all_package_classes():
         npkgs += 1
 
-        for v in pkg_cls.versions:
+        for v in list(pkg_cls.versions):
             try:
                 pkg = pkg_cls(spack.spec.Spec(pkg_cls.name))
                 fetcher = fs.for_package_version(pkg, v)

lib/spack/spack/cmd/versions.py

@@ -23,11 +23,6 @@ def setup_parser(subparser):
     output.add_argument(
         "-s", "--safe", action="store_true", help="only list safe versions of the package"
     )
-    output.add_argument(
-        "--safe-only",
-        action="store_true",
-        help="[deprecated] only list safe versions of the package",
-    )
     output.add_argument(
         "-r", "--remote", action="store_true", help="only list remote versions of the package"
     )
@@ -47,17 +42,13 @@ def versions(parser, args):
 
     safe_versions = pkg.versions
 
-    if args.safe_only:
-        tty.warn('"--safe-only" is deprecated. Use "--safe" instead.')
-        args.safe = args.safe_only
-
     if not (args.remote or args.new):
         if sys.stdout.isatty():
             tty.msg("Safe versions (already checksummed):")
 
         if not safe_versions:
             if sys.stdout.isatty():
-                tty.warn("Found no versions for {0}".format(pkg.name))
+                tty.warn(f"Found no versions for {pkg.name}")
                 tty.debug("Manually add versions to the package.")
         else:
             colify(sorted(safe_versions, reverse=True), indent=2)
@@ -83,12 +74,12 @@ def versions(parser, args):
     if not remote_versions:
         if sys.stdout.isatty():
             if not fetched_versions:
-                tty.warn("Found no versions for {0}".format(pkg.name))
+                tty.warn(f"Found no versions for {pkg.name}")
                 tty.debug(
                     "Check the list_url and list_depth attributes of "
                     "the package to help Spack find versions."
                 )
             else:
-                tty.warn("Found no unchecksummed versions for {0}".format(pkg.name))
+                tty.warn(f"Found no unchecksummed versions for {pkg.name}")
     else:
         colify(sorted(remote_versions, reverse=True), indent=2)

lib/spack/spack/compiler.py

@@ -29,6 +29,9 @@
 
 __all__ = ["Compiler"]
 
+PATH_INSTANCE_VARS = ["cc", "cxx", "f77", "fc"]
+FLAG_INSTANCE_VARS = ["cflags", "cppflags", "cxxflags", "fflags"]
+
 
 @llnl.util.lang.memoized
 def _get_compiler_version_output(compiler_path, version_arg, ignore_errors=()):
@@ -278,11 +281,6 @@ def debug_flags(self):
     def opt_flags(self):
         return ["-O", "-O0", "-O1", "-O2", "-O3"]
 
-    # Cray PrgEnv name that can be used to load this compiler
-    PrgEnv: Optional[str] = None
-    # Name of module used to switch versions of this compiler
-    PrgEnv_compiler: Optional[str] = None
-
     def __init__(
         self,
         cspec,
@@ -705,6 +703,30 @@ def compiler_environment(self):
             os.environ.clear()
             os.environ.update(backup_env)
 
+    def to_dict(self):
+        flags_dict = {fname: " ".join(fvals) for fname, fvals in self.flags.items()}
+        flags_dict.update(
+            {attr: getattr(self, attr, None) for attr in FLAG_INSTANCE_VARS if hasattr(self, attr)}
+        )
+        result = {
+            "spec": str(self.spec),
+            "paths": {attr: getattr(self, attr, None) for attr in PATH_INSTANCE_VARS},
+            "flags": flags_dict,
+            "operating_system": str(self.operating_system),
+            "target": str(self.target),
+            "modules": self.modules or [],
+            "environment": self.environment or {},
+            "extra_rpaths": self.extra_rpaths or [],
+        }
+
+        if self.enable_implicit_rpaths is not None:
+            result["implicit_rpaths"] = self.enable_implicit_rpaths
+
+        if self.alias:
+            result["alias"] = self.alias
+
+        return result
+
 
 class CompilerAccessError(spack.error.SpackError):
     def __init__(self, compiler, paths):

lib/spack/spack/compilers/__init__.py

@@ -6,12 +6,11 @@
 """This module contains functions related to finding compilers on the
 system and configuring Spack to use multiple compilers.
 """
-import collections
-import itertools
-import multiprocessing.pool
+import importlib
 import os
+import sys
 import warnings
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional
 
 import archspec.cpu
 
@@ -22,16 +21,15 @@
 import spack.compiler
 import spack.config
 import spack.error
-import spack.operating_systems
 import spack.paths
 import spack.platforms
+import spack.repo
 import spack.spec
 import spack.version
+from spack.operating_systems import windows_os
 from spack.util.environment import get_path
 from spack.util.naming import mod_to_class
 
-_path_instance_vars = ["cc", "cxx", "f77", "fc"]
-_flags_instance_vars = ["cflags", "cppflags", "cxxflags", "fflags"]
 _other_instance_vars = [
     "modules",
     "operating_system",
@@ -63,6 +61,10 @@
 }
 
 
+#: Tag used to identify packages providing a compiler
+COMPILER_TAG = "compiler"
+
+
 def pkg_spec_for_compiler(cspec):
     """Return the spec of the package that provides the compiler."""
     for spec, package in _compiler_to_pkg.items():
@@ -85,29 +87,7 @@ def converter(cspec_like, *args, **kwargs):
 
 def _to_dict(compiler):
     """Return a dict version of compiler suitable to insert in YAML."""
-    d = {}
-    d["spec"] = str(compiler.spec)
-    d["paths"] = dict((attr, getattr(compiler, attr, None)) for attr in _path_instance_vars)
-    d["flags"] = dict((fname, " ".join(fvals)) for fname, fvals in compiler.flags.items())
-    d["flags"].update(
-        dict(
-            (attr, getattr(compiler, attr, None))
-            for attr in _flags_instance_vars
-            if hasattr(compiler, attr)
-        )
-    )
-    d["operating_system"] = str(compiler.operating_system)
-    d["target"] = str(compiler.target)
-    d["modules"] = compiler.modules or []
-    d["environment"] = compiler.environment or {}
-    d["extra_rpaths"] = compiler.extra_rpaths or []
-    if compiler.enable_implicit_rpaths is not None:
-        d["implicit_rpaths"] = compiler.enable_implicit_rpaths
-
-    if compiler.alias:
-        d["alias"] = compiler.alias
-
-    return {"compiler": d}
+    return {"compiler": compiler.to_dict()}
 
 
 def get_compiler_config(
@@ -127,7 +107,7 @@ def get_compiler_config(
         # Do not init config because there is a non-empty scope
         return config
 
-    _init_compiler_config(configuration, scope=scope)
+    find_compilers(scope=scope)
     config = configuration.get("compilers", scope=scope)
     return config
 
@@ -136,125 +116,8 @@ def get_compiler_config_from_packages(
     configuration: "spack.config.Configuration", *, scope: Optional[str] = None
 ) -> List[Dict]:
     """Return the compiler configuration from packages.yaml"""
-    config = configuration.get("packages", scope=scope)
-    if not config:
-        return []
-
-    packages = []
-    compiler_package_names = supported_compilers() + list(package_name_to_compiler_name.keys())
-    for name, entry in config.items():
-        if name not in compiler_package_names:
-            continue
-        externals_config = entry.get("externals", None)
-        if not externals_config:
-            continue
-        packages.extend(_compiler_config_from_package_config(externals_config))
-
-    return packages
-
-
-def _compiler_config_from_package_config(config):
-    compilers = []
-    for entry in config:
-        compiler = _compiler_config_from_external(entry)
-        if compiler:
-            compilers.append(compiler)
-
-    return compilers
-
-
-def _compiler_config_from_external(config):
-    extra_attributes_key = "extra_attributes"
-    compilers_key = "compilers"
-    c_key, cxx_key, fortran_key = "c", "cxx", "fortran"
-
-    # Allow `@x.y.z` instead of `@=x.y.z`
-    spec = spack.spec.parse_with_version_concrete(config["spec"])
-
-    compiler_spec = spack.spec.CompilerSpec(
-        package_name_to_compiler_name.get(spec.name, spec.name), spec.version
-    )
-
-    err_header = f"The external spec '{spec}' cannot be used as a compiler"
-
-    # If extra_attributes is not there I might not want to use this entry as a compiler,
-    # therefore just leave a debug message, but don't be loud with a warning.
-    if extra_attributes_key not in config:
-        tty.debug(f"[{__file__}] {err_header}: missing the '{extra_attributes_key}' key")
-        return None
-    extra_attributes = config[extra_attributes_key]
-
-    # If I have 'extra_attributes' warn if 'compilers' is missing, or we don't have a C compiler
-    if compilers_key not in extra_attributes:
-        warnings.warn(
-            f"{err_header}: missing the '{compilers_key}' key under '{extra_attributes_key}'"
-        )
-        return None
-    attribute_compilers = extra_attributes[compilers_key]
-
-    if c_key not in attribute_compilers:
-        warnings.warn(
-            f"{err_header}: missing the C compiler path under "
-            f"'{extra_attributes_key}:{compilers_key}'"
-        )
-        return None
-    c_compiler = attribute_compilers[c_key]
-
-    # C++ and Fortran compilers are not mandatory, so let's just leave a debug trace
-    if cxx_key not in attribute_compilers:
-        tty.debug(f"[{__file__}] The external spec {spec} does not have a C++ compiler")
-
-    if fortran_key not in attribute_compilers:
-        tty.debug(f"[{__file__}] The external spec {spec} does not have a Fortran compiler")
-
-    # compilers format has cc/fc/f77, externals format has "c/fortran"
-    paths = {
-        "cc": c_compiler,
-        "cxx": attribute_compilers.get(cxx_key, None),
-        "fc": attribute_compilers.get(fortran_key, None),
-        "f77": attribute_compilers.get(fortran_key, None),
-    }
-
-    if not spec.architecture:
-        host_platform = spack.platforms.host()
-        operating_system = host_platform.operating_system("default_os")
-        target = host_platform.target("default_target").microarchitecture
-    else:
-        target = spec.architecture.target
-        if not target:
-            target = spack.platforms.host().target("default_target")
-        target = target.microarchitecture
-
-        operating_system = spec.os
-        if not operating_system:
-            host_platform = spack.platforms.host()
-            operating_system = host_platform.operating_system("default_os")
-
-    compiler_entry = {
-        "compiler": {
-            "spec": str(compiler_spec),
-            "paths": paths,
-            "flags": extra_attributes.get("flags", {}),
-            "operating_system": str(operating_system),
-            "target": str(target.family),
-            "modules": config.get("modules", []),
-            "environment": extra_attributes.get("environment", {}),
-            "extra_rpaths": extra_attributes.get("extra_rpaths", []),
-            "implicit_rpaths": extra_attributes.get("implicit_rpaths", None),
-        }
-    }
-    return compiler_entry
-
-
-def _init_compiler_config(
-    configuration: "spack.config.Configuration", *, scope: Optional[str]
-) -> None:
-    """Compiler search used when Spack has no compilers."""
-    compilers = find_compilers()
-    compilers_dict = []
-    for compiler in compilers:
-        compilers_dict.append(_to_dict(compiler))
-    configuration.set("compilers", compilers_dict, scope=scope)
+    packages_yaml = configuration.get("packages", scope=scope)
+    return CompilerConfigFactory.from_packages_yaml(packages_yaml)
 
 
 def compiler_config_files():
@@ -278,9 +141,7 @@ def add_compilers_to_config(compilers, scope=None):
         compilers: a list of Compiler objects.
         scope: configuration scope to modify.
     """
-    compiler_config = get_compiler_config(
-        configuration=spack.config.CONFIG, scope=scope, init_config=False
-    )
+    compiler_config = get_compiler_config(configuration=spack.config.CONFIG, scope=scope)
     for compiler in compilers:
         if not compiler.cc:
             tty.debug(f"{compiler.spec} does not have a C compiler")
@@ -329,9 +190,7 @@ def _remove_compiler_from_scope(compiler_spec, scope):
          True if one or more compiler entries were actually removed, False otherwise
     """
     assert scope is not None, "a specific scope is needed when calling this function"
-    compiler_config = get_compiler_config(
-        configuration=spack.config.CONFIG, scope=scope, init_config=False
-    )
+    compiler_config = get_compiler_config(configuration=spack.config.CONFIG, scope=scope)
     filtered_compiler_config = [
         compiler_entry
         for compiler_entry in compiler_config
@@ -380,79 +239,77 @@ def all_compiler_specs(scope=None, init_config=True):
 
 
 def find_compilers(
-    path_hints: Optional[List[str]] = None, *, mixed_toolchain=False
+    path_hints: Optional[List[str]] = None,
+    *,
+    scope: Optional[str] = None,
+    mixed_toolchain: bool = False,
+    max_workers: Optional[int] = None,
 ) -> List["spack.compiler.Compiler"]:
-    """Return the list of compilers found in the paths given as arguments.
+    """Searches for compiler in the paths given as argument. If any new compiler is found, the
+    configuration is updated, and the list of new compiler objects is returned.
 
     Args:
         path_hints: list of path hints where to look for. A sensible default based on the ``PATH``
             environment variable will be used if the value is None
+        scope: configuration scope to modify
         mixed_toolchain: allow mixing compilers from different toolchains if otherwise missing for
             a certain language
+        max_workers: number of processes used to search for compilers
     """
+    import spack.detection
+
+    known_compilers = set(all_compilers(init_config=False))
+
     if path_hints is None:
         path_hints = get_path("PATH")
     default_paths = fs.search_paths_for_executables(*path_hints)
+    if sys.platform == "win32":
+        default_paths.extend(windows_os.WindowsOs().compiler_search_paths)
+    compiler_pkgs = spack.repo.PATH.packages_with_tags(COMPILER_TAG, full=True)
 
-    # To detect the version of the compilers, we dispatch a certain number
-    # of function calls to different workers. Here we construct the list
-    # of arguments for each call.
-    arguments = []
-    for o in all_os_classes():
-        search_paths = getattr(o, "compiler_search_paths", default_paths)
-        arguments.extend(arguments_to_detect_version_fn(o, search_paths))
-
-    # Here we map the function arguments to the corresponding calls
-    tp = multiprocessing.pool.ThreadPool()
-    try:
-        detected_versions = tp.map(detect_version, arguments)
-    finally:
-        tp.close()
-
-    def valid_version(item: Tuple[Optional[DetectVersionArgs], Optional[str]]) -> bool:
-        value, error = item
-        if error is None:
-            return True
-        try:
-            # This will fail on Python 2.6 if a non ascii
-            # character is in the error
-            tty.debug(error)
-        except UnicodeEncodeError:
-            pass
-        return False
-
-    def remove_errors(
-        item: Tuple[Optional[DetectVersionArgs], Optional[str]]
-    ) -> DetectVersionArgs:
-        value, _ = item
-        assert value is not None
-        return value
-
-    return make_compiler_list(
-        [remove_errors(detected) for detected in detected_versions if valid_version(detected)],
-        mixed_toolchain=mixed_toolchain,
+    detected_packages = spack.detection.by_path(
+        compiler_pkgs, path_hints=default_paths, max_workers=max_workers
     )
 
+    valid_compilers = {}
+    for name, detected in detected_packages.items():
+        compilers = [x for x in detected if CompilerConfigFactory.from_external_spec(x)]
+        if not compilers:
+            continue
+        valid_compilers[name] = compilers
 
-def find_new_compilers(
-    path_hints: Optional[List[str]] = None,
-    scope: Optional[str] = None,
-    *,
-    mixed_toolchain: bool = False,
-):
-    """Same as ``find_compilers`` but return only the compilers that are not
-    already in compilers.yaml.
+    def _has_fortran_compilers(x):
+        if "compilers" not in x.extra_attributes:
+            return False
 
-    Args:
-        path_hints: list of path hints where to look for. A sensible default based on the ``PATH``
-            environment variable will be used if the value is None
-        scope: scope to look for a compiler. If None consider the merged configuration.
-        mixed_toolchain: allow mixing compilers from different toolchains if otherwise missing for
-            a certain language
-    """
-    compilers = find_compilers(path_hints, mixed_toolchain=mixed_toolchain)
+        return "fortran" in x.extra_attributes["compilers"]
 
-    return select_new_compilers(compilers, scope)
+    if mixed_toolchain:
+        gccs = [x for x in valid_compilers.get("gcc", []) if _has_fortran_compilers(x)]
+        if gccs:
+            best_gcc = sorted(
+                gccs, key=lambda x: spack.spec.parse_with_version_concrete(x).version
+            )[-1]
+            gfortran = best_gcc.extra_attributes["compilers"]["fortran"]
+            for name in ("llvm", "apple-clang"):
+                if name not in valid_compilers:
+                    continue
+                candidates = valid_compilers[name]
+                for candidate in candidates:
+                    if _has_fortran_compilers(candidate):
+                        continue
+                    candidate.extra_attributes["compilers"]["fortran"] = gfortran
+
+    new_compilers = []
+    for name, detected in valid_compilers.items():
+        for config in CompilerConfigFactory.from_specs(detected):
+            c = _compiler_from_config_entry(config["compiler"])
+            if c in known_compilers:
+                continue
+            new_compilers.append(c)
+
+    add_compilers_to_config(new_compilers, scope=scope)
+    return new_compilers
 
 
 def select_new_compilers(compilers, scope=None):
@@ -462,7 +319,9 @@ def select_new_compilers(compilers, scope=None):
     compilers_not_in_config = []
     for c in compilers:
         arch_spec = spack.spec.ArchSpec((None, c.operating_system, c.target))
-        same_specs = compilers_for_spec(c.spec, arch_spec, scope=scope, init_config=False)
+        same_specs = compilers_for_spec(
+            c.spec, arch_spec=arch_spec, scope=scope, init_config=False
+        )
         if not same_specs:
             compilers_not_in_config.append(c)
 
@@ -510,8 +369,9 @@ def replace_apple_clang(name):
     return [replace_apple_clang(name) for name in all_compiler_module_names()]
 
 
+@llnl.util.lang.memoized
 def all_compiler_module_names() -> List[str]:
-    return [name for name in llnl.util.lang.list_modules(spack.paths.compilers_path)]
+    return list(llnl.util.lang.list_modules(spack.paths.compilers_path))
 
 
 @_auto_compiler_spec
@@ -531,7 +391,12 @@ def find(compiler_spec, scope=None, init_config=True):
 def find_specs_by_arch(compiler_spec, arch_spec, scope=None, init_config=True):
     """Return specs of available compilers that match the supplied
     compiler spec.  Return an empty list if nothing found."""
-    return [c.spec for c in compilers_for_spec(compiler_spec, arch_spec, scope, True, init_config)]
+    return [
+        c.spec
+        for c in compilers_for_spec(
+            compiler_spec, arch_spec=arch_spec, scope=scope, init_config=init_config
+        )
+    ]
 
 
 def all_compilers(scope=None, init_config=True):
@@ -553,14 +418,11 @@ def all_compilers_from(configuration, scope=None, init_config=True):
 
 
 @_auto_compiler_spec
-def compilers_for_spec(
-    compiler_spec, arch_spec=None, scope=None, use_cache=True, init_config=True
-):
+def compilers_for_spec(compiler_spec, *, arch_spec=None, scope=None, init_config=True):
     """This gets all compilers that satisfy the supplied CompilerSpec.
     Returns an empty list if none are found.
     """
     config = all_compilers_config(spack.config.CONFIG, scope=scope, init_config=init_config)
-
     matches = set(find(compiler_spec, scope, init_config))
     compilers = []
     for cspec in matches:
@@ -569,7 +431,7 @@ def compilers_for_spec(
 
 
 def compilers_for_arch(arch_spec, scope=None):
-    config = all_compilers_config(spack.config.CONFIG, scope=scope)
+    config = all_compilers_config(spack.config.CONFIG, scope=scope, init_config=False)
     return list(get_compilers(config, arch_spec=arch_spec))
 
 
@@ -601,13 +463,15 @@ def compiler_from_dict(items):
     os = items.get("operating_system", None)
     target = items.get("target", None)
 
-    if not ("paths" in items and all(n in items["paths"] for n in _path_instance_vars)):
+    if not (
+        "paths" in items and all(n in items["paths"] for n in spack.compiler.PATH_INSTANCE_VARS)
+    ):
         raise InvalidCompilerConfigurationError(cspec)
 
     cls = class_for_compiler_name(cspec.name)
 
     compiler_paths = []
-    for c in _path_instance_vars:
+    for c in spack.compiler.PATH_INSTANCE_VARS:
         compiler_path = items["paths"][c]
         if compiler_path != "None":
             compiler_paths.append(compiler_path)
@@ -735,24 +599,6 @@ def compiler_for_spec(compiler_spec, arch_spec):
     return compilers[0]
 
 
-@_auto_compiler_spec
-def get_compiler_duplicates(compiler_spec, arch_spec):
-    config = spack.config.CONFIG
-
-    scope_to_compilers = {}
-    for scope in config.scopes:
-        compilers = compilers_for_spec(compiler_spec, arch_spec=arch_spec, scope=scope)
-        if compilers:
-            scope_to_compilers[scope] = compilers
-
-    cfg_file_to_duplicates = {}
-    for scope, compilers in scope_to_compilers.items():
-        config_file = config.get_config_filename(scope, "compilers")
-        cfg_file_to_duplicates[config_file] = compilers
-
-    return cfg_file_to_duplicates
-
-
 @llnl.util.lang.memoized
 def class_for_compiler_name(compiler_name):
     """Given a compiler module name, get the corresponding Compiler class."""
@@ -766,7 +612,7 @@ def class_for_compiler_name(compiler_name):
         submodule_name = compiler_name.replace("-", "_")
 
     module_name = ".".join(["spack", "compilers", submodule_name])
-    module_obj = __import__(module_name, fromlist=[None])
+    module_obj = importlib.import_module(module_name)
     cls = getattr(module_obj, mod_to_class(compiler_name))
 
     # make a note of the name in the module so we can get to it easily.
@@ -775,272 +621,10 @@ def class_for_compiler_name(compiler_name):
     return cls
 
 
-def all_os_classes():
-    """
-    Return the list of classes for all operating systems available on
-    this platform
-    """
-    classes = []
-
-    platform = spack.platforms.host()
-    for os_class in platform.operating_sys.values():
-        classes.append(os_class)
-
-    return classes
-
-
 def all_compiler_types():
     return [class_for_compiler_name(c) for c in supported_compilers()]
 
 
-#: Gathers the attribute values by which a detected compiler is considered
-#: unique in Spack.
-#:
-#:  - os: the operating system
-#:  - compiler_name: the name of the compiler (e.g. 'gcc', 'clang', etc.)
-#:  - version: the version of the compiler
-#:
-CompilerID = collections.namedtuple("CompilerID", ["os", "compiler_name", "version"])
-
-#: Variations on a matched compiler name
-NameVariation = collections.namedtuple("NameVariation", ["prefix", "suffix"])
-
-#: Groups together the arguments needed by `detect_version`. The four entries
-#: in the tuple are:
-#:
-#: - id: An instance of the CompilerID named tuple (version can be set to None
-#:       as it will be detected later)
-#: - variation: a NameVariation for file being tested
-#: - language: compiler language being tested (one of 'cc', 'cxx', 'fc', 'f77')
-#: - path: full path to the executable being tested
-#:
-DetectVersionArgs = collections.namedtuple(
-    "DetectVersionArgs", ["id", "variation", "language", "path"]
-)
-
-
-def arguments_to_detect_version_fn(
-    operating_system: spack.operating_systems.OperatingSystem, paths: List[str]
-) -> List[DetectVersionArgs]:
-    """Returns a list of DetectVersionArgs tuples to be used in a
-    corresponding function to detect compiler versions.
-
-    The ``operating_system`` instance can customize the behavior of this
-    function by providing a method called with the same name.
-
-    Args:
-        operating_system: the operating system on which we are looking for compilers
-        paths: paths to search for compilers
-
-    Returns:
-        List of DetectVersionArgs tuples. Each item in the list will be later
-        mapped to the corresponding function call to detect the version of the
-        compilers in this OS.
-    """
-
-    def _default(search_paths: List[str]) -> List[DetectVersionArgs]:
-        command_arguments: List[DetectVersionArgs] = []
-        files_to_be_tested = fs.files_in(*search_paths)
-        for compiler_name in supported_compilers_for_host_platform():
-            compiler_cls = class_for_compiler_name(compiler_name)
-
-            for language in ("cc", "cxx", "f77", "fc"):
-                # Select only the files matching a regexp
-                for (file, full_path), regexp in itertools.product(
-                    files_to_be_tested, compiler_cls.search_regexps(language)
-                ):
-                    match = regexp.match(file)
-                    if match:
-                        compiler_id = CompilerID(operating_system, compiler_name, None)
-                        detect_version_args = DetectVersionArgs(
-                            id=compiler_id,
-                            variation=NameVariation(*match.groups()),
-                            language=language,
-                            path=full_path,
-                        )
-                        command_arguments.append(detect_version_args)
-
-        return command_arguments
-
-    fn = getattr(operating_system, "arguments_to_detect_version_fn", _default)
-    return fn(paths)
-
-
-def detect_version(
-    detect_version_args: DetectVersionArgs,
-) -> Tuple[Optional[DetectVersionArgs], Optional[str]]:
-    """Computes the version of a compiler and adds it to the information
-    passed as input.
-
-    As this function is meant to be executed by worker processes it won't
-    raise any exception but instead will return a (value, error) tuple that
-    needs to be checked by the code dispatching the calls.
-
-    Args:
-        detect_version_args: information on the compiler for which we should detect the version.
-
-    Returns:
-        A ``(DetectVersionArgs, error)`` tuple. If ``error`` is ``None`` the
-        version of the compiler was computed correctly and the first argument
-        of the tuple will contain it. Otherwise ``error`` is a string
-        containing an explanation on why the version couldn't be computed.
-    """
-
-    def _default(fn_args):
-        compiler_id = fn_args.id
-        language = fn_args.language
-        compiler_cls = class_for_compiler_name(compiler_id.compiler_name)
-        path = fn_args.path
-
-        # Get compiler names and the callback to detect their versions
-        callback = getattr(compiler_cls, f"{language}_version")
-
-        try:
-            version = callback(path)
-            if version and str(version).strip() and version != "unknown":
-                value = fn_args._replace(id=compiler_id._replace(version=version))
-                return value, None
-
-            error = f"Couldn't get version for compiler {path}".format(path)
-        except spack.util.executable.ProcessError as e:
-            error = f"Couldn't get version for compiler {path}\n" + str(e)
-        except spack.util.executable.ProcessTimeoutError as e:
-            error = f"Couldn't get version for compiler {path}\n" + str(e)
-        except Exception as e:
-            # Catching "Exception" here is fine because it just
-            # means something went wrong running a candidate executable.
-            error = "Error while executing candidate compiler {0}" "\n{1}: {2}".format(
-                path, e.__class__.__name__, str(e)
-            )
-        return None, error
-
-    operating_system = detect_version_args.id.os
-    fn = getattr(operating_system, "detect_version", _default)
-    return fn(detect_version_args)
-
-
-def make_compiler_list(
-    detected_versions: List[DetectVersionArgs], mixed_toolchain: bool = False
-) -> List["spack.compiler.Compiler"]:
-    """Process a list of detected versions and turn them into a list of
-    compiler specs.
-
-    Args:
-        detected_versions: list of DetectVersionArgs containing a valid version
-        mixed_toolchain: allow mixing compilers from different toolchains if langauge is missing
-
-    Returns:
-        list: list of Compiler objects
-    """
-    group_fn = lambda x: (x.id, x.variation, x.language)
-    sorted_compilers = sorted(detected_versions, key=group_fn)
-
-    # Gather items in a dictionary by the id, name variation and language
-    compilers_d: Dict[CompilerID, Dict[NameVariation, dict]] = {}
-    for sort_key, group in itertools.groupby(sorted_compilers, key=group_fn):
-        compiler_id, name_variation, language = sort_key
-        by_compiler_id = compilers_d.setdefault(compiler_id, {})
-        by_name_variation = by_compiler_id.setdefault(name_variation, {})
-        by_name_variation[language] = next(x.path for x in group)
-
-    def _default_make_compilers(cmp_id, paths):
-        operating_system, compiler_name, version = cmp_id
-        compiler_cls = class_for_compiler_name(compiler_name)
-        spec = spack.spec.CompilerSpec(compiler_cls.name, f"={version}")
-        paths = [paths.get(x, None) for x in ("cc", "cxx", "f77", "fc")]
-        # TODO: johnwparent - revist the following line as per discussion at:
-        # https://github.com/spack/spack/pull/33385/files#r1040036318
-        target = archspec.cpu.host()
-        compiler = compiler_cls(spec, operating_system, str(target.family), paths)
-        return [compiler]
-
-    # For compilers with the same compiler id:
-    #
-    # - Prefer with C compiler to without
-    # - Prefer with C++ compiler to without
-    # - Prefer no variations to variations (e.g., clang to clang-gpu)
-    #
-    sort_fn = lambda variation: (
-        "cc" not in by_compiler_id[variation],  # None last
-        "cxx" not in by_compiler_id[variation],  # None last
-        getattr(variation, "prefix", None),
-        getattr(variation, "suffix", None),
-    )
-
-    # Flatten to a list of compiler id, primary variation and compiler dictionary
-    flat_compilers: List[Tuple[CompilerID, NameVariation, dict]] = []
-    for compiler_id, by_compiler_id in compilers_d.items():
-        ordered = sorted(by_compiler_id, key=sort_fn)
-        selected_variation = ordered[0]
-        selected = by_compiler_id[selected_variation]
-
-        # Fill any missing parts from subsequent entries (without mixing toolchains)
-        for lang in ["cxx", "f77", "fc"]:
-            if lang not in selected:
-                next_lang = next(
-                    (by_compiler_id[v][lang] for v in ordered if lang in by_compiler_id[v]), None
-                )
-                if next_lang:
-                    selected[lang] = next_lang
-
-        flat_compilers.append((compiler_id, selected_variation, selected))
-
-    # Next, fill out the blanks of missing compilers by creating a mixed toolchain (if requested)
-    if mixed_toolchain:
-        make_mixed_toolchain(flat_compilers)
-
-    # Finally, create the compiler list
-    compilers: List["spack.compiler.Compiler"] = []
-    for compiler_id, _, compiler in flat_compilers:
-        make_compilers = getattr(compiler_id.os, "make_compilers", _default_make_compilers)
-        candidates = make_compilers(compiler_id, compiler)
-        compilers.extend(x for x in candidates if x.cc is not None)
-
-    return compilers
-
-
-def make_mixed_toolchain(compilers: List[Tuple[CompilerID, NameVariation, dict]]) -> None:
-    """Add missing compilers across toolchains when they are missing for a particular language.
-    This currently only adds the most sensible gfortran to (apple)-clang if it doesn't have a
-    fortran compiler (no flang)."""
-
-    # First collect the clangs that are missing a fortran compiler
-    clangs_without_flang = [
-        (id, variation, compiler)
-        for id, variation, compiler in compilers
-        if id.compiler_name in ("clang", "apple-clang")
-        and "f77" not in compiler
-        and "fc" not in compiler
-    ]
-    if not clangs_without_flang:
-        return
-
-    # Filter on GCCs with fortran compiler
-    gccs_with_fortran = [
-        (id, variation, compiler)
-        for id, variation, compiler in compilers
-        if id.compiler_name == "gcc" and "f77" in compiler and "fc" in compiler
-    ]
-
-    # Sort these GCCs by "best variation" (no prefix / suffix first)
-    gccs_with_fortran.sort(
-        key=lambda x: (getattr(x[1], "prefix", None), getattr(x[1], "suffix", None))
-    )
-
-    # Attach the optimal GCC fortran compiler to the clangs that don't have one
-    for clang_id, _, clang_compiler in clangs_without_flang:
-        gcc_compiler = next(
-            (gcc[2] for gcc in gccs_with_fortran if gcc[0].os == clang_id.os), None
-        )
-
-        if not gcc_compiler:
-            continue
-
-        # Update the fc / f77 entries
-        clang_compiler["f77"] = gcc_compiler["f77"]
-        clang_compiler["fc"] = gcc_compiler["fc"]
-
-
 def is_mixed_toolchain(compiler):
     """Returns True if the current compiler is a mixed toolchain,
     False otherwise.
@@ -1087,20 +671,164 @@ def name_matches(name, name_list):
     return False
 
 
+_EXTRA_ATTRIBUTES_KEY = "extra_attributes"
+_COMPILERS_KEY = "compilers"
+_C_KEY = "c"
+_CXX_KEY, _FORTRAN_KEY = "cxx", "fortran"
+
+
+class CompilerConfigFactory:
+    """Class aggregating all ways of constructing a list of compiler config entries."""
+
+    @staticmethod
+    def from_specs(specs: List["spack.spec.Spec"]) -> List[dict]:
+        result = []
+        compiler_package_names = supported_compilers() + list(package_name_to_compiler_name.keys())
+        for s in specs:
+            if s.name not in compiler_package_names:
+                continue
+
+            candidate = CompilerConfigFactory.from_external_spec(s)
+            if candidate is None:
+                continue
+
+            result.append(candidate)
+        return result
+
+    @staticmethod
+    def from_packages_yaml(packages_yaml) -> List[dict]:
+        compiler_specs = []
+        compiler_package_names = supported_compilers() + list(package_name_to_compiler_name.keys())
+        for name, entry in packages_yaml.items():
+            if name not in compiler_package_names:
+                continue
+
+            externals_config = entry.get("externals", None)
+            if not externals_config:
+                continue
+
+            current_specs = []
+            for current_external in externals_config:
+                compiler = CompilerConfigFactory._spec_from_external_config(current_external)
+                if compiler:
+                    current_specs.append(compiler)
+            compiler_specs.extend(current_specs)
+
+        return CompilerConfigFactory.from_specs(compiler_specs)
+
+    @staticmethod
+    def _spec_from_external_config(config):
+        # Allow `@x.y.z` instead of `@=x.y.z`
+        err_header = f"The external spec '{config['spec']}' cannot be used as a compiler"
+        # If extra_attributes is not there I might not want to use this entry as a compiler,
+        # therefore just leave a debug message, but don't be loud with a warning.
+        if _EXTRA_ATTRIBUTES_KEY not in config:
+            tty.debug(f"[{__file__}] {err_header}: missing the '{_EXTRA_ATTRIBUTES_KEY}' key")
+            return None
+        extra_attributes = config[_EXTRA_ATTRIBUTES_KEY]
+        result = spack.spec.Spec(
+            str(spack.spec.parse_with_version_concrete(config["spec"])),
+            external_modules=config.get("modules"),
+        )
+        result.extra_attributes = extra_attributes
+        return result
+
+    @staticmethod
+    def from_external_spec(spec: "spack.spec.Spec") -> Optional[dict]:
+        spec = spack.spec.parse_with_version_concrete(spec)
+        extra_attributes = getattr(spec, _EXTRA_ATTRIBUTES_KEY, None)
+        if extra_attributes is None:
+            return None
+
+        paths = CompilerConfigFactory._extract_compiler_paths(spec)
+        if paths is None:
+            return None
+
+        compiler_spec = spack.spec.CompilerSpec(
+            package_name_to_compiler_name.get(spec.name, spec.name), spec.version
+        )
+
+        operating_system, target = CompilerConfigFactory._extract_os_and_target(spec)
+
+        compiler_entry = {
+            "compiler": {
+                "spec": str(compiler_spec),
+                "paths": paths,
+                "flags": extra_attributes.get("flags", {}),
+                "operating_system": str(operating_system),
+                "target": str(target.family),
+                "modules": getattr(spec, "external_modules", []),
+                "environment": extra_attributes.get("environment", {}),
+                "extra_rpaths": extra_attributes.get("extra_rpaths", []),
+                "implicit_rpaths": extra_attributes.get("implicit_rpaths", None),
+            }
+        }
+        return compiler_entry
+
+    @staticmethod
+    def _extract_compiler_paths(spec: "spack.spec.Spec") -> Optional[Dict[str, str]]:
+        err_header = f"The external spec '{spec}' cannot be used as a compiler"
+        extra_attributes = spec.extra_attributes
+        # If I have 'extra_attributes' warn if 'compilers' is missing,
+        # or we don't have a C compiler
+        if _COMPILERS_KEY not in extra_attributes:
+            warnings.warn(
+                f"{err_header}: missing the '{_COMPILERS_KEY}' key under '{_EXTRA_ATTRIBUTES_KEY}'"
+            )
+            return None
+        attribute_compilers = extra_attributes[_COMPILERS_KEY]
+
+        if _C_KEY not in attribute_compilers:
+            warnings.warn(
+                f"{err_header}: missing the C compiler path under "
+                f"'{_EXTRA_ATTRIBUTES_KEY}:{_COMPILERS_KEY}'"
+            )
+            return None
+        c_compiler = attribute_compilers[_C_KEY]
+
+        # C++ and Fortran compilers are not mandatory, so let's just leave a debug trace
+        if _CXX_KEY not in attribute_compilers:
+            tty.debug(f"[{__file__}] The external spec {spec} does not have a C++ compiler")
+
+        if _FORTRAN_KEY not in attribute_compilers:
+            tty.debug(f"[{__file__}] The external spec {spec} does not have a Fortran compiler")
+
+        # compilers format has cc/fc/f77, externals format has "c/fortran"
+        return {
+            "cc": c_compiler,
+            "cxx": attribute_compilers.get(_CXX_KEY, None),
+            "fc": attribute_compilers.get(_FORTRAN_KEY, None),
+            "f77": attribute_compilers.get(_FORTRAN_KEY, None),
+        }
+
+    @staticmethod
+    def _extract_os_and_target(spec: "spack.spec.Spec"):
+        if not spec.architecture:
+            host_platform = spack.platforms.host()
+            operating_system = host_platform.operating_system("default_os")
+            target = host_platform.target("default_target").microarchitecture
+        else:
+            target = spec.architecture.target
+            if not target:
+                target = spack.platforms.host().target("default_target")
+            target = target.microarchitecture
+
+            operating_system = spec.os
+            if not operating_system:
+                host_platform = spack.platforms.host()
+                operating_system = host_platform.operating_system("default_os")
+        return operating_system, target
+
+
 class InvalidCompilerConfigurationError(spack.error.SpackError):
     def __init__(self, compiler_spec):
         super().__init__(
-            'Invalid configuration for [compiler "%s"]: ' % compiler_spec,
-            "Compiler configuration must contain entries for all compilers: %s"
-            % _path_instance_vars,
+            f'Invalid configuration for [compiler "{compiler_spec}"]: ',
+            f"Compiler configuration must contain entries for "
+            f"all compilers: {spack.compiler.PATH_INSTANCE_VARS}",
         )
 
 
-class NoCompilersError(spack.error.SpackError):
-    def __init__(self):
-        super().__init__("Spack could not find any compilers!")
-
-
 class UnknownCompilerError(spack.error.SpackError):
     def __init__(self, compiler_name):
         super().__init__("Spack doesn't support the requested compiler: {0}".format(compiler_name))
@@ -1111,25 +839,3 @@ def __init__(self, compiler_spec, target):
         super().__init__(
             "No compilers for operating system %s satisfy spec %s" % (target, compiler_spec)
         )
-
-
-class CompilerDuplicateError(spack.error.SpackError):
-    def __init__(self, compiler_spec, arch_spec):
-        config_file_to_duplicates = get_compiler_duplicates(compiler_spec, arch_spec)
-        duplicate_table = list((x, len(y)) for x, y in config_file_to_duplicates.items())
-        descriptor = lambda num: "time" if num == 1 else "times"
-        duplicate_msg = lambda cfgfile, count: "{0}: {1} {2}".format(
-            cfgfile, str(count), descriptor(count)
-        )
-        msg = (
-            "Compiler configuration contains entries with duplicate"
-            + " specification ({0}, {1})".format(compiler_spec, arch_spec)
-            + " in the following files:\n\t"
-            + "\n\t".join(duplicate_msg(x, y) for x, y in duplicate_table)
-        )
-        super().__init__(msg)
-
-
-class CompilerSpecInsufficientlySpecificError(spack.error.SpackError):
-    def __init__(self, compiler_spec):
-        super().__init__("Multiple compilers satisfy spec %s" % compiler_spec)

lib/spack/spack/compilers/aocc.py

@@ -25,9 +25,6 @@ class Aocc(Compiler):
     # Subclasses use possible names of Fortran 90 compiler
     fc_names = ["flang"]
 
-    PrgEnv = "PrgEnv-aocc"
-    PrgEnv_compiler = "aocc"
-
     version_argument = "--version"
 
     @property

lib/spack/spack/compilers/cce.py

@@ -34,12 +34,9 @@ def __init__(self, *args, **kwargs):
     # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
     suffixes = [r"-mp-\d\.\d"]
 
-    PrgEnv = "PrgEnv-cray"
-    PrgEnv_compiler = "cce"
-
     @property
     def link_paths(self):
-        if any(self.PrgEnv in m for m in self.modules):
+        if any("PrgEnv-cray" in m for m in self.modules):
             # Old module-based interface to cray compilers
             return {
                 "cc": os.path.join("cce", "cc"),

lib/spack/spack/compilers/gcc.py

@@ -40,9 +40,6 @@ class Gcc(spack.compiler.Compiler):
         "fc": os.path.join("gcc", "gfortran"),
     }
 
-    PrgEnv = "PrgEnv-gnu"
-    PrgEnv_compiler = "gcc"
-
     @property
     def verbose_flag(self):
         return "-v"

lib/spack/spack/compilers/intel.py

@@ -31,9 +31,6 @@ class Intel(Compiler):
         "fc": os.path.join("intel", "ifort"),
     }
 
-    PrgEnv = "PrgEnv-intel"
-    PrgEnv_compiler = "intel"
-
     if sys.platform == "win32":
         version_argument = "/QV"
     else:
@@ -126,3 +123,14 @@ def fc_pic_flag(self):
     @property
     def stdcxx_libs(self):
         return ("-cxxlib",)
+
+    def setup_custom_environment(self, pkg, env):
+        # Edge cases for Intel's oneAPI compilers when using the legacy classic compilers:
+        # Always pass flags to disable deprecation warnings, since these warnings can
+        # confuse tools that parse the output of compiler commands (e.g. version checks).
+        if self.cc and self.cc.endswith("icc") and self.real_version >= Version("2021"):
+            env.append_flags("SPACK_ALWAYS_CFLAGS", "-diag-disable=10441")
+        if self.cxx and self.cxx.endswith("icpc") and self.real_version >= Version("2021"):
+            env.append_flags("SPACK_ALWAYS_CXXFLAGS", "-diag-disable=10441")
+        if self.fc and self.fc.endswith("ifort") and self.real_version >= Version("2021"):
+            env.append_flags("SPACK_ALWAYS_FFLAGS", "-diag-disable=10448")

lib/spack/spack/compilers/msvc.py

@@ -223,6 +223,30 @@ def get_oneapi_root(pth: str):
             )
         self.msvc_compiler_environment = CmdCall(*env_cmds)
 
+    @property
+    def cxx11_flag(self):
+        return "/std:c++11"
+
+    @property
+    def cxx14_flag(self):
+        return "/std:c++14"
+
+    @property
+    def cxx17_flag(self):
+        return "/std:c++17"
+
+    @property
+    def cxx20_flag(self):
+        return "/std:c++20"
+
+    @property
+    def c11_flag(self):
+        return "/std:c11"
+
+    @property
+    def c17_flag(self):
+        return "/std:c17"
+
     @property
     def msvc_version(self):
         """This is the VCToolset version *NOT* the actual version of the cl compiler
@@ -231,24 +255,55 @@ def msvc_version(self):
 
     @property
     def short_msvc_version(self):
+        """This is the shorthand VCToolset version of form
+        MSVC<short-ver>
         """
-        This is the shorthand VCToolset version of form
-        MSVC<short-ver> *NOT* the full version, for that see
+        return "MSVC" + self.vc_toolset_ver
+
+    @property
+    def vc_toolset_ver(self):
+        """
+        The toolset version is the version of the combined set of cl and link
+        This typically relates directly to VS version i.e. VS 2022 is v143
+        VS 19 is v142, etc.
+        This value is defined by the first three digits of the major + minor
+        version of the VS toolset (143 for 14.3x.bbbbb). Traditionally the
+        minor version has remained a static two digit number for a VS release
+        series, however, as of VS22, this is no longer true, both
+        14.4x.bbbbb and 14.3x.bbbbb are considered valid VS22 VC toolset
+        versions due to a change in toolset minor version sentiment.
+
+        This is *NOT* the full version, for that see
         Msvc.msvc_version or MSVC.platform_toolset_ver for the
         raw platform toolset version
+
         """
-        ver = self.platform_toolset_ver
-        return "MSVC" + ver
+        ver = self.msvc_version[:2].joined.string[:3]
+        return ver
 
     @property
     def platform_toolset_ver(self):
         """
         This is the platform toolset version of current MSVC compiler
-        i.e. 142.
+        i.e. 142. The platform toolset is the targeted MSVC library/compiler
+        versions by compilation (this is different from the VC Toolset)
+
+
         This is different from the VC toolset version as established
-        by `short_msvc_version`
+        by `short_msvc_version`, but typically are represented by the same
+        three digit value
         """
-        return self.msvc_version[:2].joined.string[:3]
+        # Typically VS toolset version and platform toolset versions match
+        # VS22 introduces the first divergence of VS toolset version
+        # (144 for "recent" releases) and platform toolset version (143)
+        # so it needs additional handling until MS releases v144
+        # (assuming v144 is also for VS22)
+        # or adds better support for detection
+        # TODO: (johnwparent) Update this logic for the next platform toolset
+        # or VC toolset version update
+        toolset_ver = self.vc_toolset_ver
+        vs22_toolset = Version(toolset_ver) > Version("142")
+        return toolset_ver if not vs22_toolset else "143"
 
     def _compiler_version(self, compiler):
         """Returns version object for given compiler"""

lib/spack/spack/compilers/nvhpc.py

@@ -29,9 +29,6 @@ class Nvhpc(Compiler):
         "fc": os.path.join("nvhpc", "nvfortran"),
     }
 
-    PrgEnv = "PrgEnv-nvhpc"
-    PrgEnv_compiler = "nvhpc"
-
     version_argument = "--version"
     version_regex = r"nv[^ ]* (?:[^ ]+ Dev-r)?([0-9.]+)(?:-[0-9]+)?"
 

lib/spack/spack/compilers/oneapi.py

@@ -9,6 +9,7 @@
 from llnl.util import tty
 
 from spack.compiler import Compiler
+from spack.version import Version
 
 
 class Oneapi(Compiler):
@@ -32,9 +33,6 @@ class Oneapi(Compiler):
         "fc": os.path.join("oneapi", "ifx"),
     }
 
-    PrgEnv = "PrgEnv-oneapi"
-    PrgEnv_compiler = "oneapi"
-
     version_argument = "--version"
     version_regex = r"(?:(?:oneAPI DPC\+\+(?:\/C\+\+)? Compiler)|(?:\(IFORT\))|(?:\(IFX\))) (\S+)"
 
@@ -142,6 +140,16 @@ def setup_custom_environment(self, pkg, env):
             env.prepend_path("PATH", dirname(self.cxx))
             env.prepend_path("LD_LIBRARY_PATH", join(dirname(dirname(self.cxx)), "lib"))
 
+        # Edge cases for Intel's oneAPI compilers when using the legacy classic compilers:
+        # Always pass flags to disable deprecation warnings, since these warnings can
+        # confuse tools that parse the output of compiler commands (e.g. version checks).
+        if self.cc and self.cc.endswith("icc") and self.real_version >= Version("2021"):
+            env.append_flags("SPACK_ALWAYS_CFLAGS", "-diag-disable=10441")
+        if self.cxx and self.cxx.endswith("icpc") and self.real_version >= Version("2021"):
+            env.append_flags("SPACK_ALWAYS_CXXFLAGS", "-diag-disable=10441")
+        if self.fc and self.fc.endswith("ifort") and self.real_version >= Version("2021"):
+            env.append_flags("SPACK_ALWAYS_FFLAGS", "-diag-disable=10448")
+
         # 2024 release bumped the libsycl version because of an ABI
         # change, 2024 compilers are required.  You will see this
         # error:

lib/spack/spack/compilers/pgi.py

@@ -30,9 +30,6 @@ class Pgi(Compiler):
         "fc": os.path.join("pgi", "pgfortran"),
     }
 
-    PrgEnv = "PrgEnv-pgi"
-    PrgEnv_compiler = "pgi"
-
     version_argument = "-V"
     ignore_version_errors = [2]  # `pgcc -V` on PowerPC annoyingly returns 2
     version_regex = r"pg[^ ]* ([0-9.]+)-[0-9]+ (LLVM )?[^ ]+ target on "

lib/spack/spack/compilers/rocmcc.py

@@ -23,9 +23,6 @@ class Rocmcc(spack.compilers.clang.Clang):
     # Subclasses use possible names of Fortran 90 compiler
     fc_names = ["amdflang"]
 
-    PrgEnv = "PrgEnv-amd"
-    PrgEnv_compiler = "amd"
-
     @property
     def link_paths(self):
         link_paths = {

lib/spack/spack/concretize.py

@@ -2,31 +2,12 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
 """
-Functions here are used to take abstract specs and make them concrete.
-For example, if a spec asks for a version between 1.8 and 1.9, these
-functions might take will take the most recent 1.9 version of the
-package available.  Or, if the user didn't specify a compiler for a
-spec, then this will assign a compiler to the spec based on defaults
-or user preferences.
-
-TODO: make this customizable and allow users to configure
-      concretization  policies.
+(DEPRECATED) Used to contain the code for the original concretizer
 """
-import functools
-import platform
-import tempfile
 from contextlib import contextmanager
 from itertools import chain
-from typing import Union
 
-import archspec.cpu
-
-import llnl.util.lang
-import llnl.util.tty as tty
-
-import spack.abi
 import spack.compilers
 import spack.config
 import spack.environment
@@ -37,639 +18,20 @@
 import spack.target
 import spack.tengine
 import spack.util.path
-import spack.variant as vt
-from spack.package_prefs import PackagePrefs, is_spec_buildable, spec_externals
-from spack.version import ClosedOpenRange, VersionList, ver
-
-#: impements rudimentary logic for ABI compatibility
-_abi: Union[spack.abi.ABI, llnl.util.lang.Singleton] = llnl.util.lang.Singleton(
-    lambda: spack.abi.ABI()
-)
-
-
-@functools.total_ordering
-class reverse_order:
-    """Helper for creating key functions.
-
-    This is a wrapper that inverts the sense of the natural
-    comparisons on the object.
-    """
-
-    def __init__(self, value):
-        self.value = value
-
-    def __eq__(self, other):
-        return other.value == self.value
-
-    def __lt__(self, other):
-        return other.value < self.value
 
 
 class Concretizer:
-    """You can subclass this class to override some of the default
-    concretization strategies, or you can override all of them.
-    """
+    """(DEPRECATED) Only contains logic to enable/disable compiler existence checks."""
 
     #: Controls whether we check that compiler versions actually exist
     #: during concretization. Used for testing and for mirror creation
     check_for_compiler_existence = None
 
-    #: Packages that the old concretizer cannot deal with correctly, and cannot build anyway.
-    #: Those will not be considered as providers for virtuals.
-    non_buildable_packages = {"glibc", "musl"}
-
-    def __init__(self, abstract_spec=None):
+    def __init__(self):
         if Concretizer.check_for_compiler_existence is None:
             Concretizer.check_for_compiler_existence = not spack.config.get(
                 "config:install_missing_compilers", False
             )
-        self.abstract_spec = abstract_spec
-        self._adjust_target_answer_generator = None
-
-    def concretize_develop(self, spec):
-        """
-        Add ``dev_path=*`` variant to packages built from local source.
-        """
-        env = spack.environment.active_environment()
-        dev_info = env.dev_specs.get(spec.name, {}) if env else {}
-        if not dev_info:
-            return False
-
-        path = spack.util.path.canonicalize_path(dev_info["path"], default_wd=env.path)
-
-        if "dev_path" in spec.variants:
-            assert spec.variants["dev_path"].value == path
-            changed = False
-        else:
-            spec.variants.setdefault("dev_path", vt.SingleValuedVariant("dev_path", path))
-            changed = True
-        changed |= spec.constrain(dev_info["spec"])
-        return changed
-
-    def _valid_virtuals_and_externals(self, spec):
-        """Returns a list of candidate virtual dep providers and external
-        packages that coiuld be used to concretize a spec.
-
-        Preferred specs come first in the list.
-        """
-        # First construct a list of concrete candidates to replace spec with.
-        candidates = [spec]
-        pref_key = lambda spec: 0  # no-op pref key
-
-        if spec.virtual:
-            candidates = [
-                s
-                for s in spack.repo.PATH.providers_for(spec)
-                if s.name not in self.non_buildable_packages
-            ]
-            if not candidates:
-                raise spack.error.UnsatisfiableProviderSpecError(candidates[0], spec)
-
-            # Find nearest spec in the DAG (up then down) that has prefs.
-            spec_w_prefs = find_spec(
-                spec, lambda p: PackagePrefs.has_preferred_providers(p.name, spec.name), spec
-            )  # default to spec itself.
-
-            # Create a key to sort candidates by the prefs we found
-            pref_key = PackagePrefs(spec_w_prefs.name, "providers", spec.name)
-
-        # For each candidate package, if it has externals, add those
-        # to the usable list.  if it's not buildable, then *only* add
-        # the externals.
-        usable = []
-        for cspec in candidates:
-            if is_spec_buildable(cspec):
-                usable.append(cspec)
-
-            externals = spec_externals(cspec)
-            for ext in externals:
-                if ext.intersects(spec):
-                    usable.append(ext)
-
-        # If nothing is in the usable list now, it's because we aren't
-        # allowed to build anything.
-        if not usable:
-            raise NoBuildError(spec)
-
-        # Use a sort key to order the results
-        return sorted(
-            usable,
-            key=lambda spec: (
-                not spec.external,  # prefer externals
-                pref_key(spec),  # respect prefs
-                spec.name,  # group by name
-                reverse_order(spec.versions),  # latest version
-                spec,  # natural order
-            ),
-        )
-
-    def choose_virtual_or_external(self, spec: spack.spec.Spec):
-        """Given a list of candidate virtual and external packages, try to
-        find one that is most ABI compatible.
-        """
-        candidates = self._valid_virtuals_and_externals(spec)
-        if not candidates:
-            return candidates
-
-        # Find the nearest spec in the dag that has a compiler.  We'll
-        # use that spec to calibrate compiler compatibility.
-        abi_exemplar = find_spec(spec, lambda x: x.compiler)
-        if abi_exemplar is None:
-            abi_exemplar = spec.root
-
-        # Sort candidates from most to least compatibility.
-        #   We reverse because True > False.
-        #   Sort is stable, so candidates keep their order.
-        return sorted(
-            candidates,
-            reverse=True,
-            key=lambda spec: (
-                _abi.compatible(spec, abi_exemplar, loose=True),
-                _abi.compatible(spec, abi_exemplar),
-            ),
-        )
-
-    def concretize_version(self, spec):
-        """If the spec is already concrete, return.  Otherwise take
-        the preferred version from spackconfig, and default to the package's
-        version if there are no available versions.
-
-        TODO: In many cases we probably want to look for installed
-              versions of each package and use an installed version
-              if we can link to it.  The policy implemented here will
-              tend to rebuild a lot of stuff becasue it will prefer
-              a compiler in the spec to any compiler already-
-              installed things were built with.  There is likely
-              some better policy that finds some middle ground
-              between these two extremes.
-        """
-        # return if already concrete.
-        if spec.versions.concrete:
-            return False
-
-        # List of versions we could consider, in sorted order
-        pkg_versions = spec.package_class.versions
-        usable = [v for v in pkg_versions if any(v.intersects(sv) for sv in spec.versions)]
-
-        yaml_prefs = PackagePrefs(spec.name, "version")
-
-        # The keys below show the order of precedence of factors used
-        # to select a version when concretizing.  The item with
-        # the "largest" key will be selected.
-        #
-        # NOTE: When COMPARING VERSIONS, the '@develop' version is always
-        #       larger than other versions.  BUT when CONCRETIZING,
-        #       the largest NON-develop version is selected by default.
-        keyfn = lambda v: (
-            # ------- Special direction from the user
-            # Respect order listed in packages.yaml
-            -yaml_prefs(v),
-            # The preferred=True flag (packages or packages.yaml or both?)
-            pkg_versions.get(v).get("preferred", False),
-            # ------- Regular case: use latest non-develop version by default.
-            # Avoid @develop version, which would otherwise be the "largest"
-            # in straight version comparisons
-            not v.isdevelop(),
-            # Compare the version itself
-            # This includes the logic:
-            #    a) develop > everything (disabled by "not v.isdevelop() above)
-            #    b) numeric > non-numeric
-            #    c) Numeric or string comparison
-            v,
-        )
-        usable.sort(key=keyfn, reverse=True)
-
-        if usable:
-            spec.versions = ver([usable[0]])
-        else:
-            # We don't know of any SAFE versions that match the given
-            # spec.  Grab the spec's versions and grab the highest
-            # *non-open* part of the range of versions it specifies.
-            # Someone else can raise an error if this happens,
-            # e.g. when we go to fetch it and don't know how.  But it
-            # *might* work.
-            if not spec.versions or spec.versions == VersionList([":"]):
-                raise NoValidVersionError(spec)
-            else:
-                last = spec.versions[-1]
-                if isinstance(last, ClosedOpenRange):
-                    range_as_version = VersionList([last]).concrete_range_as_version
-                    if range_as_version:
-                        spec.versions = ver([range_as_version])
-                    else:
-                        raise NoValidVersionError(spec)
-                else:
-                    spec.versions = ver([last])
-
-        return True  # Things changed
-
-    def concretize_architecture(self, spec):
-        """If the spec is empty provide the defaults of the platform. If the
-        architecture is not a string type, then check if either the platform,
-        target or operating system are concretized. If any of the fields are
-        changed then return True. If everything is concretized (i.e the
-        architecture attribute is a namedtuple of classes) then return False.
-        If the target is a string type, then convert the string into a
-        concretized architecture. If it has no architecture and the root of the
-        DAG has an architecture, then use the root otherwise use the defaults
-        on the platform.
-        """
-        # ensure type safety for the architecture
-        if spec.architecture is None:
-            spec.architecture = spack.spec.ArchSpec()
-
-        if spec.architecture.concrete:
-            return False
-
-        # Get platform of nearest spec with a platform, including spec
-        # If spec has a platform, easy
-        if spec.architecture.platform:
-            new_plat = spack.platforms.by_name(spec.architecture.platform)
-        else:
-            # Else if anyone else has a platform, take the closest one
-            # Search up, then down, along build/link deps first
-            # Then any nearest. Algorithm from compilerspec search
-            platform_spec = find_spec(spec, lambda x: x.architecture and x.architecture.platform)
-            if platform_spec:
-                new_plat = spack.platforms.by_name(platform_spec.architecture.platform)
-            else:
-                # If no platform anywhere in this spec, grab the default
-                new_plat = spack.platforms.host()
-
-        # Get nearest spec with relevant platform and an os
-        # Generally, same algorithm as finding platform, except we only
-        # consider specs that have a platform
-        if spec.architecture.os:
-            new_os = spec.architecture.os
-        else:
-            new_os_spec = find_spec(
-                spec,
-                lambda x: (
-                    x.architecture
-                    and x.architecture.platform == str(new_plat)
-                    and x.architecture.os
-                ),
-            )
-            if new_os_spec:
-                new_os = new_os_spec.architecture.os
-            else:
-                new_os = new_plat.operating_system("default_os")
-
-        # Get the nearest spec with relevant platform and a target
-        # Generally, same algorithm as finding os
-        curr_target = None
-        if spec.architecture.target:
-            curr_target = spec.architecture.target
-        if spec.architecture.target and spec.architecture.target_concrete:
-            new_target = spec.architecture.target
-        else:
-            new_target_spec = find_spec(
-                spec,
-                lambda x: (
-                    x.architecture
-                    and x.architecture.platform == str(new_plat)
-                    and x.architecture.target
-                    and x.architecture.target != curr_target
-                ),
-            )
-            if new_target_spec:
-                if curr_target:
-                    # constrain one target by the other
-                    new_target_arch = spack.spec.ArchSpec(
-                        (None, None, new_target_spec.architecture.target)
-                    )
-                    curr_target_arch = spack.spec.ArchSpec((None, None, curr_target))
-                    curr_target_arch.constrain(new_target_arch)
-                    new_target = curr_target_arch.target
-                else:
-                    new_target = new_target_spec.architecture.target
-            else:
-                # To get default platform, consider package prefs
-                if PackagePrefs.has_preferred_targets(spec.name):
-                    new_target = self.target_from_package_preferences(spec)
-                else:
-                    new_target = new_plat.target("default_target")
-                if curr_target:
-                    # convert to ArchSpec to compare satisfaction
-                    new_target_arch = spack.spec.ArchSpec((None, None, str(new_target)))
-                    curr_target_arch = spack.spec.ArchSpec((None, None, str(curr_target)))
-
-                    if not new_target_arch.intersects(curr_target_arch):
-                        # new_target is an incorrect guess based on preferences
-                        # and/or default
-                        valid_target_ranges = str(curr_target).split(",")
-                        for target_range in valid_target_ranges:
-                            t_min, t_sep, t_max = target_range.partition(":")
-                            if not t_sep:
-                                new_target = t_min
-                                break
-                            elif t_max:
-                                new_target = t_max
-                                break
-                            elif t_min:
-                                # TODO: something better than picking first
-                                new_target = t_min
-                                break
-
-        # Construct new architecture, compute whether spec changed
-        arch_spec = (str(new_plat), str(new_os), str(new_target))
-        new_arch = spack.spec.ArchSpec(arch_spec)
-        spec_changed = new_arch != spec.architecture
-        spec.architecture = new_arch
-        return spec_changed
-
-    def target_from_package_preferences(self, spec):
-        """Returns the preferred target from the package preferences if
-        there's any.
-
-        Args:
-            spec: abstract spec to be concretized
-        """
-        target_prefs = PackagePrefs(spec.name, "target")
-        target_specs = [spack.spec.Spec("target=%s" % tname) for tname in archspec.cpu.TARGETS]
-
-        def tspec_filter(s):
-            # Filter target specs by whether the architecture
-            # family is the current machine type. This ensures
-            # we only consider x86_64 targets when on an
-            # x86_64 machine, etc. This may need to change to
-            # enable setting cross compiling as a default
-            target = archspec.cpu.TARGETS[str(s.architecture.target)]
-            arch_family_name = target.family.name
-            return arch_family_name == platform.machine()
-
-        # Sort filtered targets by package prefs
-        target_specs = list(filter(tspec_filter, target_specs))
-        target_specs.sort(key=target_prefs)
-        new_target = target_specs[0].architecture.target
-        return new_target
-
-    def concretize_variants(self, spec):
-        """If the spec already has variants filled in, return.  Otherwise, add
-        the user preferences from packages.yaml or the default variants from
-        the package specification.
-        """
-        changed = False
-        preferred_variants = PackagePrefs.preferred_variants(spec.name)
-        pkg_cls = spec.package_class
-        for name, entry in pkg_cls.variants.items():
-            variant, when = entry
-            var = spec.variants.get(name, None)
-            if var and "*" in var:
-                # remove variant wildcard before concretizing
-                # wildcard cannot be combined with other variables in a
-                # multivalue variant, a concrete variant cannot have the value
-                # wildcard, and a wildcard does not constrain a variant
-                spec.variants.pop(name)
-            if name not in spec.variants and any(spec.satisfies(w) for w in when):
-                changed = True
-                if name in preferred_variants:
-                    spec.variants[name] = preferred_variants.get(name)
-                else:
-                    spec.variants[name] = variant.make_default()
-            if name in spec.variants and not any(spec.satisfies(w) for w in when):
-                raise vt.InvalidVariantForSpecError(name, when, spec)
-
-        return changed
-
-    def concretize_compiler(self, spec):
-        """If the spec already has a compiler, we're done.  If not, then take
-        the compiler used for the nearest ancestor with a compiler
-        spec and use that.  If the ancestor's compiler is not
-        concrete, then used the preferred compiler as specified in
-        spackconfig.
-
-        Intuition: Use the spackconfig default if no package that depends on
-        this one has a strict compiler requirement.  Otherwise, try to
-        build with the compiler that will be used by libraries that
-        link to this one, to maximize compatibility.
-        """
-        # Pass on concretizing the compiler if the target or operating system
-        # is not yet determined
-        if not spec.architecture.concrete:
-            # We haven't changed, but other changes need to happen before we
-            # continue. `return True` here to force concretization to keep
-            # running.
-            return True
-
-        # Only use a matching compiler if it is of the proper style
-        # Takes advantage of the proper logic already existing in
-        # compiler_for_spec Should think whether this can be more
-        # efficient
-        def _proper_compiler_style(cspec, aspec):
-            compilers = spack.compilers.compilers_for_spec(cspec, arch_spec=aspec)
-            # If the spec passed as argument is concrete we want to check
-            # the versions match exactly
-            if (
-                cspec.concrete
-                and compilers
-                and cspec.version not in [c.version for c in compilers]
-            ):
-                return []
-
-            return compilers
-
-        if spec.compiler and spec.compiler.concrete:
-            if self.check_for_compiler_existence and not _proper_compiler_style(
-                spec.compiler, spec.architecture
-            ):
-                _compiler_concretization_failure(spec.compiler, spec.architecture)
-            return False
-
-        # Find another spec that has a compiler, or the root if none do
-        other_spec = spec if spec.compiler else find_spec(spec, lambda x: x.compiler, spec.root)
-        other_compiler = other_spec.compiler
-        assert other_spec
-
-        # Check if the compiler is already fully specified
-        if other_compiler and other_compiler.concrete:
-            if self.check_for_compiler_existence and not _proper_compiler_style(
-                other_compiler, spec.architecture
-            ):
-                _compiler_concretization_failure(other_compiler, spec.architecture)
-            spec.compiler = other_compiler
-            return True
-
-        if other_compiler:  # Another node has abstract compiler information
-            compiler_list = spack.compilers.find_specs_by_arch(other_compiler, spec.architecture)
-            if not compiler_list:
-                # We don't have a matching compiler installed
-                if not self.check_for_compiler_existence:
-                    # Concretize compiler spec versions as a package to build
-                    cpkg_spec = spack.compilers.pkg_spec_for_compiler(other_compiler)
-                    self.concretize_version(cpkg_spec)
-                    spec.compiler = spack.spec.CompilerSpec(
-                        other_compiler.name, cpkg_spec.versions
-                    )
-                    return True
-                else:
-                    # No compiler with a satisfactory spec was found
-                    raise UnavailableCompilerVersionError(other_compiler, spec.architecture)
-        else:
-            # We have no hints to go by, grab any compiler
-            compiler_list = spack.compilers.all_compiler_specs()
-            if not compiler_list:
-                # Spack has no compilers.
-                raise spack.compilers.NoCompilersError()
-
-        # By default, prefer later versions of compilers
-        compiler_list = sorted(compiler_list, key=lambda x: (x.name, x.version), reverse=True)
-        ppk = PackagePrefs(other_spec.name, "compiler")
-        matches = sorted(compiler_list, key=ppk)
-
-        # copy concrete version into other_compiler
-        try:
-            spec.compiler = next(
-                c for c in matches if _proper_compiler_style(c, spec.architecture)
-            ).copy()
-        except StopIteration:
-            # No compiler with a satisfactory spec has a suitable arch
-            _compiler_concretization_failure(other_compiler, spec.architecture)
-
-        assert spec.compiler.concrete
-        return True  # things changed.
-
-    def concretize_compiler_flags(self, spec):
-        """
-        The compiler flags are updated to match those of the spec whose
-        compiler is used, defaulting to no compiler flags in the spec.
-        Default specs set at the compiler level will still be added later.
-        """
-        # Pass on concretizing the compiler flags if the target or operating
-        # system is not set.
-        if not spec.architecture.concrete:
-            # We haven't changed, but other changes need to happen before we
-            # continue. `return True` here to force concretization to keep
-            # running.
-            return True
-
-        compiler_match = lambda other: (
-            spec.compiler == other.compiler and spec.architecture == other.architecture
-        )
-
-        ret = False
-        for flag in spack.spec.FlagMap.valid_compiler_flags():
-            if flag not in spec.compiler_flags:
-                spec.compiler_flags[flag] = list()
-            try:
-                nearest = next(
-                    p
-                    for p in spec.traverse(direction="parents")
-                    if (compiler_match(p) and (p is not spec) and flag in p.compiler_flags)
-                )
-                nearest_flags = nearest.compiler_flags.get(flag, [])
-                flags = spec.compiler_flags.get(flag, [])
-                if set(nearest_flags) - set(flags):
-                    spec.compiler_flags[flag] = list(llnl.util.lang.dedupe(nearest_flags + flags))
-                    ret = True
-            except StopIteration:
-                pass
-
-        # Include the compiler flag defaults from the config files
-        # This ensures that spack will detect conflicts that stem from a change
-        # in default compiler flags.
-        try:
-            compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture)
-        except spack.compilers.NoCompilerForSpecError:
-            if self.check_for_compiler_existence:
-                raise
-            return ret
-        for flag in compiler.flags:
-            config_flags = compiler.flags.get(flag, [])
-            flags = spec.compiler_flags.get(flag, [])
-            spec.compiler_flags[flag] = list(llnl.util.lang.dedupe(config_flags + flags))
-            if set(config_flags) - set(flags):
-                ret = True
-
-        return ret
-
-    def adjust_target(self, spec):
-        """Adjusts the target microarchitecture if the compiler is too old
-        to support the default one.
-
-        Args:
-            spec: spec to be concretized
-
-        Returns:
-            True if spec was modified, False otherwise
-        """
-        # To minimize the impact on performance this function will attempt
-        # to adjust the target only at the very first call once necessary
-        # information is set. It will just return False on subsequent calls.
-        # The way this is achieved is by initializing a generator and making
-        # this function return the next answer.
-        if not (spec.architecture and spec.architecture.concrete):
-            # Not ready, but keep going because we have work to do later
-            return True
-
-        def _make_only_one_call(spec):
-            yield self._adjust_target(spec)
-            while True:
-                yield False
-
-        if self._adjust_target_answer_generator is None:
-            self._adjust_target_answer_generator = _make_only_one_call(spec)
-
-        return next(self._adjust_target_answer_generator)
-
-    def _adjust_target(self, spec):
-        """Assumes that the architecture and the compiler have been
-        set already and checks if the current target microarchitecture
-        is the default and can be optimized by the compiler.
-
-        If not, downgrades the microarchitecture until a suitable one
-        is found. If none can be found raise an error.
-
-        Args:
-            spec: spec to be concretized
-
-        Returns:
-            True if any modification happened, False otherwise
-        """
-        import archspec.cpu
-
-        # Try to adjust the target only if it is the default
-        # target for this platform
-        current_target = spec.architecture.target
-        current_platform = spack.platforms.by_name(spec.architecture.platform)
-
-        default_target = current_platform.target("default_target")
-        if PackagePrefs.has_preferred_targets(spec.name):
-            default_target = self.target_from_package_preferences(spec)
-
-        if current_target != default_target or (
-            self.abstract_spec
-            and self.abstract_spec.architecture
-            and self.abstract_spec.architecture.concrete
-        ):
-            return False
-
-        try:
-            current_target.optimization_flags(spec.compiler)
-        except archspec.cpu.UnsupportedMicroarchitecture:
-            microarchitecture = current_target.microarchitecture
-            for ancestor in microarchitecture.ancestors:
-                candidate = None
-                try:
-                    candidate = spack.target.Target(ancestor)
-                    candidate.optimization_flags(spec.compiler)
-                except archspec.cpu.UnsupportedMicroarchitecture:
-                    continue
-
-                if candidate is not None:
-                    msg = (
-                        "{0.name}@{0.version} cannot build optimized "
-                        'binaries for "{1}". Using best target possible: '
-                        '"{2}"'
-                    )
-                    msg = msg.format(spec.compiler, current_target, candidate)
-                    tty.warn(msg)
-                    spec.architecture.target = candidate
-                    return True
-            else:
-                raise
-
-        return False
 
 
 @contextmanager
@@ -719,19 +81,6 @@ def find_spec(spec, condition, default=None):
     return default  # Nothing matched the condition; return default.
 
 
-def _compiler_concretization_failure(compiler_spec, arch):
-    # Distinguish between the case that there are compilers for
-    # the arch but not with the given compiler spec and the case that
-    # there are no compilers for the arch at all
-    if not spack.compilers.compilers_for_arch(arch):
-        available_os_targets = set(
-            (c.operating_system, c.target) for c in spack.compilers.all_compilers()
-        )
-        raise NoCompilersForArchError(arch, available_os_targets)
-    else:
-        raise UnavailableCompilerVersionError(compiler_spec, arch)
-
-
 def concretize_specs_together(*abstract_specs, **kwargs):
     """Given a number of specs as input, tries to concretize them together.
 
@@ -744,12 +93,6 @@ def concretize_specs_together(*abstract_specs, **kwargs):
     Returns:
         List of concretized specs
     """
-    if spack.config.get("config:concretizer", "clingo") == "original":
-        return _concretize_specs_together_original(*abstract_specs, **kwargs)
-    return _concretize_specs_together_new(*abstract_specs, **kwargs)
-
-
-def _concretize_specs_together_new(*abstract_specs, **kwargs):
     import spack.solver.asp
 
     allow_deprecated = spack.config.get("config:deprecated", False)
@@ -760,51 +103,6 @@ def _concretize_specs_together_new(*abstract_specs, **kwargs):
     return [s.copy() for s in result.specs]
 
 
-def _concretize_specs_together_original(*abstract_specs, **kwargs):
-    abstract_specs = [spack.spec.Spec(s) for s in abstract_specs]
-    tmpdir = tempfile.mkdtemp()
-    builder = spack.repo.MockRepositoryBuilder(tmpdir)
-    # Split recursive specs, as it seems the concretizer has issue
-    # respecting conditions on dependents expressed like
-    # depends_on('foo ^bar@1.0'), see issue #11160
-    split_specs = [
-        dep.copy(deps=False) for spec1 in abstract_specs for dep in spec1.traverse(root=True)
-    ]
-    builder.add_package(
-        "concretizationroot", dependencies=[(str(x), None, None) for x in split_specs]
-    )
-
-    with spack.repo.use_repositories(builder.root, override=False):
-        # Spec from a helper package that depends on all the abstract_specs
-        concretization_root = spack.spec.Spec("concretizationroot")
-        concretization_root.concretize(tests=kwargs.get("tests", False))
-        # Retrieve the direct dependencies
-        concrete_specs = [concretization_root[spec.name].copy() for spec in abstract_specs]
-
-    return concrete_specs
-
-
-class NoCompilersForArchError(spack.error.SpackError):
-    def __init__(self, arch, available_os_targets):
-        err_msg = (
-            "No compilers found"
-            " for operating system %s and target %s."
-            "\nIf previous installations have succeeded, the"
-            " operating system may have been updated." % (arch.os, arch.target)
-        )
-
-        available_os_target_strs = list()
-        for operating_system, t in available_os_targets:
-            os_target_str = "%s-%s" % (operating_system, t) if t else operating_system
-            available_os_target_strs.append(os_target_str)
-        err_msg += (
-            "\nCompilers are defined for the following"
-            " operating systems and targets:\n\t" + "\n\t".join(available_os_target_strs)
-        )
-
-        super().__init__(err_msg, "Run 'spack compiler find' to add compilers.")
-
-
 class UnavailableCompilerVersionError(spack.error.SpackError):
     """Raised when there is no available compiler that satisfies a
     compiler spec."""
@@ -820,37 +118,3 @@ def __init__(self, compiler_spec, arch=None):
             "'spack compilers' to see which compilers are already recognized"
             " by spack.",
         )
-
-
-class NoValidVersionError(spack.error.SpackError):
-    """Raised when there is no way to have a concrete version for a
-    particular spec."""
-
-    def __init__(self, spec):
-        super().__init__(
-            "There are no valid versions for %s that match '%s'" % (spec.name, spec.versions)
-        )
-
-
-class InsufficientArchitectureInfoError(spack.error.SpackError):
-    """Raised when details on architecture cannot be collected from the
-    system"""
-
-    def __init__(self, spec, archs):
-        super().__init__(
-            "Cannot determine necessary architecture information for '%s': %s"
-            % (spec.name, str(archs))
-        )
-
-
-class NoBuildError(spack.error.SpecError):
-    """Raised when a package is configured with the buildable option False, but
-    no satisfactory external versions can be found
-    """
-
-    def __init__(self, spec):
-        msg = (
-            "The spec\n    '%s'\n    is configured as not buildable, "
-            "and no matching external installs were found"
-        )
-        super().__init__(msg % spec)

lib/spack/spack/config.py

@@ -99,7 +99,6 @@
         "dirty": False,
         "build_jobs": min(16, cpus_available()),
         "build_stage": "$tempdir/spack-stage",
-        "concretizer": "clingo",
         "license_dir": spack.paths.default_license_dir,
     }
 }
@@ -1091,7 +1090,7 @@ def validate(
 
 
 def read_config_file(
-    filename: str, schema: Optional[YamlConfigDict] = None
+    path: str, schema: Optional[YamlConfigDict] = None
 ) -> Optional[YamlConfigDict]:
     """Read a YAML configuration file.
 
@@ -1101,21 +1100,9 @@ def read_config_file(
     # to preserve flexibility in calling convention (don't need to provide
     # schema when it's not necessary) while allowing us to validate against a
     # known schema when the top-level key could be incorrect.
-
-    if not os.path.exists(filename):
-        # Ignore nonexistent files.
-        tty.debug(f"Skipping nonexistent config path {filename}", level=3)
-        return None
-
-    elif not os.path.isfile(filename):
-        raise ConfigFileError(f"Invalid configuration. {filename} exists but is not a file.")
-
-    elif not os.access(filename, os.R_OK):
-        raise ConfigFileError(f"Config file is not readable: {filename}")
-
     try:
-        tty.debug(f"Reading config from file {filename}")
-        with open(filename) as f:
+        with open(path) as f:
+            tty.debug(f"Reading config from file {path}")
             data = syaml.load_config(f)
 
         if data:
@@ -1126,15 +1113,20 @@ def read_config_file(
 
         return data
 
-    except StopIteration:
-        raise ConfigFileError(f"Config file is empty or is not a valid YAML dict: {filename}")
+    except FileNotFoundError:
+        # Ignore nonexistent files.
+        tty.debug(f"Skipping nonexistent config path {path}", level=3)
+        return None
+
+    except OSError as e:
+        raise ConfigFileError(f"Path is not a file or is not readable: {path}: {str(e)}") from e
+
+    except StopIteration as e:
+        raise ConfigFileError(f"Config file is empty or is not a valid YAML dict: {path}") from e
 
     except syaml.SpackYAMLError as e:
         raise ConfigFileError(str(e)) from e
 
-    except OSError as e:
-        raise ConfigFileError(f"Error reading configuration file {filename}: {str(e)}") from e
-
 
 def _override(string: str) -> bool:
     """Test if a spack YAML string is an override.

lib/spack/spack/cray_manifest.py

@@ -14,12 +14,14 @@
 import llnl.util.tty as tty
 
 import spack.cmd
+import spack.compilers
 import spack.deptypes as dt
 import spack.error
 import spack.hash_types as hash_types
 import spack.platforms
 import spack.repo
 import spack.spec
+import spack.store
 from spack.schema.cray_manifest import schema as manifest_schema
 
 #: Cray systems can store a Spack-compatible description of system
@@ -237,7 +239,7 @@ def read(path, apply_updates):
                 tty.debug(f"Include this\n{traceback.format_exc()}")
     if apply_updates:
         for spec in specs.values():
-            spack.store.STORE.db.add(spec, directory_layout=None)
+            spack.store.STORE.db.add(spec)
 
 
 class ManifestValidationError(spack.error.SpackError):

lib/spack/spack/database.py

@@ -59,7 +59,11 @@
 import spack.util.lock as lk
 import spack.util.spack_json as sjson
 import spack.version as vn
-from spack.directory_layout import DirectoryLayoutError, InconsistentInstallDirectoryError
+from spack.directory_layout import (
+    DirectoryLayout,
+    DirectoryLayoutError,
+    InconsistentInstallDirectoryError,
+)
 from spack.error import SpackError
 from spack.util.crypto import bit_length
 
@@ -208,7 +212,7 @@ def __init__(
         ref_count: int = 0,
         explicit: bool = False,
         installation_time: Optional[float] = None,
-        deprecated_for: Optional["spack.spec.Spec"] = None,
+        deprecated_for: Optional[str] = None,
         in_buildcache: bool = False,
         origin=None,
     ):
@@ -595,9 +599,11 @@ class Database:
     def __init__(
         self,
         root: str,
+        *,
         upstream_dbs: Optional[List["Database"]] = None,
         is_upstream: bool = False,
         lock_cfg: LockConfiguration = DEFAULT_LOCK_CFG,
+        layout: Optional[DirectoryLayout] = None,
     ) -> None:
         """Database for Spack installations.
 
@@ -620,6 +626,7 @@ def __init__(
         """
         self.root = root
         self.database_directory = os.path.join(self.root, _DB_DIRNAME)
+        self.layout = layout
 
         # Set up layout of database files within the db dir
         self._index_path = os.path.join(self.database_directory, "index.json")
@@ -664,14 +671,6 @@ def __init__(
 
         self.upstream_dbs = list(upstream_dbs) if upstream_dbs else []
 
-        # whether there was an error at the start of a read transaction
-        self._error = None
-
-        # For testing: if this is true, an exception is thrown when missing
-        # dependencies are detected (rather than just printing a warning
-        # message)
-        self._fail_when_missing_deps = False
-
         self._write_transaction_impl = lk.WriteTransaction
         self._read_transaction_impl = lk.ReadTransaction
 
@@ -774,7 +773,13 @@ def query_local_by_spec_hash(self, hash_key):
         with self.read_transaction():
             return self._data.get(hash_key, None)
 
-    def _assign_dependencies(self, spec_reader, hash_key, installs, data):
+    def _assign_dependencies(
+        self,
+        spec_reader: Type["spack.spec.SpecfileReaderBase"],
+        hash_key: str,
+        installs: dict,
+        data: Dict[str, InstallRecord],
+    ):
         # Add dependencies from other records in the install DB to
         # form a full spec.
         spec = data[hash_key].spec
@@ -787,26 +792,20 @@ def _assign_dependencies(self, spec_reader, hash_key, installs, data):
             for dname, dhash, dtypes, _, virtuals in spec_reader.read_specfile_dep_specs(
                 yaml_deps
             ):
-                # It is important that we always check upstream installations
-                # in the same order, and that we always check the local
-                # installation first: if a downstream Spack installs a package
-                # then dependents in that installation could be using it.
-                # If a hash is installed locally and upstream, there isn't
-                # enough information to determine which one a local package
-                # depends on, so the convention ensures that this isn't an
-                # issue.
-                upstream, record = self.query_by_spec_hash(dhash, data=data)
+                # It is important that we always check upstream installations in the same order,
+                # and that we always check the local installation first: if a downstream Spack
+                # installs a package then dependents in that installation could be using it. If a
+                # hash is installed locally and upstream, there isn't enough information to
+                # determine which one a local package depends on, so the convention ensures that
+                # this isn't an issue.
+                _, record = self.query_by_spec_hash(dhash, data=data)
                 child = record.spec if record else None
 
                 if not child:
-                    msg = "Missing dependency not in database: " "%s needs %s-%s" % (
-                        spec.cformat("{name}{/hash:7}"),
-                        dname,
-                        dhash[:7],
+                    tty.warn(
+                        f"Missing dependency not in database: "
+                        f"{spec.cformat('{name}{/hash:7}')} needs {dname}-{dhash[:7]}"
                     )
-                    if self._fail_when_missing_deps:
-                        raise MissingDependenciesError(msg)
-                    tty.warn(msg)
                     continue
 
                 spec._add_dependency(child, depflag=dt.canonicalize(dtypes), virtuals=virtuals)
@@ -873,8 +872,8 @@ def invalid_record(hash_key, error):
         # (i.e., its specs are a true Merkle DAG, unlike most specs.)
 
         # Pass 1: Iterate through database and build specs w/o dependencies
-        data = {}
-        installed_prefixes = set()
+        data: Dict[str, InstallRecord] = {}
+        installed_prefixes: Set[str] = set()
         for hash_key, rec in installs.items():
             try:
                 # This constructs a spec DAG from the list of all installs
@@ -911,7 +910,7 @@ def invalid_record(hash_key, error):
         self._data = data
         self._installed_prefixes = installed_prefixes
 
-    def reindex(self, directory_layout):
+    def reindex(self):
         """Build database index from scratch based on a directory layout.
 
         Locks the DB if it isn't locked already.
@@ -919,6 +918,8 @@ def reindex(self, directory_layout):
         if self.is_upstream:
             raise UpstreamDatabaseLockingError("Cannot reindex an upstream database")
 
+        error: Optional[CorruptDatabaseError] = None
+
         # Special transaction to avoid recursive reindex calls and to
         # ignore errors if we need to rebuild a corrupt database.
         def _read_suppress_error():
@@ -926,7 +927,8 @@ def _read_suppress_error():
                 if os.path.isfile(self._index_path):
                     self._read_from_file(self._index_path)
             except CorruptDatabaseError as e:
-                self._error = e
+                nonlocal error
+                error = e
                 self._data = {}
                 self._installed_prefixes = set()
 
@@ -935,14 +937,13 @@ def _read_suppress_error():
         )
 
         with transaction:
-            if self._error:
-                tty.warn("Spack database was corrupt. Will rebuild. Error was:", str(self._error))
-                self._error = None
+            if error is not None:
+                tty.warn(f"Spack database was corrupt. Will rebuild. Error was: {error}")
 
             old_data = self._data
             old_installed_prefixes = self._installed_prefixes
             try:
-                self._construct_from_directory_layout(directory_layout, old_data)
+                self._construct_from_directory_layout(old_data)
             except BaseException:
                 # If anything explodes, restore old data, skip write.
                 self._data = old_data
@@ -950,13 +951,16 @@ def _read_suppress_error():
                 raise
 
     def _construct_entry_from_directory_layout(
-        self, directory_layout, old_data, spec, deprecator=None
+        self,
+        old_data: Dict[str, InstallRecord],
+        spec: "spack.spec.Spec",
+        deprecator: Optional["spack.spec.Spec"] = None,
     ):
         # Try to recover explicit value from old DB, but
         # default it to True if DB was corrupt. This is
         # just to be conservative in case a command like
         # "autoremove" is run by the user after a reindex.
-        tty.debug("RECONSTRUCTING FROM SPEC.YAML: {0}".format(spec))
+        tty.debug(f"Reconstructing from spec file: {spec}")
         explicit = True
         inst_time = os.stat(spec.prefix).st_ctime
         if old_data is not None:
@@ -965,18 +969,17 @@ def _construct_entry_from_directory_layout(
                 explicit = old_info.explicit
                 inst_time = old_info.installation_time
 
-        extra_args = {"explicit": explicit, "installation_time": inst_time}
-        self._add(spec, directory_layout, **extra_args)
+        self._add(spec, explicit=explicit, installation_time=inst_time)
         if deprecator:
             self._deprecate(spec, deprecator)
 
-    def _construct_from_directory_layout(self, directory_layout, old_data):
-        # Read first the `spec.yaml` files in the prefixes. They should be
-        # considered authoritative with respect to DB reindexing, as
-        # entries in the DB may be corrupted in a way that still makes
-        # them readable. If we considered DB entries authoritative
-        # instead, we would perpetuate errors over a reindex.
-        with directory_layout.disable_upstream_check():
+    def _construct_from_directory_layout(self, old_data: Dict[str, InstallRecord]):
+        # Read first the spec files in the prefixes. They should be considered authoritative with
+        # respect to DB reindexing, as entries in the DB may be corrupted in a way that still makes
+        # them readable. If we considered DB entries authoritative instead, we would perpetuate
+        # errors over a reindex.
+        assert self.layout is not None, "Cannot reindex a database without a known layout"
+        with self.layout.disable_upstream_check():
             # Initialize data in the reconstructed DB
             self._data = {}
             self._installed_prefixes = set()
@@ -984,44 +987,36 @@ def _construct_from_directory_layout(self, directory_layout, old_data):
             # Start inspecting the installed prefixes
             processed_specs = set()
 
-            for spec in directory_layout.all_specs():
-                self._construct_entry_from_directory_layout(directory_layout, old_data, spec)
+            for spec in self.layout.all_specs():
+                self._construct_entry_from_directory_layout(old_data, spec)
                 processed_specs.add(spec)
 
-            for spec, deprecator in directory_layout.all_deprecated_specs():
-                self._construct_entry_from_directory_layout(
-                    directory_layout, old_data, spec, deprecator
-                )
+            for spec, deprecator in self.layout.all_deprecated_specs():
+                self._construct_entry_from_directory_layout(old_data, spec, deprecator)
                 processed_specs.add(spec)
 
-            for key, entry in old_data.items():
-                # We already took care of this spec using
-                # `spec.yaml` from its prefix.
+            for entry in old_data.values():
+                # We already took care of this spec using spec file from its prefix.
                 if entry.spec in processed_specs:
-                    msg = "SKIPPING RECONSTRUCTION FROM OLD DB: {0}"
-                    msg += " [already reconstructed from spec.yaml]"
-                    tty.debug(msg.format(entry.spec))
+                    tty.debug(
+                        f"Skipping reconstruction from old db: {entry.spec}"
+                        " [already reconstructed from spec file]"
+                    )
                     continue
 
-                # If we arrived here it very likely means that
-                # we have external specs that are not dependencies
-                # of other specs. This may be the case for externally
-                # installed compilers or externally installed
-                # applications.
-                tty.debug("RECONSTRUCTING FROM OLD DB: {0}".format(entry.spec))
+                # If we arrived here it very likely means that we have external specs that are not
+                # dependencies of other specs. This may be the case for externally installed
+                # compilers or externally installed applications.
+                tty.debug(f"Reconstructing from old db: {entry.spec}")
                 try:
-                    layout = None if entry.spec.external else directory_layout
-                    kwargs = {
-                        "spec": entry.spec,
-                        "directory_layout": layout,
-                        "explicit": entry.explicit,
-                        "installation_time": entry.installation_time,
-                    }
-                    self._add(**kwargs)
+                    self._add(
+                        spec=entry.spec,
+                        explicit=entry.explicit,
+                        installation_time=entry.installation_time,
+                    )
                     processed_specs.add(entry.spec)
                 except Exception as e:
-                    # Something went wrong, so the spec was not restored
-                    # from old data
+                    # Something went wrong, so the spec was not restored from old data
                     tty.debug(e)
 
             self._check_ref_counts()
@@ -1033,7 +1028,7 @@ def _check_ref_counts(self):
 
         Does no locking.
         """
-        counts = {}
+        counts: Dict[str, int] = {}
         for key, rec in self._data.items():
             counts.setdefault(key, 0)
             for dep in rec.spec.dependencies(deptype=_TRACKED_DEPENDENCIES):
@@ -1117,29 +1112,23 @@ def _read(self):
 
     def _add(
         self,
-        spec,
-        directory_layout=None,
-        explicit=False,
-        installation_time=None,
-        allow_missing=False,
+        spec: "spack.spec.Spec",
+        explicit: bool = False,
+        installation_time: Optional[float] = None,
+        allow_missing: bool = False,
     ):
         """Add an install record for this spec to the database.
 
-        Assumes spec is installed in ``directory_layout.path_for_spec(spec)``.
-
-        Also ensures dependencies are present and updated in the DB as
-        either installed or missing.
+        Also ensures dependencies are present and updated in the DB as either installed or missing.
 
         Args:
-            spec (spack.spec.Spec): spec to be added
-            directory_layout: layout of the spec installation
+            spec: spec to be added
             explicit:
                 Possible values: True, False, any
 
-                A spec that was installed following a specific user
-                request is marked as explicit. If instead it was
-                pulled-in as a dependency of a user requested spec
-                it's considered implicit.
+                A spec that was installed following a specific user request is marked as explicit.
+                If instead it was pulled-in as a dependency of a user requested spec it's
+                considered implicit.
 
             installation_time:
                 Date and time of installation
@@ -1150,48 +1139,42 @@ def _add(
             raise NonConcreteSpecAddError("Specs added to DB must be concrete.")
 
         key = spec.dag_hash()
-        spec_pkg_hash = spec._package_hash
+        spec_pkg_hash = spec._package_hash  # type: ignore[attr-defined]
         upstream, record = self.query_by_spec_hash(key)
         if upstream:
             return
 
-        # Retrieve optional arguments
         installation_time = installation_time or _now()
 
         for edge in spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES):
             if edge.spec.dag_hash() in self._data:
                 continue
-            # allow missing build-only deps. This prevents excessive
-            # warnings when a spec is installed, and its build dep
-            # is missing a build dep; there's no need to install the
-            # build dep's build dep first, and there's no need to warn
-            # about it missing.
-            dep_allow_missing = allow_missing or edge.depflag == dt.BUILD
             self._add(
                 edge.spec,
-                directory_layout,
                 explicit=False,
                 installation_time=installation_time,
-                allow_missing=dep_allow_missing,
+                # allow missing build-only deps. This prevents excessive warnings when a spec is
+                # installed, and its build dep is missing a build dep; there's no need to install
+                # the build dep's build dep first, and there's no need to warn about it missing.
+                allow_missing=allow_missing or edge.depflag == dt.BUILD,
             )
 
         # Make sure the directory layout agrees whether the spec is installed
-        if not spec.external and directory_layout:
-            path = directory_layout.path_for_spec(spec)
+        if not spec.external and self.layout:
+            path = self.layout.path_for_spec(spec)
             installed = False
             try:
-                directory_layout.ensure_installed(spec)
+                self.layout.ensure_installed(spec)
                 installed = True
                 self._installed_prefixes.add(path)
             except DirectoryLayoutError as e:
                 if not (allow_missing and isinstance(e, InconsistentInstallDirectoryError)):
-                    msg = (
-                        "{0} is being {1} in the database with prefix {2}, "
-                        "but this directory does not contain an installation of "
-                        "the spec, due to: {3}"
-                    )
                     action = "updated" if key in self._data else "registered"
-                    tty.warn(msg.format(spec.short_spec, action, path, str(e)))
+                    tty.warn(
+                        f"{spec.short_spec} is being {action} in the database with prefix {path}, "
+                        "but this directory does not contain an installation of "
+                        f"the spec, due to: {e}"
+                    )
         elif spec.external_path:
             path = spec.external_path
             installed = True
@@ -1202,23 +1185,27 @@ def _add(
         if key not in self._data:
             # Create a new install record with no deps initially.
             new_spec = spec.copy(deps=False)
-            extra_args = {"explicit": explicit, "installation_time": installation_time}
-            # Commands other than 'spack install' may add specs to the DB,
-            # we can record the source of an installed Spec with 'origin'
-            if hasattr(spec, "origin"):
-                extra_args["origin"] = spec.origin
-            self._data[key] = InstallRecord(new_spec, path, installed, ref_count=0, **extra_args)
+            self._data[key] = InstallRecord(
+                new_spec,
+                path=path,
+                installed=installed,
+                ref_count=0,
+                explicit=explicit,
+                installation_time=installation_time,
+                origin=None if not hasattr(spec, "origin") else spec.origin,
+            )
 
             # Connect dependencies from the DB to the new copy.
             for dep in spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES):
                 dkey = dep.spec.dag_hash()
                 upstream, record = self.query_by_spec_hash(dkey)
+                assert record, f"Missing dependency {dep.spec} in DB"
                 new_spec._add_dependency(record.spec, depflag=dep.depflag, virtuals=dep.virtuals)
                 if not upstream:
                     record.ref_count += 1
 
-            # Mark concrete once everything is built, and preserve
-            # the original hashes of concrete specs.
+            # Mark concrete once everything is built, and preserve the original hashes of concrete
+            # specs.
             new_spec._mark_concrete()
             new_spec._hash = key
             new_spec._package_hash = spec_pkg_hash
@@ -1231,7 +1218,7 @@ def _add(
         self._data[key].explicit = explicit
 
     @_autospec
-    def add(self, spec, directory_layout, explicit=False):
+    def add(self, spec: "spack.spec.Spec", *, explicit: bool = False) -> None:
         """Add spec at path to database, locking and reading DB to sync.
 
         ``add()`` will lock and read from the DB on disk.
@@ -1240,9 +1227,9 @@ def add(self, spec, directory_layout, explicit=False):
         # TODO: ensure that spec is concrete?
         # Entire add is transactional.
         with self.write_transaction():
-            self._add(spec, directory_layout, explicit=explicit)
+            self._add(spec, explicit=explicit)
 
-    def _get_matching_spec_key(self, spec, **kwargs):
+    def _get_matching_spec_key(self, spec: "spack.spec.Spec", **kwargs) -> str:
         """Get the exact spec OR get a single spec that matches."""
         key = spec.dag_hash()
         upstream, record = self.query_by_spec_hash(key)
@@ -1254,12 +1241,12 @@ def _get_matching_spec_key(self, spec, **kwargs):
         return key
 
     @_autospec
-    def get_record(self, spec, **kwargs):
+    def get_record(self, spec: "spack.spec.Spec", **kwargs) -> Optional[InstallRecord]:
         key = self._get_matching_spec_key(spec, **kwargs)
         upstream, record = self.query_by_spec_hash(key)
         return record
 
-    def _decrement_ref_count(self, spec):
+    def _decrement_ref_count(self, spec: "spack.spec.Spec") -> None:
         key = spec.dag_hash()
 
         if key not in self._data:
@@ -1276,7 +1263,7 @@ def _decrement_ref_count(self, spec):
             for dep in spec.dependencies(deptype=_TRACKED_DEPENDENCIES):
                 self._decrement_ref_count(dep)
 
-    def _increment_ref_count(self, spec):
+    def _increment_ref_count(self, spec: "spack.spec.Spec") -> None:
         key = spec.dag_hash()
 
         if key not in self._data:
@@ -1285,14 +1272,14 @@ def _increment_ref_count(self, spec):
         rec = self._data[key]
         rec.ref_count += 1
 
-    def _remove(self, spec):
+    def _remove(self, spec: "spack.spec.Spec") -> "spack.spec.Spec":
         """Non-locking version of remove(); does real work."""
         key = self._get_matching_spec_key(spec)
         rec = self._data[key]
 
         # This install prefix is now free for other specs to use, even if the
         # spec is only marked uninstalled.
-        if not rec.spec.external and rec.installed:
+        if not rec.spec.external and rec.installed and rec.path:
             self._installed_prefixes.remove(rec.path)
 
         if rec.ref_count > 0:
@@ -1316,7 +1303,7 @@ def _remove(self, spec):
         return rec.spec
 
     @_autospec
-    def remove(self, spec):
+    def remove(self, spec: "spack.spec.Spec") -> "spack.spec.Spec":
         """Removes a spec from the database.  To be called on uninstall.
 
         Reads the database, then:
@@ -1331,7 +1318,7 @@ def remove(self, spec):
         with self.write_transaction():
             return self._remove(spec)
 
-    def deprecator(self, spec):
+    def deprecator(self, spec: "spack.spec.Spec") -> Optional["spack.spec.Spec"]:
         """Return the spec that the given spec is deprecated for, or None"""
         with self.read_transaction():
             spec_key = self._get_matching_spec_key(spec)
@@ -1342,14 +1329,14 @@ def deprecator(self, spec):
             else:
                 return None
 
-    def specs_deprecated_by(self, spec):
+    def specs_deprecated_by(self, spec: "spack.spec.Spec") -> List["spack.spec.Spec"]:
         """Return all specs deprecated in favor of the given spec"""
         with self.read_transaction():
             return [
                 rec.spec for rec in self._data.values() if rec.deprecated_for == spec.dag_hash()
             ]
 
-    def _deprecate(self, spec, deprecator):
+    def _deprecate(self, spec: "spack.spec.Spec", deprecator: "spack.spec.Spec") -> None:
         spec_key = self._get_matching_spec_key(spec)
         spec_rec = self._data[spec_key]
 
@@ -1367,17 +1354,17 @@ def _deprecate(self, spec, deprecator):
         self._data[spec_key] = spec_rec
 
     @_autospec
-    def mark(self, spec, key, value):
+    def mark(self, spec: "spack.spec.Spec", key, value) -> None:
         """Mark an arbitrary record on a spec."""
         with self.write_transaction():
             return self._mark(spec, key, value)
 
-    def _mark(self, spec, key, value):
+    def _mark(self, spec: "spack.spec.Spec", key, value) -> None:
         record = self._data[self._get_matching_spec_key(spec)]
         setattr(record, key, value)
 
     @_autospec
-    def deprecate(self, spec, deprecator):
+    def deprecate(self, spec: "spack.spec.Spec", deprecator: "spack.spec.Spec") -> None:
         """Marks a spec as deprecated in favor of its deprecator"""
         with self.write_transaction():
             return self._deprecate(spec, deprecator)
@@ -1385,16 +1372,16 @@ def deprecate(self, spec, deprecator):
     @_autospec
     def installed_relatives(
         self,
-        spec,
-        direction="children",
-        transitive=True,
+        spec: "spack.spec.Spec",
+        direction: str = "children",
+        transitive: bool = True,
         deptype: Union[dt.DepFlag, dt.DepTypes] = dt.ALL,
-    ):
+    ) -> Set["spack.spec.Spec"]:
         """Return installed specs related to this one."""
         if direction not in ("parents", "children"):
             raise ValueError("Invalid direction: %s" % direction)
 
-        relatives = set()
+        relatives: Set[spack.spec.Spec] = set()
         for spec in self.query(spec):
             if transitive:
                 to_add = spec.traverse(direction=direction, root=False, deptype=deptype)
@@ -1405,17 +1392,13 @@ def installed_relatives(
 
             for relative in to_add:
                 hash_key = relative.dag_hash()
-                upstream, record = self.query_by_spec_hash(hash_key)
+                _, record = self.query_by_spec_hash(hash_key)
                 if not record:
-                    reltype = "Dependent" if direction == "parents" else "Dependency"
-                    msg = "Inconsistent state! %s %s of %s not in DB" % (
-                        reltype,
-                        hash_key,
-                        spec.dag_hash(),
+                    tty.warn(
+                        f"Inconsistent state: "
+                        f"{'dependent' if direction == 'parents' else 'dependency'} {hash_key} of "
+                        f"{spec.dag_hash()} not in DB"
                     )
-                    if self._fail_when_missing_deps:
-                        raise MissingDependenciesError(msg)
-                    tty.warn(msg)
                     continue
 
                 if not record.installed:
@@ -1425,7 +1408,7 @@ def installed_relatives(
         return relatives
 
     @_autospec
-    def installed_extensions_for(self, extendee_spec):
+    def installed_extensions_for(self, extendee_spec: "spack.spec.Spec"):
         """Returns the specs of all packages that extend the given spec"""
         for spec in self.query():
             if spec.package.extends(extendee_spec):
@@ -1684,7 +1667,7 @@ def unused_specs(
         self,
         root_hashes: Optional[Container[str]] = None,
         deptype: Union[dt.DepFlag, dt.DepTypes] = dt.LINK | dt.RUN,
-    ) -> "List[spack.spec.Spec]":
+    ) -> List["spack.spec.Spec"]:
         """Return all specs that are currently installed but not needed by root specs.
 
         By default, roots are all explicit specs in the database. If a set of root

lib/spack/spack/detection/__init__.py

@@ -2,17 +2,11 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-from .common import (
-    DetectedPackage,
-    executable_prefix,
-    set_virtuals_nonbuildable,
-    update_configuration,
-)
+from .common import executable_prefix, set_virtuals_nonbuildable, update_configuration
 from .path import by_path, executables_in_path
 from .test import detection_tests
 
 __all__ = [
-    "DetectedPackage",
     "by_path",
     "executables_in_path",
     "executable_prefix",

lib/spack/spack/detection/common.py

@@ -6,9 +6,9 @@
 function to update packages.yaml given a list of detected packages.
 
 Ideally, each detection method should be placed in a specific subpackage
-and implement at least a function that returns a list of DetectedPackage
-objects. The update in packages.yaml can then be done using the function
-provided here.
+and implement at least a function that returns a list of specs.
+
+The update in packages.yaml can then be done using the function provided here.
 
 The module also contains other functions that might be useful across different
 detection mechanisms.
@@ -17,9 +17,10 @@
 import itertools
 import os
 import os.path
+import pathlib
 import re
 import sys
-from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
+from typing import Dict, List, Optional, Set, Tuple, Union
 
 import llnl.util.tty
 
@@ -30,25 +31,6 @@
 import spack.util.windows_registry
 
 
-class DetectedPackage(NamedTuple):
-    """Information on a package that has been detected."""
-
-    #: Spec that was detected
-    spec: spack.spec.Spec
-    #: Prefix of the spec
-    prefix: str
-
-    def __reduce__(self):
-        return DetectedPackage.restore, (str(self.spec), self.prefix, self.spec.extra_attributes)
-
-    @staticmethod
-    def restore(
-        spec_str: str, prefix: str, extra_attributes: Optional[Dict[str, str]]
-    ) -> "DetectedPackage":
-        spec = spack.spec.Spec.from_detection(spec_str=spec_str, extra_attributes=extra_attributes)
-        return DetectedPackage(spec=spec, prefix=prefix)
-
-
 def _externals_in_packages_yaml() -> Set[spack.spec.Spec]:
     """Returns all the specs mentioned as externals in packages.yaml"""
     packages_yaml = spack.config.get("packages")
@@ -63,7 +45,7 @@ def _externals_in_packages_yaml() -> Set[spack.spec.Spec]:
 
 
 def _pkg_config_dict(
-    external_pkg_entries: List[DetectedPackage],
+    external_pkg_entries: List["spack.spec.Spec"],
 ) -> Dict[str, Union[bool, List[Dict[str, ExternalEntryType]]]]:
     """Generate a package specific config dict according to the packages.yaml schema.
 
@@ -83,22 +65,19 @@ def _pkg_config_dict(
     pkg_dict = spack.util.spack_yaml.syaml_dict()
     pkg_dict["externals"] = []
     for e in external_pkg_entries:
-        if not _spec_is_valid(e.spec):
+        if not _spec_is_valid(e):
             continue
 
         external_items: List[Tuple[str, ExternalEntryType]] = [
-            ("spec", str(e.spec)),
-            ("prefix", e.prefix),
+            ("spec", str(e)),
+            ("prefix", pathlib.Path(e.external_path).as_posix()),
         ]
-        if e.spec.external_modules:
-            external_items.append(("modules", e.spec.external_modules))
+        if e.external_modules:
+            external_items.append(("modules", e.external_modules))
 
-        if e.spec.extra_attributes:
+        if e.extra_attributes:
             external_items.append(
-                (
-                    "extra_attributes",
-                    spack.util.spack_yaml.syaml_dict(e.spec.extra_attributes.items()),
-                )
+                ("extra_attributes", spack.util.spack_yaml.syaml_dict(e.extra_attributes.items()))
             )
 
         # external_items.extend(e.spec.extra_attributes.items())
@@ -136,10 +115,10 @@ def path_to_dict(search_paths: List[str]):
     # entry overrides later entries
     for search_path in reversed(search_paths):
         try:
-            for lib in os.listdir(search_path):
-                lib_path = os.path.join(search_path, lib)
-                if llnl.util.filesystem.is_readable_file(lib_path):
-                    path_to_lib[lib_path] = lib
+            with os.scandir(search_path) as entries:
+                path_to_lib.update(
+                    {entry.path: entry.name for entry in entries if entry.is_file()}
+                )
         except OSError as e:
             msg = f"cannot scan '{search_path}' for external software: {str(e)}"
             llnl.util.tty.debug(msg)
@@ -219,33 +198,32 @@ def library_prefix(library_dir: str) -> str:
 
 
 def update_configuration(
-    detected_packages: Dict[str, List[DetectedPackage]],
+    detected_packages: Dict[str, List["spack.spec.Spec"]],
     scope: Optional[str] = None,
     buildable: bool = True,
 ) -> List[spack.spec.Spec]:
     """Add the packages passed as arguments to packages.yaml
 
     Args:
-        detected_packages: list of DetectedPackage objects to be added
+        detected_packages: list of specs to be added
         scope: configuration scope where to add the detected packages
         buildable: whether the detected packages are buildable or not
     """
     predefined_external_specs = _externals_in_packages_yaml()
     pkg_to_cfg, all_new_specs = {}, []
     for package_name, entries in detected_packages.items():
-        new_entries = [e for e in entries if (e.spec not in predefined_external_specs)]
+        new_entries = [s for s in entries if s not in predefined_external_specs]
 
         pkg_config = _pkg_config_dict(new_entries)
         external_entries = pkg_config.get("externals", [])
         assert not isinstance(external_entries, bool), "unexpected value for external entry"
 
-        all_new_specs.extend([spack.spec.Spec(x["spec"]) for x in external_entries])
+        all_new_specs.extend(new_entries)
         if buildable is False:
             pkg_config["buildable"] = False
         pkg_to_cfg[package_name] = pkg_config
 
     pkgs_cfg = spack.config.get("packages", scope=scope)
-
     pkgs_cfg = spack.config.merge_yaml(pkgs_cfg, pkg_to_cfg)
     spack.config.set("packages", pkgs_cfg, scope=scope)