not == to !=

Signed-off-by: Todd Gamblin <tgamblin@llnl.gov>

.flake8

@@ -28,7 +28,7 @@ max-line-length = 99
 # - F821: undefined name `name`
 #
 per-file-ignores =
-  var/spack/*/package.py:F403,F405,F821
+  var/spack/repos/*/package.py:F403,F405,F821
   *-ci-package.py:F403,F405,F821
 
 # exclude things we usually do not want linting for.

.gitattributes

@@ -1,3 +1,4 @@
 *.py diff=python
+*.lp linguist-language=Prolog
 lib/spack/external/* linguist-vendored
-*.bat text eol=crlf
+*.bat text eol=crlf

.github/workflows/bootstrap.yml

@@ -26,7 +26,7 @@ jobs:
           dnf install -y \
               bzip2 curl file gcc-c++ gcc gcc-gfortran git gzip \
               make patch unzip which xz python3 python3-devel tree \
-              cmake bison bison-devel libstdc++-static gawk
+              cmake bison bison-devel libstdc++-static
       - name: Setup OpenSUSE
         if: ${{ matrix.image == 'opensuse/leap:latest' }}
         run: |

.github/workflows/ci.yaml

@@ -42,17 +42,17 @@ jobs:
           # built-in repository or documentation
           filters: |
             bootstrap:
-            - 'var/spack/repos/spack_repo/builtin/packages/clingo-bootstrap/**'
-            - 'var/spack/repos/spack_repo/builtin/packages/clingo/**'
-            - 'var/spack/repos/spack_repo/builtin/packages/python/**'
-            - 'var/spack/repos/spack_repo/builtin/packages/re2c/**'
-            - 'var/spack/repos/spack_repo/builtin/packages/gnupg/**'
-            - 'var/spack/repos/spack_repo/builtin/packages/libassuan/**'
-            - 'var/spack/repos/spack_repo/builtin/packages/libgcrypt/**'
-            - 'var/spack/repos/spack_repo/builtin/packages/libgpg-error/**'
-            - 'var/spack/repos/spack_repo/builtin/packages/libksba/**'
-            - 'var/spack/repos/spack_repo/builtin/packages/npth/**'
-            - 'var/spack/repos/spack_repo/builtin/packages/pinentry/**'
+            - 'var/spack/repos/builtin/packages/clingo-bootstrap/**'
+            - 'var/spack/repos/builtin/packages/clingo/**'
+            - 'var/spack/repos/builtin/packages/python/**'
+            - 'var/spack/repos/builtin/packages/re2c/**'
+            - 'var/spack/repos/builtin/packages/gnupg/**'
+            - 'var/spack/repos/builtin/packages/libassuan/**'
+            - 'var/spack/repos/builtin/packages/libgcrypt/**'
+            - 'var/spack/repos/builtin/packages/libgpg-error/**'
+            - 'var/spack/repos/builtin/packages/libksba/**'
+            - 'var/spack/repos/builtin/packages/npth/**'
+            - 'var/spack/repos/builtin/packages/pinentry/**'
             - 'lib/spack/**'
             - 'share/spack/**'
             - '.github/workflows/bootstrap.yml'

.github/workflows/prechecks.yml

@@ -25,16 +25,14 @@ jobs:
       with:
         python-version: '3.13'
         cache: 'pip'
-        cache-dependency-path: '.github/workflows/requirements/style/requirements.txt'
     - name: Install Python Packages
       run: |
+        pip install --upgrade pip setuptools
         pip install -r .github/workflows/requirements/style/requirements.txt
     - name: vermin (Spack's Core)
-      run: |
-        vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
+      run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
     - name: vermin (Repositories)
-      run: |
-        vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv var/spack/repos var/spack/test_repos
+      run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv var/spack/repos
 
   # Run style checks on the files that have been changed
   style:
@@ -42,20 +40,23 @@ jobs:
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       with:
-        fetch-depth: 2
+        fetch-depth: 0
     - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b
       with:
         python-version: '3.13'
         cache: 'pip'
-        cache-dependency-path: '.github/workflows/requirements/style/requirements.txt'
     - name: Install Python packages
       run: |
+        pip install --upgrade pip setuptools
         pip install -r .github/workflows/requirements/style/requirements.txt
+    - name: Setup git configuration
+      run: |
+        # Need this for the git tests to succeed.
+        git --version
+        . .github/workflows/bin/setup_git.sh
     - name: Run style tests
       run: |
-        bin/spack style --base HEAD^1
-        bin/spack license verify
-        pylint -j $(nproc) --disable=all --enable=unspecified-encoding --ignore-paths=lib/spack/external lib
+          share/spack/qa/run-style-tests
 
   audit:
     uses: ./.github/workflows/audit.yaml
@@ -65,11 +66,7 @@ jobs:
       python_version: '3.13'
 
   verify-checksums:
-    # do not run if the commit message or PR description contains [skip-verify-checksums]
-    if: >-
-      ${{ inputs.with_packages == 'true' &&
-      !contains(github.event.pull_request.body, '[skip-verify-checksums]') &&
-      !contains(github.event.head_commit.message, '[skip-verify-checksums]') }}
+    if: ${{ inputs.with_packages == 'true' }}
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
@@ -106,3 +103,21 @@ jobs:
           spack -d bootstrap now --dev
           spack -d style -t black
           spack unit-test -V
+
+  # Further style checks from pylint
+  pylint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b
+        with:
+          python-version: '3.13'
+          cache: 'pip'
+      - name: Install Python packages
+        run: |
+          pip install --upgrade pip setuptools pylint
+      - name: Pylint (Spack Core)
+        run: |
+          pylint -j 4 --disable=all --enable=unspecified-encoding --ignore-paths=lib/spack/external lib

.github/workflows/requirements/style/requirements.txt

@@ -1,8 +1,7 @@
 black==25.1.0
-clingo==5.8.0
+clingo==5.7.1
 flake8==7.2.0
 isort==6.0.1
 mypy==1.15.0
-types-six==1.17.0.20250403
+types-six==1.17.0.20250304
 vermin==1.6.0
-pylint==3.3.7

.github/workflows/sync-packages.yaml

@@ -1,34 +0,0 @@
-name: sync with spack/spack-packages
-
-on:
-  push:
-    branches:
-    - develop
-
-jobs:
-  sync:
-    if: github.repository == 'spack/spack'
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout spack/spack
-      run: git clone https://github.com/spack/spack.git
-    - name: Checkout spack/spack-packages
-      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      with:
-        ssh-key: ${{ secrets.SYNC_PACKAGES_KEY }}
-        path: spack-packages
-        repository: spack/spack-packages
-    - name: Install git-filter-repo
-      run: |
-        curl -LfsO https://raw.githubusercontent.com/newren/git-filter-repo/refs/tags/v2.47.0/git-filter-repo
-        echo "67447413e273fc76809289111748870b6f6072f08b17efe94863a92d810b7d94  git-filter-repo" | sha256sum -c -
-        chmod +x git-filter-repo
-        sudo mv git-filter-repo /usr/local/bin/
-    - name: Sync spack/spack-packages with spack/spack
-      run: |
-        cd spack-packages
-        git-filter-repo --quiet --source ../spack --subdirectory-filter var/spack/repos --refs develop
-    - name: Push
-      run: |
-        cd spack-packages
-        git push git@github.com:spack/spack-packages.git develop:develop --force

README.md

@@ -46,42 +46,18 @@ See the
 [Feature Overview](https://spack.readthedocs.io/en/latest/features.html)
 for examples and highlights.
 
-Installation
-----------------
-
-To install spack, first make sure you have Python & Git.
+To install spack and your first package, make sure you have Python & Git.
 Then:
 
-```bash
-git clone -c feature.manyFiles=true --depth=2 https://github.com/spack/spack.git
-```
-
-<details>
-<summary>What are <code>manyFiles=true</code> and <code>--depth=2</code>?</summary>
-<br>
+    $ git clone -c feature.manyFiles=true --depth=2 https://github.com/spack/spack.git
+    $ cd spack/bin
+    $ ./spack install zlib
 
+> [!TIP]
 > `-c feature.manyFiles=true` improves git's performance on repositories with 1,000+ files.
 >
 > `--depth=2` prunes the git history to reduce the size of the Spack installation.
 
-</details>
-
-```bash
-# For bash/zsh/sh
-. spack/share/spack/setup-env.sh
-
-# For tcsh/csh
-source spack/share/spack/setup-env.csh
-
-# For fish
-. spack/share/spack/setup-env.fish
-```
-
-```bash
-# Now you're ready to install a package!
-spack install zlib-ng
-```
-
 Documentation
 ----------------
 

etc/spack/defaults/config.yaml

@@ -90,9 +90,10 @@ config:
   misc_cache: $user_cache_path/cache
 
 
-  # Abort downloads after this many seconds if not data is received.
-  # Setting this to 0 will disable the timeout.
-  connect_timeout: 30
+  # Timeout in seconds used for downloading sources etc. This only applies
+  # to the connection phase and can be increased for slow connections or
+  # servers. 0 means no timeout.
+  connect_timeout: 10
 
 
   # If this is false, tools like curl that use SSL will not verify

etc/spack/defaults/darwin/packages.yaml

@@ -25,8 +25,6 @@ packages:
       glu: [apple-glu]
       unwind: [apple-libunwind]
       uuid: [apple-libuuid]
-  apple-clang:
-    buildable: false
   apple-gl:
     buildable: false
     externals:

etc/spack/defaults/packages.yaml

@@ -72,8 +72,6 @@ packages:
     permissions:
       read: world
       write: user
-  cce:
-    buildable: false
   cray-fftw:
     buildable: false
   cray-libsci:
@@ -88,23 +86,13 @@ packages:
     buildable: false
   essl:
     buildable: false
-  fj:
-    buildable: false
   fujitsu-mpi:
     buildable: false
   fujitsu-ssl2:
     buildable: false
-  glibc:
-    buildable: false
   hpcx-mpi:
     buildable: false
-  iconv:
-    prefer: [libiconv]
   mpt:
     buildable: false
-  musl:
-    buildable: false
   spectrum-mpi:
     buildable: false
-  xl:
-    buildable: false

etc/spack/defaults/repos.yaml

@@ -11,4 +11,4 @@
 #   ~/.spack/repos.yaml
 # -------------------------------------------------------------------------
 repos:
-  - $spack/var/spack/repos/spack_repo/builtin
+  - $spack/var/spack/repos/builtin

etc/spack/defaults/windows/packages.yaml

@@ -23,5 +23,3 @@ packages:
   mpi:
     require:
     - one_of: [msmpi]
-  msvc:
-    buildable: false

lib/spack/docs/basic_usage.rst

@@ -1291,61 +1291,55 @@ based on site policies.
 Variants
 ^^^^^^^^
 
-Variants are named options associated with a particular package and are
-typically used to enable or disable certain features at build time. They
-are optional, as each package must provide default values for each variant
-it makes available.
-
-The names of variants available for a particular package depend on
+Variants are named options associated with a particular package. They are
+optional, as each package must provide default values for each variant it
+makes available. Variants can be specified using
+a flexible parameter syntax ``name=<value>``. For example,
+``spack install mercury debug=True`` will install mercury built with debug
+flags. The names of particular variants available for a package depend on
 what was provided by the package author. ``spack info <package>`` will
 provide information on what build variants are available.
 
-There are different types of variants:
+For compatibility with earlier versions, variants which happen to be
+boolean in nature can be specified by a syntax that represents turning
+options on and off. For example, in the previous spec we could have
+supplied ``mercury +debug`` with the same effect of enabling the debug
+compile time option for the libelf package.
 
-1. Boolean variants. Typically used to enable or disable a feature at
-   compile time. For example, a package might have a ``debug`` variant that
-   can be explicitly enabled with ``+debug`` and disabled with ``~debug``.
-2. Single-valued variants. Often used to set defaults. For example, a package
-   might have a ``compression`` variant that determines the default
-   compression algorithm, which users could set to ``compression=gzip`` or
-   ``compression=zstd``.
-3. Multi-valued variants. A package might have a ``fabrics`` variant that
-   determines which network fabrics to support. Users could set this to
-   ``fabrics=verbs,ofi`` to enable both InfiniBand verbs and OpenFabrics
-   interfaces. The values are separated by commas.
+Depending on the package a variant may have any default value.  For
+``mercury`` here, ``debug`` is ``False`` by default, and we turned it on
+with ``debug=True`` or ``+debug``.  If a variant is ``True`` by default
+you can turn it off by either adding ``-name`` or ``~name`` to the spec.
 
-   The meaning of ``fabrics=verbs,ofi`` is to enable *at least* the specified
-   fabrics, but other fabrics may be enabled as well. If the intent is to
-   enable *only* the specified fabrics, then the ``fabrics:=verbs,ofi``
-   syntax should be used with the ``:=`` operator.
+There are two syntaxes here because, depending on context, ``~`` and
+``-`` may mean different things.  In most shells, the following will
+result in the shell performing home directory substitution:
 
-.. note::
+.. code-block:: sh
 
-   In certain shells, the the ``~`` character is expanded to the home
-   directory. To avoid these issues, avoid whitespace between the package
-   name and the variant:
+   mpileaks ~debug   # shell may try to substitute this!
+   mpileaks~debug    # use this instead
 
-   .. code-block:: sh
+If there is a user called ``debug``, the ``~`` will be incorrectly
+expanded.  In this situation, you would want to write ``libelf
+-debug``.  However, ``-`` can be ambiguous when included after a
+package name without spaces:
 
-      mpileaks ~debug   # shell may try to substitute this!
-      mpileaks~debug    # use this instead
+.. code-block:: sh
 
-   Alternatively, you can use the ``-`` character to disable a variant,
-   but be aware that this requires a space between the package name and
-   the variant:
+   mpileaks-debug     # wrong!
+   mpileaks -debug    # right
 
-   .. code-block:: sh
+Spack allows the ``-`` character to be part of package names, so the
+above will be interpreted as a request for the ``mpileaks-debug``
+package, not a request for ``mpileaks`` built without ``debug``
+options.  In this scenario, you should write ``mpileaks~debug`` to
+avoid ambiguity.
 
-      mpileaks-debug     # wrong: refers to a package named "mpileaks-debug"
-      mpileaks -debug    # right: refers to a package named mpileaks with debug disabled
-
-   As a last resort, ``debug=False`` can also be used to disable a boolean variant.
-
-
-
-"""""""""""""""""""""""""""""""""""
-Variant propagation to dependencies
-"""""""""""""""""""""""""""""""""""
+When spack normalizes specs, it prints them out with no spaces boolean
+variants using the backwards compatibility syntax and uses only ``~``
+for disabled boolean variants.  The ``-`` and spaces on the command
+line are provided for convenience and legibility.
 
 Spack allows variants to propagate their value to the package's
 dependency by using ``++``, ``--``, and ``~~`` for boolean variants.
@@ -1916,7 +1910,7 @@ diagnostics. Issues, if found, are reported to stdout:
    PKG-DIRECTIVES: 1 issue found
    1. lammps: wrong variant in "conflicts" directive
        the variant 'adios' does not exist
-       in /home/spack/spack/var/spack/repos/spack_repo/builtin/packages/lammps/package.py
+       in /home/spack/spack/var/spack/repos/builtin/packages/lammps/package.py
 
 
 ------------

lib/spack/docs/binary_caches.rst

@@ -45,14 +45,10 @@ provided binary cache, which can be a local directory or a remote URL.
 Here is an example where a build cache is created in a local directory named
 "spack-cache", to which we push the "ninja" spec:
 
-ninja-1.12.1-vmvycib6vmiofkdqgrblo7zsvp7odwut
-
 .. code-block:: console
 
     $ spack buildcache push ./spack-cache ninja
-    ==> Selected 30 specs to push to file:///home/spackuser/spack/spack-cache
-    ...
-    ==> [30/30] Pushed ninja@1.12.1/ngldn2k
+    ==> Pushing binary packages to file:///home/spackuser/spack/spack-cache/build_cache
 
 Note that ``ninja`` must be installed locally for this to work.
 
@@ -102,10 +98,9 @@ Now you can use list:
 .. code-block:: console
 
     $  spack buildcache list
-    ==> 24 cached builds.
-    -- linux-ubuntu22.04-sapphirerapids / gcc@12.3.0 ----------------
-    [ ... ]
-    ninja@1.12.1
+    ==> 1 cached build.
+    -- linux-ubuntu20.04-skylake / gcc@9.3.0 ------------------------
+    ninja@1.10.2
 
 With ``mymirror`` configured and an index available, Spack will automatically
 use it during concretization and installation. That means that you can expect
@@ -116,17 +111,17 @@ verify by re-installing ninja:
 
     $ spack uninstall ninja
     $ spack install ninja
-    [ ... ]
-    ==> Installing ninja-1.12.1-ngldn2kpvb6lqc44oqhhow7fzg7xu7lh [24/24]
-    gpg: Signature made Thu 06 Mar 2025 10:03:38 AM MST
-    gpg:                using RSA key 75BC0528114909C076E2607418010FFAD73C9B07
+    ==> Installing ninja-1.11.1-yxferyhmrjkosgta5ei6b4lqf6bxbscz
+    ==> Fetching file:///home/spackuser/spack/spack-cache/build_cache/linux-ubuntu20.04-skylake-gcc-9.3.0-ninja-1.10.2-yxferyhmrjkosgta5ei6b4lqf6bxbscz.spec.json.sig
+    gpg: Signature made Do 12 Jan 2023 16:01:04 CET
+    gpg:                using RSA key 61B82B2B2350E171BD17A1744E3A689061D57BF6
     gpg: Good signature from "example (GPG created for Spack) <example@example.com>" [ultimate]
-    ==> Fetching file:///home/spackuser/spack/spack-cache/blobs/sha256/f0/f08eb62661ad159d2d258890127fc6053f5302a2f490c1c7f7bd677721010ee0
-    ==> Fetching file:///home/spackuser/spack/spack-cache/blobs/sha256/c7/c79ac6e40dfdd01ac499b020e52e57aa91151febaea3ad183f90c0f78b64a31a
-    ==> Extracting ninja-1.12.1-ngldn2kpvb6lqc44oqhhow7fzg7xu7lh from binary cache
-    ==> ninja: Successfully installed ninja-1.12.1-ngldn2kpvb6lqc44oqhhow7fzg7xu7lh
-      Search: 0.00s.  Fetch: 0.11s.  Install: 0.11s.  Extract: 0.10s.  Relocate: 0.00s.  Total: 0.22s
-    [+] /home/spackuser/spack/opt/spack/linux-ubuntu22.04-sapphirerapids/gcc-12.3.0/ninja-1.12.1-ngldn2kpvb6lqc44oqhhow7fzg7xu7lh
+    ==> Fetching file:///home/spackuser/spack/spack-cache/build_cache/linux-ubuntu20.04-skylake/gcc-9.3.0/ninja-1.10.2/linux-ubuntu20.04-skylake-gcc-9.3.0-ninja-1.10.2-yxferyhmrjkosgta5ei6b4lqf6bxbscz.spack
+    ==> Extracting ninja-1.10.2-yxferyhmrjkosgta5ei6b4lqf6bxbscz from binary cache
+    ==> ninja: Successfully installed ninja-1.11.1-yxferyhmrjkosgta5ei6b4lqf6bxbscz
+    Search: 0.00s.  Fetch: 0.17s.  Install: 0.12s.  Total: 0.29s
+    [+] /home/harmen/spack/opt/spack/linux-ubuntu20.04-skylake/gcc-9.3.0/ninja-1.11.1-yxferyhmrjkosgta5ei6b4lqf6bxbscz
+
 
 It worked! You've just completed a full example of creating a build cache with
 a spec of interest, adding it as a mirror, updating its index, listing the contents,
@@ -349,18 +344,19 @@ which lets you get started quickly. See the following resources for more informa
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Create tarball of installed Spack package and all dependencies.
-Tarballs and specfiles are compressed and checksummed, manifests are signed if gpg2 is available.
-Commands like ``spack buildcache install`` will search Spack mirrors to get the list of build caches.
+Tarballs are checksummed and signed if gpg2 is available.
+Places them in a directory ``build_cache`` that can be copied to a mirror.
+Commands like ``spack buildcache install`` will search Spack mirrors for build_cache to get the list of build caches.
 
 ==============  ========================================================================================================================
 Arguments       Description
 ==============  ========================================================================================================================
 ``<specs>``     list of partial specs or hashes with a leading ``/`` to match from installed packages and used for creating build caches
-``-d <path>``   directory in which ``v3`` and ``blobs`` directories are created, defaults to ``.``
-``-f``          overwrite compressed tarball and spec metadata files if they already exist
+``-d <path>``   directory in which ``build_cache`` directory is created, defaults to ``.``
+``-f``          overwrite ``.spack`` file in ``build_cache`` directory if it exists
 ``-k <key>``    the key to sign package with. In the case where multiple keys exist, the package will be unsigned unless ``-k`` is used.
 ``-r``          make paths in binaries relative before creating tarball
-``-y``          answer yes to all questions about creating unsigned build caches
+``-y``          answer yes to all create unsigned ``build_cache`` questions
 ==============  ========================================================================================================================
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -401,165 +397,6 @@ List public keys available on Spack mirror.
 =========  ==============================================
 Arguments  Description
 =========  ==============================================
-``-it``    trust the keys downloaded with prompt for each
+``-i``     trust the keys downloaded with prompt for each
 ``-y``     answer yes to all trust all keys downloaded
 =========  ==============================================
-
-.. _build_cache_layout:
-
-------------------
-Build Cache Layout
-------------------
-
-This section describes the structure and content of URL-style build caches, as
-distinguished from OCI-style build caches.
-
-The entry point for a binary package is a manifest json file that points to at
-least two other files stored as content-addressed blobs. These files include a spec
-metadata file, as well as the installation directory of the package stored as
-a compressed archive file. Binary package manifest files are named to indicate
-the package name and version, as well as the hash of the concrete spec. For
-example::
-
-  gcc-runtime-12.3.0-qyu2lvgt3nxh7izxycugdbgf5gsdpkjt.spec.manifest.json
-
-would contain the manifest for a binary package of ``gcc-runtime@12.3.0``.
-The id of the built package is defined to be the DAG hash of the concrete spec,
-and exists in the name of the file as well. The id distinguishes a particular
-binary package from all other binary packages with the same package name and
-version. Below is an example binary package manifest file. Such a file would
-live in the versioned spec manifests directory of a binary mirror, for example
-``v3/manifests/spec/``::
-
-  {
-    "version": 3,
-    "data": [
-      {
-        "contentLength": 10731083,
-        "mediaType": "application/vnd.spack.install.v2.tar+gzip",
-        "compression": "gzip",
-        "checksumAlgorithm": "sha256",
-        "checksum": "0f24aa6b5dd7150067349865217acd3f6a383083f9eca111d2d2fed726c88210"
-      },
-      {
-        "contentLength": 1000,
-        "mediaType": "application/vnd.spack.spec.v5+json",
-        "compression": "gzip",
-        "checksumAlgorithm": "sha256",
-        "checksum": "fba751c4796536737c9acbb718dad7429be1fa485f5585d450ab8b25d12ae041"
-      }
-    ]
-  }
-
-The manifest points to both the compressed tar file as well as the compressed
-spec metadata file, and contains the checksum of each. This checksum
-is also used as the address of the associated file, and hence, must be
-known in order to locate the tarball or spec file within the mirror. Once the
-tarball or spec metadata file is downloaded, the checksum should be computed locally
-and compared to the checksum in the manifest to ensure the contents have not changed
-since the binary package was pushed. Spack stores all data files (including compressed
-tar files, spec metadata, indices, public keys, etc) within a ``blobs/<hash-algorithm>/``
-directory, using the first two characters of the checksum as a sub-directory
-to reduce the number files in a single folder.  Here is a depiction of the
-organization of binary mirror contents::
-
-  mirror_directory/
-    v3/
-      layout.json
-      manifests/
-        spec/
-          gcc-runtime/
-            gcc-runtime-12.3.0-s2nqujezsce4x6uhtvxscu7jhewqzztx.spec.manifest.json
-          gmake/
-            gmake-4.4.1-lpr4j77rcgkg5536tmiuzwzlcjsiomph.spec.manifest.json
-          compiler-wrapper/
-            compiler-wrapper-1.0-s7ieuyievp57vwhthczhaq2ogowf3ohe.spec.manifest.json
-        index/
-          index.manifest.json
-        key/
-          75BC0528114909C076E2607418010FFAD73C9B07.key.manifest.json
-          keys.manifest.json
-    blobs/
-      sha256/
-        0f/
-          0f24aa6b5dd7150067349865217acd3f6a383083f9eca111d2d2fed726c88210
-        fb/
-          fba751c4796536737c9acbb718dad7429be1fa485f5585d450ab8b25d12ae041
-        2a/
-          2a21836d206ccf0df780ab0be63fdf76d24501375306a35daa6683c409b7922f
-        ...
-
-Files within the ``manifests`` directory are organized into subdirectories by
-the type of entity they represent. Binary package manifests live in the ``spec/``
-directory, binary cache index manifests live in the ``index/`` directory, and
-manifests for public keys and their indices live in the ``key/`` subdirectory.
-Regardless of the type of entity they represent, all manifest files are named
-with an extension ``.manifest.json``.
-
-Every manifest contains a ``data`` array, each element of which refers to an
-associated file stored a content-addressed blob.  Considering the example spec
-manifest shown above, the compressed installation archive can be found by
-picking out the data blob with the appropriate ``mediaType``, which in this
-case would be ``application/vnd.spack.install.v1.tar+gzip``. The associated
-file is found by looking in the blobs directory under ``blobs/sha256/fb/`` for
-the file named with the complete checksum value.
-
-As mentioned above, every entity in a binary mirror (aka build cache) is stored
-as a content-addressed blob pointed to by a manifest. While an example spec
-manifest (i.e. a manifest for a binary package) is shown above, here is what
-the manifest of a build cache index looks like::
-
-  {
-    "version": 3,
-    "data": [
-      {
-        "contentLength": 6411,
-        "mediaType": "application/vnd.spack.db.v8+json",
-        "compression": "none",
-        "checksumAlgorithm": "sha256",
-        "checksum": "225a3e9da24d201fdf9d8247d66217f5b3f4d0fc160db1498afd998bfd115234"
-      }
-    ]
-  }
-
-Some things to note about this manifest are that it points to a blob that is not
-compressed (``compression: "none"``), and that the ``mediaType`` is one we have
-not seen yet, ``application/vnd.spack.db.v8+json``. The decision not to compress
-build cache indices stems from the fact that spack does not yet sign build cache
-index manifests. Once that changes, you may start to see these indices stored as
-compressed blobs.
-
-For completeness, here are examples of manifests for the other two types of entities
-you might find in a spack build cache. First a public key manifest::
-
-  {
-    "version": 3,
-    "data": [
-      {
-        "contentLength": 2472,
-        "mediaType": "application/pgp-keys",
-        "compression": "none",
-        "checksumAlgorithm": "sha256",
-        "checksum": "9fc18374aebc84deb2f27898da77d4d4410e5fb44c60c6238cb57fb36147e5c7"
-      }
-    ]
-  }
-
-Note the ``mediaType`` of ``application/pgp-keys``. Finally, a public key index manifest::
-
-  {
-    "version": 3,
-    "data": [
-      {
-        "contentLength": 56,
-        "mediaType": "application/vnd.spack.keyindex.v1+json",
-        "compression": "none",
-        "checksumAlgorithm": "sha256",
-        "checksum": "29b3a0eb6064fd588543bc43ac7d42d708a69058dafe4be0859e3200091a9a1c"
-      }
-    ]
-  }
-
-Again note the ``mediaType`` of ``application/vnd.spack.keyindex.v1+json``. Also note
-that both the above manifest examples refer to uncompressed blobs, this is for the same
-reason spack does not yet compress build cache index blobs.

lib/spack/docs/build_systems.rst

@@ -83,7 +83,7 @@ packages. You can quickly find examples by running:
 
 .. code-block:: console
 
-   $ cd var/spack/repos/spack_repo/builtin/packages
+   $ cd var/spack/repos/builtin/packages
    $ grep -l QMakePackage */package.py
 
 

lib/spack/docs/build_systems/bundlepackage.rst

@@ -27,10 +27,10 @@ it could use the ``require`` directive as follows:
 
 Spack has a number of built-in bundle packages, such as:
 
-* `AmdAocl <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/amd_aocl/package.py>`_
-* `EcpProxyApps <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/ecp_proxy_apps/package.py>`_
-* `Libc <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/libc/package.py>`_
-* `Xsdk <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/xsdk/package.py>`_
+* `AmdAocl <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/amd-aocl/package.py>`_
+* `EcpProxyApps <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/ecp-proxy-apps/package.py>`_
+* `Libc <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/libc/package.py>`_
+* `Xsdk <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/xsdk/package.py>`_
 
 where ``Xsdk`` also inherits from ``CudaPackage`` and ``RocmPackage`` and
 ``Libc`` is a virtual bundle package for the C standard library.

lib/spack/docs/build_systems/cmakepackage.rst

@@ -199,7 +199,7 @@ a variant to control this:
 However, not every CMake package accepts all four of these options.
 Grep the ``CMakeLists.txt`` file to see if the default values are
 missing or replaced. For example, the
-`dealii <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/dealii/package.py>`_
+`dealii <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/dealii/package.py>`_
 package overrides the default variant with:
 
 .. code-block:: python

lib/spack/docs/build_systems/custompackage.rst

@@ -20,8 +20,8 @@ start is to look at the definitions of other build systems. This guide
 focuses mostly on how Spack's build systems work.
 
 In this guide, we will be using the
-`perl <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/perl/package.py>`_ and
-`cmake <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/cmake/package.py>`_
+`perl <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/perl/package.py>`_ and
+`cmake <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/cmake/package.py>`_
 packages as examples. ``perl``'s build system is a hand-written
 ``Configure`` shell script, while ``cmake`` bootstraps itself during
 installation. Both of these packages require custom build systems.

lib/spack/docs/build_systems/makefilepackage.rst

@@ -91,14 +91,14 @@ there are any other variables you need to set, you can do this in the
 
 .. code-block:: python
 
-   def setup_build_environment(self, env: EnvironmentModifications) -> None:
+   def setup_build_environment(self, env):
        env.set("PREFIX", prefix)
        env.set("BLASLIB", spec["blas"].libs.ld_flags)
 
 
-`cbench <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/cbench/package.py>`_
+`cbench <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/cbench/package.py>`_
 is a good example of a simple package that does this, while
-`esmf <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/esmf/package.py>`_
+`esmf <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/esmf/package.py>`_
 is a good example of a more complex package.
 
 """"""""""""""""""""""
@@ -129,7 +129,7 @@ If you do need access to the spec, you can create a property like so:
        ]
 
 
-`cloverleaf <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/cloverleaf/package.py>`_
+`cloverleaf <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/cloverleaf/package.py>`_
 is a good example of a package that uses this strategy.
 
 """""""""""""
@@ -152,7 +152,7 @@ and a ``filter`` method to help with this. For example:
        makefile.filter(r"^\s*FC\s*=.*",  f"FC = {spack_fc}")
 
 
-`stream <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/stream/package.py>`_
+`stream <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/stream/package.py>`_
 is a good example of a package that involves editing a Makefile to set
 the appropriate variables.
 
@@ -192,7 +192,7 @@ well for storing variables:
                inc.write(f"{key} = {config[key]}\n")
 
 
-`elk <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/elk/package.py>`_
+`elk <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/elk/package.py>`_
 is a good example of a package that uses a dictionary to store
 configuration variables.
 
@@ -213,7 +213,7 @@ them in a list:
                inc.write(f"{var}\n")
 
 
-`hpl <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/hpl/package.py>`_
+`hpl <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/hpl/package.py>`_
 is a good example of a package that uses a list to store
 configuration variables.
 

lib/spack/docs/build_systems/rpackage.rst

@@ -39,7 +39,7 @@ for "CRAN <package-name>" and you should quickly find what you want.
 If it isn't on CRAN, try Bioconductor, another common R repository.
 
 For the purposes of this tutorial, we will be walking through
-`r-caret <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/r_caret/package.py>`_
+`r-caret <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/r-caret/package.py>`_
 as an example. If you search for "CRAN caret", you will quickly find what
 you are looking for at https://cran.r-project.org/package=caret.
 https://cran.r-project.org is the main CRAN website. However, CRAN also
@@ -337,7 +337,7 @@ Non-R dependencies
 ^^^^^^^^^^^^^^^^^^
 
 Some packages depend on non-R libraries for linking. Check out the
-`r-stringi <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/r_stringi/package.py>`_
+`r-stringi <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/r-stringi/package.py>`_
 package for an example: https://cloud.r-project.org/package=stringi.
 If you search for the text "SystemRequirements", you will see:
 
@@ -352,7 +352,7 @@ Passing arguments to the installation
 
 Some R packages provide additional flags that can be passed to
 ``R CMD INSTALL``, often to locate non-R dependencies.
-`r-rmpi <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/r_rmpi/package.py>`_
+`r-rmpi <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/r-rmpi/package.py>`_
 is an example of this, and flags for linking to an MPI library. To pass
 these to the installation command, you can override ``configure_args``
 like so:

lib/spack/docs/build_systems/sconspackage.rst

@@ -104,10 +104,10 @@ Finding available options
 
 The first place to start when looking for a list of valid options to
 build a package is ``scons --help``. Some packages like
-`kahip <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/kahip/package.py>`_
+`kahip <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/kahip/package.py>`_
 don't bother overwriting the default SCons help message, so this isn't
 very useful, but other packages like
-`serf <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/serf/package.py>`_
+`serf <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/serf/package.py>`_
 print a list of valid command-line variables:
 
 .. code-block:: console
@@ -177,7 +177,7 @@ print a list of valid command-line variables:
 
 
 More advanced packages like
-`cantera <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/cantera/package.py>`_
+`cantera <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/cantera/package.py>`_
 use ``scons --help`` to print a list of subcommands:
 
 .. code-block:: console

lib/spack/docs/conf.py

@@ -225,14 +225,8 @@ def setup(sphinx):
     ("py:class", "llnl.util.lang.T"),
     ("py:class", "llnl.util.lang.KT"),
     ("py:class", "llnl.util.lang.VT"),
-    ("py:class", "llnl.util.lang.K"),
-    ("py:class", "llnl.util.lang.V"),
-    ("py:class", "llnl.util.lang.ClassPropertyType"),
     ("py:obj", "llnl.util.lang.KT"),
     ("py:obj", "llnl.util.lang.VT"),
-    ("py:obj", "llnl.util.lang.ClassPropertyType"),
-    ("py:obj", "llnl.util.lang.K"),
-    ("py:obj", "llnl.util.lang.V"),
 ]
 
 # The reST default role (used for this markup: `text`) to use for all documents.

lib/spack/docs/configuration.rst

@@ -46,12 +46,6 @@ Each Spack configuration file is nested under a top-level section
 corresponding to its name. So, ``config.yaml`` starts with ``config:``,
 ``mirrors.yaml`` starts with ``mirrors:``, etc.
 
-.. tip::
-
-   Validation and autocompletion of Spack config files can be enabled in
-   your editor with the YAML language server. See `spack/schemas
-   <https://github.com/spack/schemas>`_ for more information.
-
 .. _configuration-scopes:
 
 --------------------

lib/spack/docs/contribution_guide.rst

@@ -226,9 +226,9 @@ If all is well, you'll see something like this:
 
    Modified files:
 
-     var/spack/repos/spack_repo/builtin/packages/hdf5/package.py
-     var/spack/repos/spack_repo/builtin/packages/hdf/package.py
-     var/spack/repos/spack_repo/builtin/packages/netcdf/package.py
+     var/spack/repos/builtin/packages/hdf5/package.py
+     var/spack/repos/builtin/packages/hdf/package.py
+     var/spack/repos/builtin/packages/netcdf/package.py
    =======================================================
    Flake8 checks were clean.
 
@@ -236,9 +236,9 @@ However, if you aren't compliant with PEP 8, flake8 will complain:
 
 .. code-block:: console
 
-   var/spack/repos/spack_repo/builtin/packages/netcdf/package.py:26: [F401] 'os' imported but unused
-   var/spack/repos/spack_repo/builtin/packages/netcdf/package.py:61: [E303] too many blank lines (2)
-   var/spack/repos/spack_repo/builtin/packages/netcdf/package.py:106: [E501] line too long (92 > 79 characters)
+   var/spack/repos/builtin/packages/netcdf/package.py:26: [F401] 'os' imported but unused
+   var/spack/repos/builtin/packages/netcdf/package.py:61: [E303] too many blank lines (2)
+   var/spack/repos/builtin/packages/netcdf/package.py:106: [E501] line too long (92 > 79 characters)
    Flake8 found errors.
 
 Most of the error messages are straightforward, but if you don't understand what
@@ -280,7 +280,7 @@ All of these can be installed with Spack, e.g.
 
 .. warning::
 
-   Sphinx has `several required dependencies <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/py-sphinx/package.py>`_.
+   Sphinx has `several required dependencies <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/py-sphinx/package.py>`_.
    If you're using a ``python`` from Spack and you installed
    ``py-sphinx`` and friends, you need to make them available to your
    ``python``. The easiest way to do this is to run:

lib/spack/docs/developer_guide.rst

@@ -154,7 +154,9 @@ Package-related modules
 
 :mod:`spack.util.naming`
   Contains functions for mapping between Spack package names,
-  Python module names, and Python class names.
+  Python module names, and Python class names. Functions like
+  :func:`~spack.util.naming.mod_to_class` handle mapping package
+  module names to class names.
 
 :mod:`spack.directives`
   *Directives* are functions that can be called inside a package definition

lib/spack/docs/env_vars_yaml.rst

@@ -1,34 +0,0 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
-
-   SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-.. _env-vars-yaml:
-
-=============================================
-Environment Variable Settings (env_vars.yaml)
-=============================================
-
-Spack allows you to include shell environment variable modifications
-for a spack environment by including an ``env_vars.yaml``.  Environment
-varaibles can be modified by setting, unsetting, appending, and prepending
-variables in the shell environment.
-The changes to the shell environment will take effect when the spack
-environment is activated.
-
-for example,
-
-.. code-block:: yaml
-
-  env_vars:
-    set:
-      ENVAR_TO_SET_IN_ENV_LOAD: "FOO"
-    unset:
-      ENVAR_TO_UNSET_IN_ENV_LOAD:
-    prepend_path:
-      PATH_LIST: "path/to/prepend"
-    append_path:
-      PATH_LIST: "path/to/append"
-    remove_path:
-      PATH_LIST: "path/to/remove"
-
-

lib/spack/docs/environments.rst

@@ -539,9 +539,7 @@ from the command line.
 
 You can also include an environment directly in the ``spack.yaml`` file. It
 involves adding the ``include_concrete`` heading in the yaml followed by the
-absolute path to the independent environments. Note, that you may use Spack
-config variables such as ``$spack`` or environment variables as long as the
-expression expands to an absolute path.
+absolute path to the independent environments.
 
 .. code-block:: yaml
 
@@ -551,7 +549,7 @@ expression expands to an absolute path.
          unify: true
      include_concrete:
      - /absolute/path/to/environment1
-     - $spack/../path/to/environment2
+     - /absolute/path/to/environment2
 
 
 Once the ``spack.yaml`` has been updated you must concretize the environment to
@@ -1002,28 +1000,6 @@ For example, the following environment has three root packages:
 This allows for a much-needed reduction in redundancy between packages
 and constraints.
 
--------------------------------
-Modifying Environment Variables
--------------------------------
-
-Spack Environments can modify the active shell's environment variables when activated.  The environment can be
-configured to set, unset, prepend, or append using ``env_vars`` configuration in the ``spack.yaml``  or through config scopes
-file:
-
-.. code-block:: yaml
-
-  spack:
-    env_vars:
-      set:
-        ENVAR_TO_SET_IN_ENV_LOAD: "FOO"
-      unset:
-        ENVAR_TO_UNSET_IN_ENV_LOAD:
-      prepend_path:
-        PATH_LIST: "path/to/prepend"
-      append_path:
-        PATH_LIST: "path/to/append"
-      remove_path:
-        PATH_LIST: "path/to/remove"
 
 -----------------
 Environment Views

lib/spack/docs/features.rst

@@ -131,7 +131,7 @@ creates a simple python file:
 It doesn't take much python coding to get from there to a working
 package:
 
-.. literalinclude:: _spack_root/var/spack/repos/spack_repo/builtin/packages/libelf/package.py
+.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/libelf/package.py
    :lines: 5-
 
 Spack also provides wrapper functions around common commands like

lib/spack/docs/index.rst

@@ -75,7 +75,6 @@ or refer to the full manual below.
    packages_yaml
    build_settings
    environments
-   env_vars_yaml
    containers
    mirrors
    module_file_support

lib/spack/docs/module_file_support.rst

@@ -128,7 +128,7 @@ depend on the spec:
 
 .. code-block:: python
 
-   def setup_run_environment(self, env: EnvironmentModifications) -> None:
+   def setup_run_environment(self, env):
        if self.spec.satisfies("+foo"):
            env.set("FOO", "bar")
 
@@ -142,7 +142,7 @@ For example, a simplified version of the ``python`` package could look like this
 
 .. code-block:: python
 
-   def setup_dependent_run_environment(self, env: EnvironmentModifications, dependent_spec: Spec) -> None:
+   def setup_dependent_run_environment(self, env, dependent_spec):
        if dependent_spec.package.extends(self.spec):
            env.prepend_path("PYTHONPATH", dependent_spec.prefix.lib.python)
 

lib/spack/docs/packaging_guide.rst

@@ -369,9 +369,9 @@ If you have a collection of software expected to work well together with
 no source code of its own, you can create a :ref:`BundlePackage <bundlepackage>`.
 Examples where bundle packages can be useful include defining suites of
 applications (e.g, `EcpProxyApps
-<https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/ecp_proxy_apps/package.py>`_), commonly used libraries
-(e.g., `AmdAocl <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/amd_aocl/package.py>`_),
-and software development kits (e.g., `EcpDataVisSdk <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/ecp_data_vis_sdk/package.py>`_).
+<https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/ecp-proxy-apps/package.py>`_), commonly used libraries
+(e.g., `AmdAocl <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/amd-aocl/package.py>`_),
+and software development kits (e.g., `EcpDataVisSdk <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/ecp-data-vis-sdk/package.py>`_).
 
 These versioned packages primarily consist of dependencies on the associated
 software packages. They can include :ref:`variants <variants>` to ensure
@@ -443,7 +443,7 @@ lives in:
 .. code-block:: console
 
    $ spack location -p gmp
-   ${SPACK_ROOT}/var/spack/repos/spack_repo/builtin/packages/gmp/package.py
+   ${SPACK_ROOT}/var/spack/repos/builtin/packages/gmp/package.py
 
 but ``spack edit`` provides a much simpler shortcut and saves you the
 trouble of typing the full path.
@@ -457,19 +457,19 @@ live in Spack's directory structure.  In general, :ref:`cmd-spack-create`
 handles creating package files for you, so you can skip most of the
 details here.
 
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-``var/spack/repos/spack_repo/builtin/packages``
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+``var/spack/repos/builtin/packages``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 A Spack installation directory is structured like a standard UNIX
 install prefix (``bin``, ``lib``, ``include``, ``var``, ``opt``,
 etc.).  Most of the code for Spack lives in ``$SPACK_ROOT/lib/spack``.
-Packages themselves live in ``$SPACK_ROOT/var/spack/repos/spack_repo/builtin/packages``.
+Packages themselves live in ``$SPACK_ROOT/var/spack/repos/builtin/packages``.
 
 If you ``cd`` to that directory, you will see directories for each
 package:
 
-.. command-output:: cd $SPACK_ROOT/var/spack/repos/spack_repo/builtin/packages && ls
+.. command-output:: cd $SPACK_ROOT/var/spack/repos/builtin/packages && ls
    :shell:
    :ellipsis: 10
 
@@ -479,7 +479,7 @@ package lives in:
 
 .. code-block:: none
 
-   $SPACK_ROOT/var/spack/repos/spack_repo/builtin/packages/libelf/package.py
+   $SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py
 
 Alongside the ``package.py`` file, a package may contain extra
 directories or files (like patches) that it needs to build.
@@ -492,12 +492,12 @@ Packages are named after the directory containing ``package.py``. So,
 ``libelf``'s ``package.py`` lives in a directory called ``libelf``.
 The ``package.py`` file defines a class called ``Libelf``, which
 extends Spack's ``Package`` class.  For example, here is
-``$SPACK_ROOT/var/spack/repos/spack_repo/builtin/packages/libelf/package.py``:
+``$SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py``:
 
 .. code-block:: python
    :linenos:
 
-   from spack.package import *
+   from spack import *
 
    class Libelf(Package):
        """ ... description ... """
@@ -520,7 +520,7 @@ these:
    $ spack install libelf@0.8.13
 
 Spack sees the package name in the spec and looks for
-``libelf/package.py`` in ``var/spack/repos/spack_repo/builtin/packages``.
+``libelf/package.py`` in ``var/spack/repos/builtin/packages``.
 Likewise, if you run ``spack install py-numpy``, Spack looks for
 ``py-numpy/package.py``.
 
@@ -686,7 +686,7 @@ https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.bz2
 In order to handle this, you can define a ``url_for_version()`` function
 like so:
 
-.. literalinclude:: _spack_root/var/spack/repos/spack_repo/builtin/packages/openmpi/package.py
+.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/openmpi/package.py
    :pyobject: Openmpi.url_for_version
 
 With the use of this ``url_for_version()``, Spack knows to download OpenMPI ``2.1.1``
@@ -787,7 +787,7 @@ of GNU. For that, Spack goes a step further and defines a mixin class that
 takes care of all of the plumbing and requires packagers to just define a proper
 ``gnu_mirror_path`` attribute:
 
-.. literalinclude:: _spack_root/var/spack/repos/spack_repo/builtin/packages/autoconf/package.py
+.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/autoconf/package.py
    :lines: 9-18
 
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1089,7 +1089,7 @@ You've already seen the ``homepage`` and ``url`` package attributes:
 .. code-block:: python
    :linenos:
 
-   from spack.package import *
+   from spack import *
 
 
    class Mpich(Package):
@@ -1995,7 +1995,7 @@ structure like this:
 
 .. code-block:: none
 
-   $SPACK_ROOT/var/spack/repos/spack_repo/builtin/packages/
+   $SPACK_ROOT/var/spack/repos/builtin/packages/
        mvapich2/
            package.py
            ad_lustre_rwcontig_open_source.patch
@@ -2133,7 +2133,7 @@ handles ``RPATH``:
 
 .. _pyside-patch:
 
-.. literalinclude:: _spack_root/var/spack/repos/spack_repo/builtin/packages/py_pyside/package.py
+.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/py-pyside/package.py
    :pyobject: PyPyside.patch
    :linenos:
 
@@ -2201,7 +2201,7 @@ using the ``spack resource show`` command::
 
   $ spack resource show 3877ab54
   3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00
-      path:       /home/spackuser/src/spack/var/spack/repos/spack_repo/builtin/packages/m4/gnulib-pgi.patch
+      path:       /home/spackuser/src/spack/var/spack/repos/builtin/packages/m4/gnulib-pgi.patch
       applies to: builtin.m4
 
 ``spack resource show`` looks up downloadable resources from package
@@ -2219,7 +2219,7 @@ wonder where the extra boost patches are coming from::
           ^boost@1.68.0%apple-clang@9.0.0+atomic+chrono~clanglibcpp cxxstd=default +date_time~debug+exception+filesystem+graph~icu+iostreams+locale+log+math~mpi+multithreaded~numpy patches=2ab6c72d03dec6a4ae20220a9dfd5c8c572c5294252155b85c6874d97c323199,b37164268f34f7133cbc9a4066ae98fda08adf51e1172223f6a969909216870f ~pic+program_options~python+random+regex+serialization+shared+signals~singlethreaded+system~taggedlayout+test+thread+timer~versionedlayout+wave arch=darwin-highsierra-x86_64
   $ spack resource show b37164268
   b37164268f34f7133cbc9a4066ae98fda08adf51e1172223f6a969909216870f
-      path:       /home/spackuser/src/spack/var/spack/repos/spack_repo/builtin/packages/dealii/boost_1.68.0.patch
+      path:       /home/spackuser/src/spack/var/spack/repos/builtin/packages/dealii/boost_1.68.0.patch
       applies to: builtin.boost
       patched by: builtin.dealii
 
@@ -2930,7 +2930,7 @@ this, Spack provides four different methods that can be overridden in a package:
 
 The Qt package, for instance, uses this call:
 
-.. literalinclude:: _spack_root/var/spack/repos/spack_repo/builtin/packages/qt/package.py
+.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/qt/package.py
    :pyobject: Qt.setup_dependent_build_environment
    :linenos:
 
@@ -2958,7 +2958,7 @@ variables to be used by the dependent. This is done by implementing
 :meth:`setup_dependent_package <spack.package_base.PackageBase.setup_dependent_package>`. An
 example of this can be found in the ``Python`` package:
 
-.. literalinclude:: _spack_root/var/spack/repos/spack_repo/builtin/packages/python/package.py
+.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/python/package.py
    :pyobject: Python.setup_dependent_package
    :linenos:
 
@@ -3785,7 +3785,7 @@ It is usually sufficient for a packager to override a few
 build system specific helper methods or attributes to provide, for instance,
 configure arguments:
 
-.. literalinclude::  _spack_root/var/spack/repos/spack_repo/builtin/packages/m4/package.py
+.. literalinclude::  _spack_root/var/spack/repos/builtin/packages/m4/package.py
     :pyobject: M4.configure_args
     :linenos:
 
@@ -4110,7 +4110,7 @@ Shell command functions
 
 Recall the install method from ``libelf``:
 
-.. literalinclude::  _spack_root/var/spack/repos/spack_repo/builtin/packages/libelf/package.py
+.. literalinclude::  _spack_root/var/spack/repos/builtin/packages/libelf/package.py
    :pyobject: Libelf.install
    :linenos:
 
@@ -4901,7 +4901,7 @@ the one passed to install, only the MPI implementations all set some
 additional properties on it to help you out.  E.g., in openmpi, you'll
 find this:
 
-.. literalinclude:: _spack_root/var/spack/repos/spack_repo/builtin/packages/openmpi/package.py
+.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/openmpi/package.py
    :pyobject: Openmpi.setup_dependent_package
 
 That code allows the ``openmpi`` package to associate an ``mpicc`` property
@@ -6001,16 +6001,16 @@ with those implemented in the package itself.
    * - Parent/Provider Package
      - Stand-alone Tests
    * - `C
-       <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/c>`_
+       <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/c>`_
      - Compiles ``hello.c`` and runs it
    * - `Cxx
-       <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/cxx>`_
+       <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/cxx>`_
      - Compiles and runs several ``hello`` programs
    * - `Fortran
-       <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/fortran>`_
+       <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/fortran>`_
      - Compiles and runs ``hello`` programs (``F`` and ``f90``)
    * - `Mpi
-       <https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/mpi>`_
+       <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/mpi>`_
      - Compiles and runs ``mpi_hello`` (``c``, ``fortran``)
    * - :ref:`PythonPackage <pythonpackage>`
      - Imports modules listed in the ``self.import_modules`` property with defaults derived from the tarball
@@ -6031,7 +6031,7 @@ maintainers provide additional stand-alone tests customized to the package.
 One example of a package that adds its own stand-alone tests to those
 "inherited" by the virtual package it provides an implementation for is
 the `Openmpi package
-<https://github.com/spack/spack/blob/develop/var/spack/repos/spack_repo/builtin/packages/openmpi/package.py>`_.
+<https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/openmpi/package.py>`_.
 
 Below are snippets from running and viewing the stand-alone test results
 for ``openmpi``:
@@ -6183,7 +6183,7 @@ running:
 
 .. code-block:: python
 
-   from spack.package import *
+   from spack import *
 
 This is already part of the boilerplate for packages created with
 ``spack create``.

lib/spack/docs/repositories.rst

@@ -9,7 +9,7 @@ Package Repositories (repos.yaml)
 =================================
 
 Spack comes with thousands of built-in package recipes in
-``var/spack/repos/spack_repo/builtin/``.  This is a **package repository** -- a
+``var/spack/repos/builtin/``.  This is a **package repository** -- a
 directory that Spack searches when it needs to find a package by name.
 You may need to maintain packages for restricted, proprietary or
 experimental software separately from the built-in repository. Spack
@@ -69,7 +69,7 @@ The default ``etc/spack/defaults/repos.yaml`` file looks like this:
 .. code-block:: yaml
 
   repos:
-  - $spack/var/spack/repos/spack_repo/builtin
+  - $spack/var/spack/repos/builtin
 
 The file starts with ``repos:`` and contains a single ordered list of
 paths to repositories. Each path is on a separate line starting with
@@ -78,16 +78,16 @@ paths to repositories. Each path is on a separate line starting with
 .. code-block:: yaml
 
   repos:
-  - /opt/repos/spack_repo/local_repo
-  - $spack/var/spack/repos/spack_repo/builtin
+  - /opt/local-repo
+  - $spack/var/spack/repos/builtin
 
 When Spack interprets a spec, e.g., ``mpich`` in ``spack install mpich``,
 it searches these repositories in order (first to last) to resolve each
 package name.  In this example, Spack will look for the following
 packages and use the first valid file:
 
-1. ``/opt/repos/spack_repo/local_repo/packages/mpich/package.py``
-2. ``$spack/var/spack/repos/spack_repo/builtin/packages/mpich/package.py``
+1. ``/opt/local-repo/packages/mpich/package.py``
+2. ``$spack/var/spack/repos/builtin/packages/mpich/package.py``
 
 .. note::
 
@@ -101,15 +101,14 @@ Namespaces
 
 Every repository in Spack has an associated **namespace** defined in its
 top-level ``repo.yaml`` file.  If you look at
-``var/spack/repos/spack_repo/builtin/repo.yaml`` in the built-in repository, you'll
+``var/spack/repos/builtin/repo.yaml`` in the built-in repository, you'll
 see that its namespace is ``builtin``:
 
 .. code-block:: console
 
-  $ cat var/spack/repos/spack_repo/builtin/repo.yaml
+  $ cat var/spack/repos/builtin/repo.yaml
   repo:
     namespace: builtin
-    api: v2.0
 
 Spack records the repository namespace of each installed package.  For
 example, if you install the ``mpich`` package from the ``builtin`` repo,
@@ -218,15 +217,15 @@ Suppose you have three repositories: the builtin Spack repo
 repo containing your own prototype packages (``proto``).  Suppose they
 contain packages as follows:
 
-  +--------------+-----------------------------------------------+-----------------------------+
-  | Namespace    | Path to repo                                  | Packages                    |
-  +==============+===============================================+=============================+
-  | ``proto``    | ``~/my_spack_repos/spack_repo/proto``         | ``mpich``                   |
-  +--------------+-----------------------------------------------+-----------------------------+
-  | ``llnl``     | ``/usr/local/repos/spack_repo/llnl``          | ``hdf5``                    |
-  +--------------+-----------------------------------------------+-----------------------------+
-  | ``builtin``  | ``$spack/var/spack/repos/spack_repo/builtin`` | ``mpich``, ``hdf5``, others |
-  +--------------+-----------------------------------------------+-----------------------------+
+  +--------------+------------------------------------+-----------------------------+
+  | Namespace    | Path to repo                       | Packages                    |
+  +==============+====================================+=============================+
+  | ``proto``    | ``~/proto``                        | ``mpich``                   |
+  +--------------+------------------------------------+-----------------------------+
+  | ``llnl``     | ``/usr/local/llnl``                | ``hdf5``                    |
+  +--------------+------------------------------------+-----------------------------+
+  | ``builtin``  | ``$spack/var/spack/repos/builtin`` | ``mpich``, ``hdf5``, others |
+  +--------------+------------------------------------+-----------------------------+
 
 Suppose that ``hdf5`` depends on ``mpich``.  You can override the
 built-in ``hdf5`` by adding the ``llnl`` repo to ``repos.yaml``:
@@ -234,8 +233,8 @@ built-in ``hdf5`` by adding the ``llnl`` repo to ``repos.yaml``:
 .. code-block:: yaml
 
    repos:
-   - /usr/local/repos/spack_repo/llnl
-   - $spack/var/spack/repos/spack_repo/builtin
+   - /usr/local/llnl
+   - $spack/var/spack/repos/builtin
 
 ``spack install hdf5`` will install ``llnl.hdf5 ^builtin.mpich``.
 
@@ -244,9 +243,9 @@ If, instead, ``repos.yaml`` looks like this:
 .. code-block:: yaml
 
    repos:
-   - ~/my_spack_repos/spack_repo/proto
-   - /usr/local/repos/spack_repo/llnl
-   - $spack/var/spack/repos/spack_repo/builtin
+   - ~/proto
+   - /usr/local/llnl
+   - $spack/var/spack/repos/builtin
 
 ``spack install hdf5`` will install ``llnl.hdf5 ^proto.mpich``.
 
@@ -327,8 +326,8 @@ files, use ``spack repo list``.
 
   $ spack repo list
   ==> 2 package repositories.
-  myrepo     v2.0    ~/my_spack_repos/spack_repo/myrepo
-  builtin    v2.0    ~/spack/var/spack/repos/spack_repo/builtin
+  myrepo     ~/myrepo
+  builtin    ~/spack/var/spack/repos/builtin
 
 Each repository is listed with its associated namespace.  To get the raw,
 merged YAML from all configuration files, use ``spack config get repos``:
@@ -336,9 +335,9 @@ merged YAML from all configuration files, use ``spack config get repos``:
 .. code-block:: console
 
    $ spack config get repos
-   repos:
-   - ~/my_spack_repos/spack_repo/myrepo
-   - $spack/var/spack/repos/spack_repo/builtin
+   repos:srepos:
+   - ~/myrepo
+   - $spack/var/spack/repos/builtin
 
 Note that, unlike ``spack repo list``, this does not include the
 namespace, which is read from each repo's ``repo.yaml``.
@@ -352,54 +351,66 @@ yourself; you can use the ``spack repo create`` command.
 
 .. code-block:: console
 
-  $ spack repo create ~/my_spack_repos myrepo
+  $ spack repo create myrepo
   ==> Created repo with namespace 'myrepo'.
   ==> To register it with spack, run this command:
-    spack repo add ~/my_spack_repos/spack_repo/myrepo
+    spack repo add ~/myrepo
 
-  $ ls ~/my_spack_repos/spack_repo/myrepo
+  $ ls myrepo
   packages/  repo.yaml
 
-  $ cat ~/my_spack_repos/spack_repo/myrepo/repo.yaml
+  $ cat myrepo/repo.yaml
   repo:
     namespace: 'myrepo'
-    api: v2.0
 
-Namespaces can also be nested, which can be useful if you have
-multiple package repositories for an organization. Spack will
-create the corresponding directory structure for you:
+By default, the namespace of a new repo matches its directory's name.
+You can supply a custom namespace with a second argument, e.g.:
 
 .. code-block:: console
 
-  $ spack repo create ~/my_spack_repos llnl.comp
+  $ spack repo create myrepo llnl.comp
   ==> Created repo with namespace 'llnl.comp'.
   ==> To register it with spack, run this command:
-    spack repo add ~/my_spack_repos/spack_repo/llnl/comp
+    spack repo add ~/myrepo
 
-
-  $ cat ~/my_spack_repos/spack_repo/llnl/comp/repo.yaml
+  $ cat myrepo/repo.yaml
   repo:
     namespace: 'llnl.comp'
-    api: v2.0
+
+You can also create repositories with custom structure with the ``-d/--subdirectory``
+argument, e.g.:
+
+.. code-block:: console
+
+  $ spack repo create -d applications myrepo apps
+  ==> Created repo with namespace 'apps'.
+  ==> To register it with Spack, run this command:
+    spack repo add ~/myrepo
+
+  $ ls myrepo
+  applications/  repo.yaml
+
+  $ cat myrepo/repo.yaml
+  repo:
+    namespace: apps
+    subdirectory: applications
 
 ^^^^^^^^^^^^^^^^^^
 ``spack repo add``
 ^^^^^^^^^^^^^^^^^^
 
 Once your repository is created, you can register it with Spack with
-``spack repo add``. You nee to specify the path to the directory that
-contains the ``repo.yaml`` file.
+``spack repo add``:
 
 .. code-block:: console
 
-   $ spack repo add ~/my_spack_repos/spack_repo/llnl/comp
+   $ spack repo add ./myrepo
    ==> Added repo with namespace 'llnl.comp'.
 
    $ spack repo list
    ==> 2 package repositories.
-  llnl.comp    v2.0    ~/my_spack_repos/spack_repo/llnl/comp
-  builtin      v2.0    ~/spack/var/spack/repos/spack_repo/builtin
-
+   llnl.comp    ~/myrepo
+   builtin      ~/spack/var/spack/repos/builtin
 
 This simply adds the repo to your ``repos.yaml`` file.
 
@@ -421,43 +432,46 @@ By namespace:
 .. code-block:: console
 
   $ spack repo rm llnl.comp
-  ==> Removed repository ~/my_spack_repos/spack_repo/llnl/comp with namespace 'llnl.comp'.
+  ==> Removed repository ~/myrepo with namespace 'llnl.comp'.
 
   $ spack repo list
   ==> 1 package repository.
-  builtin    ~/spack/var/spack/repos/spack_repo/builtin
+  builtin    ~/spack/var/spack/repos/builtin
 
 By path:
 
 .. code-block:: console
 
-  $ spack repo rm ~/my_spack_repos/spack_repo/llnl/comp
-  ==> Removed repository ~/my_spack_repos/spack_repo/llnl/comp
+  $ spack repo rm ~/myrepo
+  ==> Removed repository ~/myrepo
 
   $ spack repo list
   ==> 1 package repository.
-  builtin    ~/spack/var/spack/repos/spack_repo/builtin
+  builtin    ~/spack/var/spack/repos/builtin
 
 --------------------------------
 Repo namespaces and Python
 --------------------------------
 
-Package repositories are implemented as Python packages. To be precise,
-they are `namespace packages
-<https://packaging.python.org/en/latest/guides/packaging-namespace-packages/>`_
-with ``spack_repo`` the top-level namespace, followed by the repository
-namespace as submodules. For example, the builtin repository corresponds
-to the Python module ``spack_repo.builtin.packages``.
+You may have noticed that namespace notation for repositories is similar
+to the notation for namespaces in Python.  As it turns out, you *can*
+treat Spack repositories like Python packages; this is how they are
+implemented.
 
-This structure allows you to extend a ``builtin`` package in your own
+You could, for example, extend a ``builtin`` package in your own
 repository:
 
 .. code-block:: python
 
-   from spack_repo.builtin.packages.mpich.package import Mpich
+   from spack.pkg.builtin.mpich import Mpich
 
    class MyPackage(Mpich):
        ...
 
-Spack populates ``sys.path`` at runtime with the path to the root of your
-package repository's ``spack_repo`` directory.
+Spack repo namespaces are actually Python namespaces tacked on under
+``spack.pkg``.  The search semantics of ``repos.yaml`` are actually
+implemented using Python's built-in `sys.path
+<https://docs.python.org/2/library/sys.html#sys.path>`_ search.  The
+:py:mod:`spack.repo` module implements a custom `Python importer
+<https://docs.python.org/2/library/imp.html>`_.
+

lib/spack/docs/requirements.txt

@@ -5,9 +5,9 @@ sphinx-rtd-theme==3.0.2
 python-levenshtein==0.27.1
 docutils==0.21.2
 pygments==2.19.1
-urllib3==2.4.0
+urllib3==2.3.0
 pytest==8.3.5
 isort==6.0.1
 black==25.1.0
-flake8==7.2.0
+flake8==7.1.2
 mypy==1.11.1

lib/spack/docs/signing.rst

@@ -176,72 +176,92 @@ community without needing deep familiarity with GnuPG or Public Key
 Infrastructure.
 
 
-.. _build_cache_signing:
+.. _build_cache_format:
 
--------------------
-Build Cache Signing
--------------------
+------------------
+Build Cache Format
+------------------
 
-For an in-depth description of the layout of a binary mirror, see
-the :ref:`documentation<build_cache_layout>` covering binary caches. The
-key takeaway from that discussion that applies here is that the entry point
-to a binary package is it's manifest.  The manifest refers unambiguously to the
-spec metadata and compressed archive, which are stored as content-addressed
-blobs.
+A binary package consists of a metadata file unambiguously defining the
+built package (and including other details such as how to relocate it)
+and the installation directory of the package stored as a compressed
+archive file. The metadata files can either be unsigned, in which case
+the contents are simply the json-serialized concrete spec plus metadata,
+or they can be signed, in which case the json-serialized concrete spec
+plus metadata is wrapped in a gpg cleartext signature. Built package
+metadata files are named to indicate the operating system and
+architecture for which the package was built as well as the compiler
+used to build it and the packages name and version. For example::
 
-The manifest files can either be signed or unsigned, but are always given
-a name ending with ``.spec.manifest.json`` regardless. The difference between
-signed and unsigned manifests is simply that the signed version is wrapped in
-a gpg cleartext signature, as illustrated below::
+  linux-ubuntu18.04-haswell-gcc-7.5.0-zlib-1.2.12-llv2ysfdxnppzjrt5ldybb5c52qbmoow.spec.json.sig
+
+would contain the concrete spec and binary metadata for a binary package
+of ``zlib@1.2.12``, built for the ``ubuntu`` operating system and ``haswell``
+architecture. The id of the built package exists in the name of the file
+as well (after the package name and version) and in this case begins
+with ``llv2ys``. The id distinguishes a particular built package from all
+other built packages with the same os/arch, compiler, name, and version.
+Below is an example of a signed binary package metadata file. Such a
+file would live in the ``build_cache`` directory of a binary mirror::
 
   -----BEGIN PGP SIGNED MESSAGE-----
   Hash: SHA512
 
   {
-    "version": 3,
-    "data": [
-      {
-        "contentLength": 10731083,
-        "mediaType": "application/vnd.spack.install.v2.tar+gzip",
-        "compression": "gzip",
-        "checksumAlgorithm": "sha256",
-        "checksum": "0f24aa6b5dd7150067349865217acd3f6a383083f9eca111d2d2fed726c88210"
-      },
-      {
-        "contentLength": 1000,
-        "mediaType": "application/vnd.spack.spec.v5+json",
-        "compression": "gzip",
-        "checksumAlgorithm": "sha256",
-        "checksum": "fba751c4796536737c9acbb718dad7429be1fa485f5585d450ab8b25d12ae041"
-      }
-    ]
-  }
-  -----BEGIN PGP SIGNATURE-----
+    "spec": {
+      <concrete-spec-contents-omitted>
+    },
 
-  iQGzBAEBCgAdFiEEdbwFKBFJCcB24mB0GAEP+tc8mwcFAmf2rr4ACgkQGAEP+tc8
-  mwfefwv+KJs8MsQ5ovFaBdmyx5H/3k4rO4QHBzuSPOB6UaxErA9IyOB31iP6vNTU
-  HzYpxz6F5dJCJWmmNEMN/0+vjhMHEOkqd7M1l5reVcxduTF2yc4tBZUO2gienEHL
-  W0e+SnUznl1yc/aVpChUiahO2zToCsI8HZRNT4tu6iCnE/OpghqjsSdBOZHmSNDD
-  5wuuCxfDUyWI6ZlLclaaB7RdbCUUJf/iqi711J+wubvnDFhc6Ynwm1xai5laJ1bD
-  ev3NrSb2AAroeNFVo4iECA0fZC1OZQYzaRmAEhBXtCideGJ5Zf2Cp9hmCwNK8Hq6
-  bNt94JP9LqC3FCCJJOMsPyOOhMSA5MU44zyyzloRwEQpHHLuFzVdbTHA3dmTc18n
-  HxNLkZoEMYRc8zNr40g0yb2lCbc+P11TtL1E+5NlE34MX15mPewRCiIFTMwhCnE3
-  gFSKtW1MKustZE35/RUwd2mpJRf+mSRVCl1f1RiFjktLjz7vWQq7imIUSam0fPDr
-  XD4aDogm
-  =RrFX
+    "buildcache_layout_version": 1,
+    "binary_cache_checksum": {
+      "hash_algorithm": "sha256",
+      "hash": "4f1e46452c35a5e61bcacca205bae1bfcd60a83a399af201a29c95b7cc3e1423"
+     }
+  }
+
+  -----BEGIN PGP SIGNATURE-----
+  iQGzBAEBCgAdFiEETZn0sLle8jIrdAPLx/P+voVcifMFAmKAGvwACgkQx/P+voVc
+  ifNoVgv/VrhA+wurVs5GB9PhmMA1m5U/AfXZb4BElDRwpT8ZcTPIv5X8xtv60eyn
+  4EOneGVbZoMThVxgev/NKARorGmhFXRqhWf+jknJZ1dicpqn/qpv34rELKUpgXU+
+  QDQ4d1P64AIdTczXe2GI9ZvhOo6+bPvK7LIsTkBbtWmopkomVxF0LcMuxAVIbA6b
+  887yBvVO0VGlqRnkDW7nXx49r3AG2+wDcoU1f8ep8QtjOcMNaPTPJ0UnjD0VQGW6
+  4ZFaGZWzdo45MY6tF3o5mqM7zJkVobpoW3iUz6J5tjz7H/nMlGgMkUwY9Kxp2PVH
+  qoj6Zip3LWplnl2OZyAY+vflPFdFh12Xpk4FG7Sxm/ux0r+l8tCAPvtw+G38a5P7
+  QEk2JBr8qMGKASmnRlJUkm1vwz0a95IF3S9YDfTAA2vz6HH3PtsNLFhtorfx8eBi
+  Wn5aPJAGEPOawEOvXGGbsH4cDEKPeN0n6cy1k92uPEmBLDVsdnur8q42jk5c2Qyx
+  j3DXty57
+  =3gvm
   -----END PGP SIGNATURE-----
 
 If a user has trusted the public key associated with the private key
-used to sign the above manifest file, the signature can be verified with
+used to sign the above spec file, the signature can be verified with
 gpg, as follows::
 
-  $ gpg --verify gcc-runtime-12.3.0-s2nqujezsce4x6uhtvxscu7jhewqzztx.spec.manifest.json
+  $ gpg –verify linux-ubuntu18.04-haswell-gcc-7.5.0-zlib-1.2.12-llv2ysfdxnppzjrt5ldybb5c52qbmoow.spec.json.sig
 
-When attempting to install a binary package that has been signed, spack will
-attempt to verify the signature with one of the trusted keys in its keyring,
-and will fail if unable to do so.  While not recommended, it is possible to
-force installation of a signed package without verification by providing the
-``--no-check-signature`` argument to ``spack install ...``.
+The metadata (regardless whether signed or unsigned) contains the checksum
+of the ``.spack`` file containing the actual installation. The checksum should
+be compared to a checksum computed locally on the ``.spack`` file to ensure the
+contents have not changed since the binary spec plus metadata were signed. The
+``.spack`` files are actually tarballs containing the compressed archive of the
+install tree.  These files, along with the metadata files, live within the
+``build_cache`` directory of the mirror, and together are organized as follows::
+
+  build_cache/
+    # unsigned metadata (for indexing, contains sha256 of .spack file)
+    <arch>-<compiler>-<name>-<ver>-24zvipcqgg2wyjpvdq2ajy5jnm564hen.spec.json
+    # clearsigned metadata (same as above, but signed)
+    <arch>-<compiler>-<name>-<ver>-24zvipcqgg2wyjpvdq2ajy5jnm564hen.spec.json.sig
+    <arch>/
+      <compiler>/
+        <name>-<ver>/
+          # tar.gz-compressed prefix (may support more compression formats later)
+          <arch>-<compiler>-<name>-<ver>-24zvipcqgg2wyjpvdq2ajy5jnm564hen.spack
+
+Uncompressing and extracting the ``.spack`` file results in the install tree.
+This is in contrast to previous versions of spack, where the ``.spack`` file
+contained a (duplicated) metadata file, a signature file and a nested tarball
+containing the install tree.
 
 .. _internal_implementation:
 
@@ -300,10 +320,10 @@ the following way:
    Reputational Public Key are imported into a keyring by the ``spack gpg …``
    sub-command. This is initiated by the job’s build script which is created by
    the generate job at the beginning of the pipeline.
-4. Assuming the package has dependencies those spec manifests are verified using
+4. Assuming the package has dependencies those specs are verified using
    the keyring.
-5. The package is built and the spec manifest is generated
-6. The spec manifest is signed by the keyring and uploaded to the mirror’s
+5. The package is built and the spec.json is generated
+6. The spec.json is signed by the keyring and uploaded to the mirror’s
    build cache.
 
 **Reputational Key**
@@ -356,24 +376,24 @@ following way:
 4.  In addition to the secret, the runner creates a tmpfs memory mounted
     directory where the GnuPG keyring will be created to verify, and
     then resign the package specs.
-5.  The job script syncs all spec manifest files from the build cache to
+5.  The job script syncs all spec.json.sig files from the build cache to
     a working directory in the job’s execution environment.
 6.  The job script then runs the ``sign.sh`` script built into the
     notary Docker image.
 7.  The ``sign.sh`` script imports the public components of the
     Reputational and Intermediate CI Keys and uses them to verify good
-    signatures on the spec.manifest.json files. If any signed manifest
-    does not verify, the job immediately fails.
-8.  Assuming all manifests are verified, the ``sign.sh`` script then unpacks
-    the manifest json data from the signed file in preparation for being
+    signatures on the spec.json.sig files. If any signed spec does not
+    verify the job immediately fails.
+8.  Assuming all specs are verified, the ``sign.sh`` script then unpacks
+    the spec json data from the signed file in preparation for being
     re-signed with the Reputational Key.
 9.  The private components of the Reputational Key are decrypted to
     standard out using ``aws-encryption-cli`` directly into a ``gpg
     –import …`` statement which imports the key into the
     keyring mounted in-memory.
-10. The private key is then used to sign each of the manifests and the
+10. The private key is then used to sign each of the json specs and the
     keyring is removed from disk.
-11. The re-signed manifests are resynced to the AWS S3 Mirror and the
+11. The re-signed json specs are resynced to the AWS S3 Mirror and the
     public signing of the packages for the develop or release pipeline
     that created them is complete.
 

lib/spack/external/__init__.py

@@ -11,7 +11,6 @@
 * Homepage: https://altgraph.readthedocs.io/en/latest/index.html
 * Usage: dependency of macholib
 * Version: 0.17.3
-* License: MIT
 
 archspec
 --------
@@ -19,7 +18,6 @@
 * Homepage: https://pypi.python.org/pypi/archspec
 * Usage: Labeling, comparison and detection of microarchitectures
 * Version: 0.2.5 (commit 38ce485258ffc4fc6dd6688f8dc90cb269478c47)
-* License: Apache-2.0 or MIT
 
 astunparse
 ----------------
@@ -27,7 +25,6 @@
 * Homepage: https://github.com/simonpercivall/astunparse
 * Usage: Unparsing Python ASTs for package hashes in Spack
 * Version: 1.6.3 (plus modifications)
-* License: PSF-2.0
 * Note: This is in ``spack.util.unparse`` because it's very heavily
   modified, and we want to track coverage for it.
   Specifically, we have modified this library to generate consistent unparsed ASTs
@@ -44,7 +41,6 @@
 * Homepage: https://github.com/python-attrs/attrs
 * Usage: Needed by jsonschema.
 * Version: 22.1.0
-* License: MIT
 
 ctest_log_parser
 ----------------
@@ -52,7 +48,6 @@
 * Homepage: https://github.com/Kitware/CMake/blob/master/Source/CTest/cmCTestBuildHandler.cxx
 * Usage: Functions to parse build logs and extract error messages.
 * Version: Unversioned
-* License: BSD-3-Clause
 * Note: This is a homemade port of Kitware's CTest build handler.
 
 distro
@@ -61,7 +56,6 @@
 * Homepage: https://pypi.python.org/pypi/distro
 * Usage: Provides a more stable linux distribution detection.
 * Version: 1.8.0
-* License: Apache-2.0
 
 jinja2
 ------
@@ -69,7 +63,6 @@
 * Homepage: https://pypi.python.org/pypi/Jinja2
 * Usage: A modern and designer-friendly templating language for Python.
 * Version: 3.0.3 (last version supporting Python 3.6)
-* License: BSD-3-Clause
 
 jsonschema
 ----------
@@ -77,7 +70,6 @@
 * Homepage: https://pypi.python.org/pypi/jsonschema
 * Usage: An implementation of JSON Schema for Python.
 * Version: 3.2.0 (last version before 2.7 and 3.6 support was dropped)
-* License: MIT
 * Note: We don't include tests or benchmarks; just what Spack needs.
 
 macholib
@@ -86,7 +78,6 @@
 * Homepage: https://macholib.readthedocs.io/en/latest/index.html#
 * Usage: Manipulation of Mach-o binaries for relocating macOS buildcaches on Linux
 * Version: 1.16.2
-* License: MIT
 
 markupsafe
 ----------
@@ -94,7 +85,6 @@
 * Homepage: https://pypi.python.org/pypi/MarkupSafe
 * Usage: Implements a XML/HTML/XHTML Markup safe string for Python.
 * Version: 2.0.1 (last version supporting Python 3.6)
-* License: BSD-3-Clause
 
 pyrsistent
 ----------
@@ -102,7 +92,6 @@
 * Homepage: http://github.com/tobgu/pyrsistent/
 * Usage: Needed by `jsonschema`
 * Version: 0.18.0
-* License: MIT
 
 ruamel.yaml
 ------
@@ -112,7 +101,6 @@
   actively maintained and has more features, including round-tripping
   comments read from config files.
 * Version: 0.17.21
-* License: MIT
 
 six
 ---
@@ -120,6 +108,5 @@
 * Homepage: https://pypi.python.org/pypi/six
 * Usage: Python 2 and 3 compatibility utilities.
 * Version: 1.16.0
-* License: MIT
 
 """

lib/spack/llnl/util/filesystem.py

@@ -764,7 +764,7 @@ def copy_tree(
 
     files = glob.glob(src)
     if not files:
-        raise OSError("No such file or directory: '{0}'".format(src), errno.ENOENT)
+        raise OSError("No such file or directory: '{0}'".format(src))
 
     # For Windows hard-links and junctions, the source path must exist to make a symlink. Add
     # all symlinks to this list while traversing the tree, then when finished, make all

lib/spack/llnl/util/lang.py

@@ -15,20 +15,7 @@
 import typing
 import warnings
 from datetime import datetime, timedelta
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    Generic,
-    Iterable,
-    Iterator,
-    List,
-    Mapping,
-    Optional,
-    Tuple,
-    TypeVar,
-    Union,
-)
+from typing import Callable, Dict, Iterable, List, Mapping, Optional, Tuple, TypeVar
 
 # Ignore emacs backups when listing modules
 ignore_modules = r"^\.#|~$"
@@ -437,39 +424,46 @@ def add_func_to_class(name, func):
     return cls
 
 
-K = TypeVar("K")
-V = TypeVar("V")
-
-
 @lazy_lexicographic_ordering
-class HashableMap(typing.MutableMapping[K, V]):
+class HashableMap(collections.abc.MutableMapping):
     """This is a hashable, comparable dictionary.  Hash is performed on
     a tuple of the values in the dictionary."""
 
     __slots__ = ("dict",)
 
     def __init__(self):
-        self.dict: Dict[K, V] = {}
+        self.dict = {}
 
-    def __getitem__(self, key: K) -> V:
+    def __getitem__(self, key):
         return self.dict[key]
 
-    def __setitem__(self, key: K, value: V) -> None:
+    def __setitem__(self, key, value):
         self.dict[key] = value
 
-    def __iter__(self) -> Iterator[K]:
+    def __iter__(self):
         return iter(self.dict)
 
-    def __len__(self) -> int:
+    def __len__(self):
         return len(self.dict)
 
-    def __delitem__(self, key: K) -> None:
+    def __delitem__(self, key):
         del self.dict[key]
 
     def _cmp_iter(self):
         for _, v in sorted(self.items()):
             yield v
 
+    def copy(self):
+        """Type-agnostic clone method.  Preserves subclass type."""
+        # Construct a new dict of my type
+        self_type = type(self)
+        clone = self_type()
+
+        # Copy everything from this dict into it.
+        for key in self:
+            clone[key] = self[key].copy()
+        return clone
+
 
 def match_predicate(*args):
     """Utility function for making string matching predicates.
@@ -1053,28 +1047,19 @@ def __exit__(self, exc_type, exc_value, tb):
         return True
 
 
-ClassPropertyType = TypeVar("ClassPropertyType")
-
-
-class classproperty(Generic[ClassPropertyType]):
+class classproperty:
     """Non-data descriptor to evaluate a class-level property. The function that performs
-    the evaluation is injected at creation time and takes an owner (i.e., the class that
-    originated the instance).
+    the evaluation is injected at creation time and take an instance (could be None) and
+    an owner (i.e. the class that originated the instance)
     """
 
-    def __init__(self, callback: Callable[[Any], ClassPropertyType]) -> None:
+    def __init__(self, callback):
         self.callback = callback
 
-    def __get__(self, instance, owner) -> ClassPropertyType:
+    def __get__(self, instance, owner):
         return self.callback(owner)
 
 
-#: A type alias that represents either a classproperty descriptor or a constant value of the same
-#: type. This allows derived classes to override a computed class-level property with a constant
-#: value while retaining type compatibility.
-ClassProperty = Union[ClassPropertyType, classproperty[ClassPropertyType]]
-
-
 class DeprecatedProperty:
     """Data descriptor to error or warn when a deprecated property is accessed.
 

lib/spack/spack/__init__.py

@@ -18,7 +18,7 @@
 #: version is incremented when the package API is extended in a backwards-compatible way. The major
 #: version is incremented upon breaking changes. This version is changed independently from the
 #: Spack version.
-package_api_version = (2, 0)
+package_api_version = (1, 0)
 
 #: The minimum Package API version that this version of Spack is compatible with. This should
 #: always be a tuple of the form ``(major, 0)``, since compatibility with vX.Y implies

lib/spack/spack/aliases.py

@@ -7,7 +7,7 @@
     "llvm": "clang",
     "intel-oneapi-compilers": "oneapi",
     "llvm-amdgpu": "rocmcc",
-    "intel-oneapi-compilers-classic": "intel",
+    "intel-oneapi-compiler-classic": "intel",
     "acfl": "arm",
 }
 
@@ -15,6 +15,6 @@
     "clang": "llvm",
     "oneapi": "intel-oneapi-compilers",
     "rocmcc": "llvm-amdgpu",
-    "intel": "intel-oneapi-compilers-classic",
+    "intel": "intel-oneapi-compiler-classic",
     "arm": "acfl",
 }

lib/spack/spack/audit.py

@@ -350,7 +350,7 @@ def _ensure_no_folders_without_package_py(error_cls):
     for repository in spack.repo.PATH.repos:
         missing = []
         for entry in os.scandir(repository.packages_path):
-            if not entry.is_dir() or entry.name == "__pycache__":
+            if not entry.is_dir():
                 continue
             package_py = pathlib.Path(entry.path) / spack.repo.package_file_name
             if not package_py.exists():

lib/spack/spack/bootstrap/environment.py

@@ -133,7 +133,7 @@ def mypy_root_spec() -> str:
 
 def black_root_spec() -> str:
     """Return the root spec used to bootstrap black"""
-    return _root_spec("py-black@:25.1.0")
+    return _root_spec("py-black@:24.1.0")
 
 
 def flake8_root_spec() -> str:

lib/spack/spack/build_environment.py

@@ -36,11 +36,9 @@
 import multiprocessing
 import os
 import re
-import signal
 import sys
 import traceback
 import types
-import warnings
 from collections import defaultdict
 from enum import Flag, auto
 from itertools import chain
@@ -574,10 +572,12 @@ def set_package_py_globals(pkg, context: Context = Context.BUILD):
     module.make = DeprecatedExecutable(pkg.name, "make", "gmake")
     module.gmake = DeprecatedExecutable(pkg.name, "gmake", "gmake")
     module.ninja = DeprecatedExecutable(pkg.name, "ninja", "ninja")
-
+    # TODO: johnwparent: add package or builder support to define these build tools
+    # for now there is no entrypoint for builders to define these on their
+    # own
     if sys.platform == "win32":
-        module.nmake = DeprecatedExecutable(pkg.name, "nmake", "msvc")
-        module.msbuild = DeprecatedExecutable(pkg.name, "msbuild", "msvc")
+        module.nmake = Executable("nmake")
+        module.msbuild = Executable("msbuild")
         # analog to configure for win32
         module.cscript = Executable("cscript")
 
@@ -1189,9 +1189,11 @@ def _setup_pkg_and_run(
         if isinstance(e, (spack.multimethod.NoSuchMethodError, AttributeError)):
             process = "test the installation" if context == "test" else "build from sources"
             error_msg = (
-                "The '{}' package cannot find an attribute while trying to {}. You can fix this "
-                "by updating the {} recipe, and you can also report the issue as a build-error or "
-                "a bug at https://github.com/spack/spack/issues"
+                "The '{}' package cannot find an attribute while trying to {}. "
+                "This might be due to a change in Spack's package format "
+                "to support multiple build-systems for a single package. You can fix this "
+                "by updating the {} recipe, and you can also report the issue as a bug. "
+                "More information at https://spack.readthedocs.io/en/latest/packaging_guide.html#installation-procedure"
             ).format(pkg.name, process, context)
             error_msg = colorize("@*R{{{}}}".format(error_msg))
             error_msg = "{}\n\n{}".format(str(e), error_msg)
@@ -1216,45 +1218,15 @@ def _setup_pkg_and_run(
             input_pipe.close()
 
 
-class BuildProcess:
-    def __init__(self, *, target, args) -> None:
-        self.p = multiprocessing.Process(target=target, args=args)
-
-    def start(self) -> None:
-        self.p.start()
-
-    def is_alive(self) -> bool:
-        return self.p.is_alive()
-
-    def join(self, *, timeout: Optional[int] = None):
-        self.p.join(timeout=timeout)
-
-    def terminate(self):
-        # Opportunity for graceful termination
-        self.p.terminate()
-        self.p.join(timeout=1)
-
-        # If the process didn't gracefully terminate, forcefully kill
-        if self.p.is_alive():
-            # TODO (python 3.6 removal): use self.p.kill() instead, consider removing this class
-            assert isinstance(self.p.pid, int), f"unexpected value for PID: {self.p.pid}"
-            os.kill(self.p.pid, signal.SIGKILL)
-            self.p.join()
-
-    @property
-    def exitcode(self):
-        return self.p.exitcode
-
-
-def start_build_process(pkg, function, kwargs, *, timeout: Optional[int] = None):
+def start_build_process(pkg, function, kwargs):
     """Create a child process to do part of a spack build.
 
     Args:
 
         pkg (spack.package_base.PackageBase): package whose environment we should set up the
             child process for.
-        function (typing.Callable): argless function to run in the child process.
-        timeout: maximum time allowed to finish the execution of function
+        function (typing.Callable): argless function to run in the child
+            process.
 
     Usage::
 
@@ -1282,14 +1254,14 @@ def child_fun():
         # Forward sys.stdin when appropriate, to allow toggling verbosity
         if sys.platform != "win32" and sys.stdin.isatty() and hasattr(sys.stdin, "fileno"):
             input_fd = Connection(os.dup(sys.stdin.fileno()))
-        mflags = os.environ.get("MAKEFLAGS")
-        if mflags is not None:
+        mflags = os.environ.get("MAKEFLAGS", False)
+        if mflags:
             m = re.search(r"--jobserver-[^=]*=(\d),(\d)", mflags)
             if m:
                 jobserver_fd1 = Connection(int(m.group(1)))
                 jobserver_fd2 = Connection(int(m.group(2)))
 
-        p = BuildProcess(
+        p = multiprocessing.Process(
             target=_setup_pkg_and_run,
             args=(
                 serialized_pkg,
@@ -1323,17 +1295,14 @@ def exitcode_msg(p):
         typ = "exit" if p.exitcode >= 0 else "signal"
         return f"{typ} {abs(p.exitcode)}"
 
-    p.join(timeout=timeout)
-    if p.is_alive():
-        warnings.warn(f"Terminating process, since the timeout of {timeout}s was exceeded")
-        p.terminate()
-        p.join()
-
     try:
         child_result = read_pipe.recv()
     except EOFError:
+        p.join()
         raise InstallError(f"The process has stopped unexpectedly ({exitcode_msg(p)})")
 
+    p.join()
+
     # If returns a StopPhase, raise it
     if isinstance(child_result, spack.error.StopPhase):
         # do not print

lib/spack/spack/build_systems/autotools.py

@@ -16,7 +16,6 @@
 import spack.package_base
 import spack.phase_callbacks
 import spack.spec
-import spack.util.environment
 import spack.util.prefix
 from spack.directives import build_system, conflicts, depends_on
 from spack.multimethod import when
@@ -847,9 +846,7 @@ def _remove_libtool_archives(self) -> None:
             with open(self._removed_la_files_log, mode="w", encoding="utf-8") as f:
                 f.write("\n".join(libtool_files))
 
-    def setup_build_environment(
-        self, env: spack.util.environment.EnvironmentModifications
-    ) -> None:
+    def setup_build_environment(self, env):
         if self.spec.platform == "darwin" and macos_version() >= Version("11"):
             # Many configure files rely on matching '10.*' for macOS version
             # detection and fail to add flags if it shows as version 11.

lib/spack/spack/build_systems/cargo.py

@@ -8,7 +8,6 @@
 import spack.package_base
 import spack.phase_callbacks
 import spack.spec
-import spack.util.environment
 import spack.util.prefix
 from spack.directives import build_system, depends_on
 from spack.multimethod import when
@@ -87,9 +86,7 @@ def check_args(self):
         """Argument for ``cargo test`` during check phase"""
         return []
 
-    def setup_build_environment(
-        self, env: spack.util.environment.EnvironmentModifications
-    ) -> None:
+    def setup_build_environment(self, env):
         env.set("CARGO_HOME", self.stage.path)
 
     def build(

lib/spack/spack/build_systems/compiler.py

@@ -36,7 +36,7 @@ class CompilerPackage(spack.package_base.PackageBase):
 
     #: Compiler argument(s) that produces version information
     #: If multiple arguments, the earlier arguments must produce errors when invalid
-    compiler_version_argument: Union[str, Tuple[str, ...]] = "-dumpversion"
+    compiler_version_argument: Union[str, Tuple[str]] = "-dumpversion"
 
     #: Regex used to extract version from compiler's output
     compiler_version_regex: str = "(.*)"
@@ -47,11 +47,6 @@ class CompilerPackage(spack.package_base.PackageBase):
     #: Relative path to compiler wrappers
     compiler_wrapper_link_paths: Dict[str, str] = {}
 
-    #: Optimization flags
-    opt_flags: Sequence[str] = []
-    #: Flags for generating debug information
-    debug_flags: Sequence[str] = []
-
     def __init__(self, spec: "spack.spec.Spec"):
         super().__init__(spec)
         msg = f"Supported languages for {spec} are not a subset of possible supported languages"

lib/spack/spack/build_systems/go.py

@@ -8,7 +8,6 @@
 import spack.package_base
 import spack.phase_callbacks
 import spack.spec
-import spack.util.environment
 import spack.util.prefix
 from spack.directives import build_system, depends_on
 from spack.multimethod import when
@@ -69,9 +68,7 @@ class GoBuilder(BuilderWithDefaults):
     #: Callback names for install-time test
     install_time_test_callbacks = ["check"]
 
-    def setup_build_environment(
-        self, env: spack.util.environment.EnvironmentModifications
-    ) -> None:
+    def setup_build_environment(self, env):
         env.set("GO111MODULE", "on")
         env.set("GOTOOLCHAIN", "local")
         env.set("GOPATH", fs.join_path(self.pkg.stage.path, "go"))

lib/spack/spack/build_systems/intel.py

@@ -23,7 +23,6 @@
 
 import spack.error
 import spack.phase_callbacks
-import spack.spec
 from spack.build_environment import dso_suffix
 from spack.error import InstallError
 from spack.util.environment import EnvironmentModifications
@@ -1017,7 +1016,7 @@ def libs(self):
         debug_print(result)
         return result
 
-    def setup_run_environment(self, env: EnvironmentModifications) -> None:
+    def setup_run_environment(self, env):
         """Adds environment variables to the generated module file.
 
         These environment variables come from running:
@@ -1050,13 +1049,11 @@ def setup_run_environment(self, env: EnvironmentModifications) -> None:
             env.set("F77", self.prefix.bin.ifort)
             env.set("F90", self.prefix.bin.ifort)
 
-    def setup_dependent_build_environment(
-        self, env: EnvironmentModifications, dependent_spec: spack.spec.Spec
-    ) -> None:
+    def setup_dependent_build_environment(self, env, dependent_spec):
         # NB: This function is overwritten by 'mpi' provider packages:
         #
-        # var/spack/repos/spack_repo/builtin/packages/intel_mpi/package.py
-        # var/spack/repos/spack_repo/builtin/packages/intel_parallel_studio/package.py
+        # var/spack/repos/builtin/packages/intel-mpi/package.py
+        # var/spack/repos/builtin/packages/intel-parallel-studio/package.py
         #
         # They call _setup_dependent_env_callback() as well, but with the
         # dictionary kwarg compilers_of_client{} present and populated.
@@ -1064,12 +1061,7 @@ def setup_dependent_build_environment(
         # Handle everything in a callback version.
         self._setup_dependent_env_callback(env, dependent_spec)
 
-    def _setup_dependent_env_callback(
-        self,
-        env: EnvironmentModifications,
-        dependent_spec: spack.spec.Spec,
-        compilers_of_client={},
-    ) -> None:
+    def _setup_dependent_env_callback(self, env, dependent_spec, compilers_of_client={}):
         # Expected to be called from a client's
         # setup_dependent_build_environment(),
         # with args extended to convey the client's compilers as needed.

lib/spack/spack/build_systems/lua.py

@@ -8,7 +8,6 @@
 import spack.builder
 import spack.package_base
 import spack.spec
-import spack.util.environment
 import spack.util.executable
 import spack.util.prefix
 from spack.directives import build_system, depends_on, extends
@@ -115,7 +114,5 @@ def install(
     def _luarocks_config_path(self):
         return os.path.join(self.pkg.stage.source_path, "spack_luarocks.lua")
 
-    def setup_build_environment(
-        self, env: spack.util.environment.EnvironmentModifications
-    ) -> None:
+    def setup_build_environment(self, env):
         env.set("LUAROCKS_CONFIG", self._luarocks_config_path())

lib/spack/spack/build_systems/octave.py

@@ -4,7 +4,6 @@
 import spack.builder
 import spack.package_base
 import spack.spec
-import spack.util.environment
 import spack.util.prefix
 from spack.directives import build_system, extends
 from spack.multimethod import when
@@ -58,9 +57,7 @@ def install(
             "pkg prefix %s; pkg install %s" % (prefix, self.pkg.stage.archive_file),
         )
 
-    def setup_build_environment(
-        self, env: spack.util.environment.EnvironmentModifications
-    ) -> None:
+    def setup_build_environment(self, env):
         # octave does not like those environment variables to be set:
         env.unset("CC")
         env.unset("CXX")

lib/spack/spack/build_systems/oneapi.py

@@ -106,8 +106,8 @@ def install_component(self, installer_path):
 
             bash = Executable("bash")
 
-            # Installer writes files in ~/intel set HOME so it goes to staging directory
-            bash.add_default_env("HOME", join_path(self.stage.path, "home"))
+            # Installer writes files in ~/intel set HOME so it goes to prefix
+            bash.add_default_env("HOME", self.prefix)
             # Installer checks $XDG_RUNTIME_DIR/.bootstrapper_lock_file as well
             bash.add_default_env("XDG_RUNTIME_DIR", join_path(self.stage.path, "runtime"))
 
@@ -132,7 +132,7 @@ def install_component(self, installer_path):
         if not isdir(install_dir):
             raise RuntimeError("install failed to directory: {0}".format(install_dir))
 
-    def setup_run_environment(self, env: EnvironmentModifications) -> None:
+    def setup_run_environment(self, env):
         """Adds environment variables to the generated module file.
 
         These environment variables come from running:

lib/spack/spack/build_systems/python.py

@@ -13,9 +13,9 @@
 import archspec
 
 import llnl.util.filesystem as fs
+import llnl.util.lang as lang
 import llnl.util.tty as tty
 from llnl.util.filesystem import HeaderList, LibraryList, join_path
-from llnl.util.lang import ClassProperty, classproperty, match_predicate
 
 import spack.builder
 import spack.config
@@ -139,7 +139,7 @@ def view_file_conflicts(self, view, merge_map):
             ext_map = view.extensions_layout.extension_map(self.extendee_spec)
             namespaces = set(x.package.py_namespace for x in ext_map.values())
             namespace_re = r"site-packages/{0}/__init__.py".format(self.py_namespace)
-            find_namespace = match_predicate(namespace_re)
+            find_namespace = lang.match_predicate(namespace_re)
             if self.py_namespace in namespaces:
                 conflicts = list(x for x in conflicts if not find_namespace(x))
 
@@ -206,7 +206,7 @@ def remove_files_from_view(self, view, merge_map):
                 spec.package.py_namespace for name, spec in ext_map.items() if name != self.name
             )
             if self.py_namespace in remaining_namespaces:
-                namespace_init = match_predicate(
+                namespace_init = lang.match_predicate(
                     r"site-packages/{0}/__init__.py".format(self.py_namespace)
                 )
                 ignore_namespace = True
@@ -324,27 +324,6 @@ def get_external_python_for_prefix(self):
         raise StopIteration("No external python could be detected for %s to depend on" % self.spec)
 
 
-def _homepage(cls: "PythonPackage") -> Optional[str]:
-    """Get the homepage from PyPI if available."""
-    if cls.pypi:
-        name = cls.pypi.split("/")[0]
-        return f"https://pypi.org/project/{name}/"
-    return None
-
-
-def _url(cls: "PythonPackage") -> Optional[str]:
-    if cls.pypi:
-        return f"https://files.pythonhosted.org/packages/source/{cls.pypi[0]}/{cls.pypi}"
-    return None
-
-
-def _list_url(cls: "PythonPackage") -> Optional[str]:
-    if cls.pypi:
-        name = cls.pypi.split("/")[0]
-        return f"https://pypi.org/simple/{name}/"
-    return None
-
-
 class PythonPackage(PythonExtension):
     """Specialized class for packages that are built using pip."""
 
@@ -372,9 +351,25 @@ class PythonPackage(PythonExtension):
 
     py_namespace: Optional[str] = None
 
-    homepage: ClassProperty[Optional[str]] = classproperty(_homepage)
-    url: ClassProperty[Optional[str]] = classproperty(_url)
-    list_url: ClassProperty[Optional[str]] = classproperty(_list_url)
+    @lang.classproperty
+    def homepage(cls) -> Optional[str]:  # type: ignore[override]
+        if cls.pypi:
+            name = cls.pypi.split("/")[0]
+            return f"https://pypi.org/project/{name}/"
+        return None
+
+    @lang.classproperty
+    def url(cls) -> Optional[str]:
+        if cls.pypi:
+            return f"https://files.pythonhosted.org/packages/source/{cls.pypi[0]}/{cls.pypi}"
+        return None
+
+    @lang.classproperty
+    def list_url(cls) -> Optional[str]:  # type: ignore[override]
+        if cls.pypi:
+            name = cls.pypi.split("/")[0]
+            return f"https://pypi.org/simple/{name}/"
+        return None
 
     @property
     def python_spec(self) -> Spec:

lib/spack/spack/build_systems/r.py

@@ -3,8 +3,8 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 from typing import Optional, Tuple
 
+import llnl.util.lang as lang
 from llnl.util.filesystem import mkdirp
-from llnl.util.lang import ClassProperty, classproperty
 
 from spack.directives import extends
 
@@ -54,32 +54,6 @@ def install(self, pkg, spec, prefix):
         pkg.module.R(*args)
 
 
-def _homepage(cls: "RPackage") -> Optional[str]:
-    if cls.cran:
-        return f"https://cloud.r-project.org/package={cls.cran}"
-    elif cls.bioc:
-        return f"https://bioconductor.org/packages/{cls.bioc}"
-    return None
-
-
-def _url(cls: "RPackage") -> Optional[str]:
-    if cls.cran:
-        return f"https://cloud.r-project.org/src/contrib/{cls.cran}_{str(list(cls.versions)[0])}.tar.gz"
-    return None
-
-
-def _list_url(cls: "RPackage") -> Optional[str]:
-    if cls.cran:
-        return f"https://cloud.r-project.org/src/contrib/Archive/{cls.cran}/"
-    return None
-
-
-def _git(cls: "RPackage") -> Optional[str]:
-    if cls.bioc:
-        return f"https://git.bioconductor.org/packages/{cls.bioc}"
-    return None
-
-
 class RPackage(Package):
     """Specialized class for packages that are built using R.
 
@@ -103,7 +77,24 @@ class RPackage(Package):
 
     extends("r")
 
-    homepage: ClassProperty[Optional[str]] = classproperty(_homepage)
-    url: ClassProperty[Optional[str]] = classproperty(_url)
-    list_url: ClassProperty[Optional[str]] = classproperty(_list_url)
-    git: ClassProperty[Optional[str]] = classproperty(_git)
+    @lang.classproperty
+    def homepage(cls):
+        if cls.cran:
+            return f"https://cloud.r-project.org/package={cls.cran}"
+        elif cls.bioc:
+            return f"https://bioconductor.org/packages/{cls.bioc}"
+
+    @lang.classproperty
+    def url(cls):
+        if cls.cran:
+            return f"https://cloud.r-project.org/src/contrib/{cls.cran}_{str(list(cls.versions)[0])}.tar.gz"
+
+    @lang.classproperty
+    def list_url(cls):
+        if cls.cran:
+            return f"https://cloud.r-project.org/src/contrib/Archive/{cls.cran}/"
+
+    @lang.classproperty
+    def git(cls):
+        if cls.bioc:
+            return f"https://git.bioconductor.org/packages/{cls.bioc}"

lib/spack/spack/build_systems/racket.py

@@ -5,8 +5,8 @@
 from typing import Optional, Tuple
 
 import llnl.util.filesystem as fs
+import llnl.util.lang as lang
 import llnl.util.tty as tty
-from llnl.util.lang import ClassProperty, classproperty
 
 import spack.builder
 import spack.spec
@@ -19,12 +19,6 @@
 from spack.util.executable import Executable, ProcessError
 
 
-def _homepage(cls: "RacketPackage") -> Optional[str]:
-    if cls.racket_name:
-        return f"https://pkgs.racket-lang.org/package/{cls.racket_name}"
-    return None
-
-
 class RacketPackage(PackageBase):
     """Specialized class for packages that are built using Racket's
     `raco pkg install` and `raco setup` commands.
@@ -43,7 +37,13 @@ class RacketPackage(PackageBase):
     extends("racket", when="build_system=racket")
 
     racket_name: Optional[str] = None
-    homepage: ClassProperty[Optional[str]] = classproperty(_homepage)
+    parallel = True
+
+    @lang.classproperty
+    def homepage(cls):
+        if cls.racket_name:
+            return "https://pkgs.racket-lang.org/package/{0}".format(cls.racket_name)
+        return None
 
 
 @spack.builder.builder("racket")

lib/spack/spack/buildcache_migrate.py

@@ -1,351 +0,0 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-import codecs
-import json
-import os
-import pathlib
-import tempfile
-from typing import NamedTuple
-
-import llnl.util.tty as tty
-
-import spack.binary_distribution as bindist
-import spack.database as spack_db
-import spack.error
-import spack.mirrors.mirror
-import spack.spec
-import spack.stage
-import spack.util.crypto
-import spack.util.parallel
-import spack.util.url as url_util
-import spack.util.web as web_util
-
-from .enums import InstallRecordStatus
-from .url_buildcache import (
-    BlobRecord,
-    BuildcacheComponent,
-    compressed_json_from_dict,
-    get_url_buildcache_class,
-    sign_file,
-    try_verify,
-)
-
-
-def v2_tarball_directory_name(spec):
-    """
-    Return name of the tarball directory according to the convention
-    <os>-<architecture>/<compiler>/<package>-<version>/
-    """
-    return spec.format_path("{architecture}/{compiler.name}-{compiler.version}/{name}-{version}")
-
-
-def v2_tarball_name(spec, ext):
-    """
-    Return the name of the tarfile according to the convention
-    <os>-<architecture>-<package>-<dag_hash><ext>
-    """
-    spec_formatted = spec.format_path(
-        "{architecture}-{compiler.name}-{compiler.version}-{name}-{version}-{hash}"
-    )
-    return f"{spec_formatted}{ext}"
-
-
-def v2_tarball_path_name(spec, ext):
-    """
-    Return the full path+name for a given spec according to the convention
-    <tarball_directory_name>/<tarball_name>
-    """
-    return os.path.join(v2_tarball_directory_name(spec), v2_tarball_name(spec, ext))
-
-
-class MigrateSpecResult(NamedTuple):
-    success: bool
-    message: str
-
-
-class MigrationException(spack.error.SpackError):
-    """
-    Raised when migration fails irrevocably
-    """
-
-    def __init__(self, msg):
-        super().__init__(msg)
-
-
-def _migrate_spec(
-    s: spack.spec.Spec, mirror_url: str, tmpdir: str, unsigned: bool = False, signing_key: str = ""
-) -> MigrateSpecResult:
-    """Parallelizable function to migrate a single spec"""
-    print_spec = f"{s.name}/{s.dag_hash()[:7]}"
-
-    # Check if the spec file exists in the new location and exit early if so
-
-    v3_cache_class = get_url_buildcache_class(layout_version=3)
-    v3_cache_entry = v3_cache_class(mirror_url, s, allow_unsigned=unsigned)
-    exists = v3_cache_entry.exists([BuildcacheComponent.SPEC, BuildcacheComponent.TARBALL])
-    v3_cache_entry.destroy()
-
-    if exists:
-        msg = f"No need to migrate {print_spec}"
-        return MigrateSpecResult(True, msg)
-
-    # Try to fetch the spec metadata
-    v2_metadata_urls = [
-        url_util.join(mirror_url, "build_cache", v2_tarball_name(s, ".spec.json.sig"))
-    ]
-
-    if unsigned:
-        v2_metadata_urls.append(
-            url_util.join(mirror_url, "build_cache", v2_tarball_name(s, ".spec.json"))
-        )
-
-    spec_contents = None
-
-    for meta_url in v2_metadata_urls:
-        try:
-            _, _, meta_file = web_util.read_from_url(meta_url)
-            spec_contents = codecs.getreader("utf-8")(meta_file).read()
-            v2_spec_url = meta_url
-            break
-        except (web_util.SpackWebError, OSError):
-            pass
-    else:
-        msg = f"Unable to read metadata for {print_spec}"
-        return MigrateSpecResult(False, msg)
-
-    spec_dict = {}
-
-    if unsigned:
-        # User asked for unsigned, if we found a signed specfile, just ignore
-        # the signature
-        if v2_spec_url.endswith(".sig"):
-            spec_dict = spack.spec.Spec.extract_json_from_clearsig(spec_contents)
-        else:
-            spec_dict = json.loads(spec_contents)
-    else:
-        # User asked for signed, we must successfully verify the signature
-        local_signed_pre_verify = os.path.join(
-            tmpdir, f"{s.name}_{s.dag_hash()}_verify.spec.json.sig"
-        )
-        with open(local_signed_pre_verify, "w", encoding="utf-8") as fd:
-            fd.write(spec_contents)
-        if not try_verify(local_signed_pre_verify):
-            return MigrateSpecResult(False, f"Failed to verify signature of {print_spec}")
-        with open(local_signed_pre_verify, encoding="utf-8") as fd:
-            spec_dict = spack.spec.Spec.extract_json_from_clearsig(fd.read())
-
-    # Read out and remove the bits needed to rename and position the archive
-    bcc = spec_dict.pop("binary_cache_checksum", None)
-    if not bcc:
-        msg = "Cannot migrate a spec that does not have 'binary_cache_checksum'"
-        return MigrateSpecResult(False, msg)
-
-    algorithm = bcc["hash_algorithm"]
-    checksum = bcc["hash"]
-
-    # TODO: Remove this key once oci buildcache no longer uses it
-    spec_dict["buildcache_layout_version"] = 2
-
-    v2_archive_url = url_util.join(mirror_url, "build_cache", v2_tarball_path_name(s, ".spack"))
-
-    # spacks web utilities do not include direct copying of s3 objects, so we
-    # need to download the archive locally, and then push it back to the target
-    # location
-    archive_stage_path = os.path.join(tmpdir, f"archive_stage_{s.name}_{s.dag_hash()}")
-    archive_stage = spack.stage.Stage(v2_archive_url, path=archive_stage_path)
-
-    try:
-        archive_stage.create()
-        archive_stage.fetch()
-    except spack.error.FetchError:
-        return MigrateSpecResult(False, f"Unable to fetch archive for {print_spec}")
-
-    local_tarfile_path = archive_stage.save_filename
-
-    # As long as we have to download the tarball anyway, we might as well compute the
-    # checksum locally and check it against the expected value
-    local_checksum = spack.util.crypto.checksum(
-        spack.util.crypto.hash_fun_for_algo(algorithm), local_tarfile_path
-    )
-
-    if local_checksum != checksum:
-        return MigrateSpecResult(
-            False, f"Checksum mismatch for {print_spec}: expected {checksum}, got {local_checksum}"
-        )
-
-    spec_dict["archive_size"] = os.stat(local_tarfile_path).st_size
-
-    # Compress the spec dict and compute its checksum
-    metadata_checksum_algo = "sha256"
-    spec_json_path = os.path.join(tmpdir, f"{s.name}_{s.dag_hash()}.spec.json")
-    metadata_checksum, metadata_size = compressed_json_from_dict(
-        spec_json_path, spec_dict, metadata_checksum_algo
-    )
-
-    tarball_blob_record = BlobRecord(
-        spec_dict["archive_size"], v3_cache_class.TARBALL_MEDIATYPE, "gzip", algorithm, checksum
-    )
-
-    metadata_blob_record = BlobRecord(
-        metadata_size,
-        v3_cache_class.SPEC_MEDIATYPE,
-        "gzip",
-        metadata_checksum_algo,
-        metadata_checksum,
-    )
-
-    # Compute the urls to the new blobs
-    v3_archive_url = v3_cache_class.get_blob_url(mirror_url, tarball_blob_record)
-    v3_spec_url = v3_cache_class.get_blob_url(mirror_url, metadata_blob_record)
-
-    # First push the tarball
-    tty.debug(f"Pushing {local_tarfile_path} to {v3_archive_url}")
-
-    try:
-        web_util.push_to_url(local_tarfile_path, v3_archive_url, keep_original=True)
-    except Exception:
-        return MigrateSpecResult(False, f"Failed to push archive for {print_spec}")
-
-    # Then push the spec file
-    tty.debug(f"Pushing {spec_json_path} to {v3_spec_url}")
-
-    try:
-        web_util.push_to_url(spec_json_path, v3_spec_url, keep_original=True)
-    except Exception:
-        return MigrateSpecResult(False, f"Failed to push spec metadata for {print_spec}")
-
-    # Generate the manifest and write it to a temporary location
-    manifest = {
-        "version": v3_cache_class.get_layout_version(),
-        "data": [tarball_blob_record.to_dict(), metadata_blob_record.to_dict()],
-    }
-
-    manifest_path = os.path.join(tmpdir, f"{s.dag_hash()}.manifest.json")
-    with open(manifest_path, "w", encoding="utf-8") as f:
-        json.dump(manifest, f, indent=0, separators=(",", ":"))
-        # Note: when using gpg clear sign, we need to avoid long lines (19995
-        # chars). If lines are longer, they are truncated without error. So,
-        # here we still add newlines, but no indent, so save on file size and
-        # line length.
-
-    # Possibly sign the manifest
-    if not unsigned:
-        manifest_path = sign_file(signing_key, manifest_path)
-
-    v3_manifest_url = v3_cache_class.get_manifest_url(s, mirror_url)
-
-    # Push the manifest
-    try:
-        web_util.push_to_url(manifest_path, v3_manifest_url, keep_original=True)
-    except Exception:
-        return MigrateSpecResult(False, f"Failed to push manifest for {print_spec}")
-
-    return MigrateSpecResult(True, f"Successfully migrated {print_spec}")
-
-
-def migrate(
-    mirror: spack.mirrors.mirror.Mirror, unsigned: bool = False, delete_existing: bool = False
-) -> None:
-    """Perform migration of the given mirror
-
-    If unsigned is True, signatures on signed specs will be ignored, and specs
-    will not be re-signed before pushing to the new location.  Otherwise, spack
-    will attempt to verify signatures and re-sign specs, and will fail if not
-    able to do so.  If delete_existing is True, spack will delete the original
-    contents of the mirror once the migration is complete."""
-    signing_key = ""
-    if not unsigned:
-        try:
-            signing_key = bindist.select_signing_key()
-        except (bindist.NoKeyException, bindist.PickKeyException):
-            raise MigrationException(
-                "Signed migration requires exactly one secret key in keychain"
-            )
-
-    delete_action = "deleting" if delete_existing else "keeping"
-    sign_action = "an unsigned" if unsigned else "a signed"
-    mirror_url = mirror.fetch_url
-
-    tty.msg(
-        f"Performing {sign_action} migration of {mirror.push_url} "
-        f"and {delete_action} existing contents"
-    )
-
-    index_url = url_util.join(mirror_url, "build_cache", spack_db.INDEX_JSON_FILE)
-    contents = None
-
-    try:
-        _, _, index_file = web_util.read_from_url(index_url)
-        contents = codecs.getreader("utf-8")(index_file).read()
-    except (web_util.SpackWebError, OSError):
-        raise MigrationException("Buildcache migration requires a buildcache index")
-
-    with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:
-        index_path = os.path.join(tmpdir, "_tmp_index.json")
-        with open(index_path, "w", encoding="utf-8") as fd:
-            fd.write(contents)
-
-        db = bindist.BuildCacheDatabase(tmpdir)
-        db._read_from_file(pathlib.Path(index_path))
-
-        specs_to_migrate = [
-            s
-            for s in db.query_local(installed=InstallRecordStatus.ANY)
-            if not s.external and db.query_local_by_spec_hash(s.dag_hash()).in_buildcache
-        ]
-
-        # Run the tasks in parallel if possible
-        executor = spack.util.parallel.make_concurrent_executor()
-        migrate_futures = [
-            executor.submit(_migrate_spec, spec, mirror_url, tmpdir, unsigned, signing_key)
-            for spec in specs_to_migrate
-        ]
-
-        success_count = 0
-
-        tty.msg("Migration summary:")
-        for spec, migrate_future in zip(specs_to_migrate, migrate_futures):
-            result = migrate_future.result()
-            msg = f"  {spec.name}/{spec.dag_hash()[:7]}: {result.message}"
-            if result.success:
-                success_count += 1
-                tty.msg(msg)
-            else:
-                tty.error(msg)
-            # The migrated index should have the same specs as the original index,
-            # modulo any specs that we failed to migrate for whatever reason. So
-            # to avoid having to re-fetch all the spec files now, just mark them
-            # appropriately in the existing database and push that.
-            db.mark(spec, "in_buildcache", result.success)
-
-        if success_count > 0:
-            tty.msg("Updating index and pushing keys")
-
-            # If the layout.json doesn't yet exist on this mirror, push it
-            v3_cache_class = get_url_buildcache_class(layout_version=3)
-            v3_cache_class.maybe_push_layout_json(mirror_url)
-
-            # Push the migrated mirror index
-            index_tmpdir = os.path.join(tmpdir, "rebuild_index")
-            os.mkdir(index_tmpdir)
-            bindist._push_index(db, index_tmpdir, mirror_url)
-
-            # Push the public part of the signing key
-            if not unsigned:
-                keys_tmpdir = os.path.join(tmpdir, "keys")
-                os.mkdir(keys_tmpdir)
-                bindist._url_push_keys(
-                    mirror_url, keys=[signing_key], update_index=True, tmpdir=keys_tmpdir
-                )
-        else:
-            tty.warn("No specs migrated, did you mean to perform an unsigned migration instead?")
-
-        # Delete the old layout if the user requested it
-        if delete_existing:
-            delete_prefix = url_util.join(mirror_url, "build_cache")
-            tty.msg(f"Recursively deleting {delete_prefix}")
-            web_util.remove_url(delete_prefix, recursive=True)
-
-    tty.msg("Migration complete")

lib/spack/spack/builder.py

@@ -59,7 +59,7 @@ def __call__(self, spec, prefix):
 def get_builder_class(pkg, name: str) -> Optional[Type["Builder"]]:
     """Return the builder class if a package module defines it."""
     cls = getattr(pkg.module, name, None)
-    if cls and spack.repo.is_package_module(cls.__module__):
+    if cls and cls.__module__.startswith(spack.repo.ROOT_PYTHON_NAMESPACE):
         return cls
     return None
 
@@ -121,7 +121,6 @@ def __init__(self, wrapped_pkg_object, root_builder):
                 new_cls_name,
                 bases,
                 {
-                    "__module__": package_cls.__module__,
                     "run_tests": property(lambda x: x.wrapped_package_object.run_tests),
                     "test_requires_compiler": property(
                         lambda x: x.wrapped_package_object.test_requires_compiler
@@ -130,6 +129,7 @@ def __init__(self, wrapped_pkg_object, root_builder):
                     "tester": property(lambda x: x.wrapped_package_object.tester),
                 },
             )
+            new_cls.__module__ = package_cls.__module__
             self.__class__ = new_cls
             self.__dict__.update(wrapped_pkg_object.__dict__)
 
@@ -185,16 +185,10 @@ def __init__(self, pkg):
         # These two methods don't follow the (self, spec, prefix) signature of phases nor
         # the (self) signature of methods, so they are added explicitly to avoid using a
         # catch-all (*args, **kwargs)
-        def setup_build_environment(
-            self, env: spack.util.environment.EnvironmentModifications
-        ) -> None:
+        def setup_build_environment(self, env):
             return self.pkg_with_dispatcher.setup_build_environment(env)
 
-        def setup_dependent_build_environment(
-            self,
-            env: spack.util.environment.EnvironmentModifications,
-            dependent_spec: spack.spec.Spec,
-        ) -> None:
+        def setup_dependent_build_environment(self, env, dependent_spec):
             return self.pkg_with_dispatcher.setup_dependent_build_environment(env, dependent_spec)
 
     return Adapter(pkg)
@@ -408,7 +402,7 @@ def fixup_install(self):
                 # do something after the package is installed
                 pass
 
-           def setup_build_environment(self, env: EnvironmentModifications) -> None:
+           def setup_build_environment(self, env):
                 env.set("MY_ENV_VAR", "my_value")
 
         class CMakeBuilder(cmake.CMakeBuilder, AnyBuilder):

lib/spack/spack/ci/__init__.py

@@ -14,7 +14,7 @@
 import tempfile
 import zipfile
 from collections import namedtuple
-from typing import Callable, Dict, List, Optional, Set, Union
+from typing import Callable, Dict, List, Set, Union
 from urllib.request import Request
 
 import llnl.path
@@ -24,7 +24,6 @@
 
 import spack
 import spack.binary_distribution as bindist
-import spack.builder
 import spack.config as cfg
 import spack.environment as ev
 import spack.error
@@ -33,7 +32,6 @@
 import spack.paths
 import spack.repo
 import spack.spec
-import spack.stage
 import spack.store
 import spack.util.git
 import spack.util.gpg as gpg_util
@@ -151,10 +149,10 @@ def get_stack_changed(env_path, rev1="HEAD^", rev2="HEAD"):
     return False
 
 
-def compute_affected_packages(rev1: str = "HEAD^", rev2: str = "HEAD") -> Set[str]:
+def compute_affected_packages(rev1="HEAD^", rev2="HEAD"):
     """Determine which packages were added, removed or changed
     between rev1 and rev2, and return the names as a set"""
-    return spack.repo.get_all_package_diffs("ARC", spack.repo.builtin_repo(), rev1=rev1, rev2=rev2)
+    return spack.repo.get_all_package_diffs("ARC", rev1=rev1, rev2=rev2)
 
 
 def get_spec_filter_list(env, affected_pkgs, dependent_traverse_depth=None):
@@ -246,9 +244,7 @@ def rebuild_filter(s: spack.spec.Spec) -> RebuildDecision:
         if not spec_locations:
             return RebuildDecision(True, "not found anywhere")
 
-        urls = ",".join(
-            [f"{loc.url_and_version.url}@v{loc.url_and_version.version}" for loc in spec_locations]
-        )
+        urls = ",".join([loc["mirror_url"] for loc in spec_locations])
         message = f"up-to-date [{urls}]"
         return RebuildDecision(False, message)
 
@@ -617,40 +613,32 @@ def copy_stage_logs_to_artifacts(job_spec: spack.spec.Spec, job_log_dir: str) ->
     job_spec, and attempts to copy the files into the directory given
     by job_log_dir.
 
-    Parameters:
+    Args:
         job_spec: spec associated with spack install log
         job_log_dir: path into which build log should be copied
     """
     tty.debug(f"job spec: {job_spec}")
-    if not job_spec.concrete:
-        tty.warn("Cannot copy artifacts for non-concrete specs")
+
+    try:
+        package_metadata_root = pathlib.Path(spack.store.STORE.layout.metadata_path(job_spec))
+    except spack.error.SpackError as e:
+        tty.error(f"Cannot copy logs: {str(e)}")
         return
 
-    package_metadata_root = pathlib.Path(spack.store.STORE.layout.metadata_path(job_spec))
-    if not os.path.isdir(package_metadata_root):
-        # Fallback to using the stage directory
-        job_pkg = job_spec.package
-
-        package_metadata_root = pathlib.Path(job_pkg.stage.path)
-        archive_files = spack.builder.create(job_pkg).archive_files
-        tty.warn("Package not installed, falling back to use stage dir")
-        tty.debug(f"stage dir: {package_metadata_root}")
+    # Get the package's archived files
+    archive_files = []
+    archive_root = package_metadata_root / "archived-files"
+    if archive_root.is_dir():
+        archive_files = [f for f in archive_root.rglob("*") if f.is_file()]
     else:
-        # Get the package's archived files
-        archive_files = []
-        archive_root = package_metadata_root / "archived-files"
-        if os.path.isdir(archive_root):
-            archive_files = [str(f) for f in archive_root.rglob("*") if os.path.isfile(f)]
-        else:
-            tty.debug(f"No archived files detected at {archive_root}")
+        msg = "Cannot copy package archived files: archived-files must be a directory"
+        tty.warn(msg)
 
-    # Try zipped and unzipped versions of the build log
     build_log_zipped = package_metadata_root / "spack-build-out.txt.gz"
-    build_log = package_metadata_root / "spack-build-out.txt"
     build_env_mods = package_metadata_root / "spack-build-env.txt"
 
-    for f in [build_log_zipped, build_log, build_env_mods, *archive_files]:
-        copy_files_to_artifacts(str(f), job_log_dir, compress_artifacts=True)
+    for f in [build_log_zipped, build_env_mods, *archive_files]:
+        copy_files_to_artifacts(str(f), job_log_dir)
 
 
 def copy_test_logs_to_artifacts(test_stage, job_test_dir):
@@ -663,12 +651,11 @@ def copy_test_logs_to_artifacts(test_stage, job_test_dir):
     """
     tty.debug(f"test stage: {test_stage}")
     if not os.path.exists(test_stage):
-        tty.error(f"Cannot copy test logs: job test stage ({test_stage}) does not exist")
+        msg = f"Cannot copy test logs: job test stage ({test_stage}) does not exist"
+        tty.error(msg)
         return
 
-    copy_files_to_artifacts(
-        os.path.join(test_stage, "*", "*.txt"), job_test_dir, compress_artifacts=True
-    )
+    copy_files_to_artifacts(os.path.join(test_stage, "*", "*.txt"), job_test_dir)
 
 
 def download_and_extract_artifacts(url, work_dir) -> str:
@@ -1245,31 +1232,33 @@ def write_broken_spec(url, pkg_name, stack_name, job_url, pipeline_url, spec_dic
     """Given a url to write to and the details of the failed job, write an entry
     in the broken specs list.
     """
-    with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:
-        file_path = os.path.join(tmpdir, "broken.txt")
+    tmpdir = tempfile.mkdtemp()
+    file_path = os.path.join(tmpdir, "broken.txt")
 
-        broken_spec_details = {
-            "broken-spec": {
-                "job-name": pkg_name,
-                "job-stack": stack_name,
-                "job-url": job_url,
-                "pipeline-url": pipeline_url,
-                "concrete-spec-dict": spec_dict,
-            }
+    broken_spec_details = {
+        "broken-spec": {
+            "job-name": pkg_name,
+            "job-stack": stack_name,
+            "job-url": job_url,
+            "pipeline-url": pipeline_url,
+            "concrete-spec-dict": spec_dict,
         }
+    }
 
-        try:
-            with open(file_path, "w", encoding="utf-8") as fd:
-                syaml.dump(broken_spec_details, fd)
-            web_util.push_to_url(
-                file_path, url, keep_original=False, extra_args={"ContentType": "text/plain"}
-            )
-        except Exception as err:
-            # If there is an S3 error (e.g., access denied or connection
-            # error), the first non boto-specific class in the exception
-            # hierarchy is Exception.  Just print a warning and return
-            msg = f"Error writing to broken specs list {url}: {err}"
-            tty.warn(msg)
+    try:
+        with open(file_path, "w", encoding="utf-8") as fd:
+            syaml.dump(broken_spec_details, fd)
+        web_util.push_to_url(
+            file_path, url, keep_original=False, extra_args={"ContentType": "text/plain"}
+        )
+    except Exception as err:
+        # If there is an S3 error (e.g., access denied or connection
+        # error), the first non boto-specific class in the exception
+        # hierarchy is Exception.  Just print a warning and return
+        msg = f"Error writing to broken specs list {url}: {err}"
+        tty.warn(msg)
+    finally:
+        shutil.rmtree(tmpdir)
 
 
 def read_broken_spec(broken_spec_url):
@@ -1305,34 +1294,35 @@ def display_broken_spec_messages(base_url, hashes):
         tty.msg(msg)
 
 
-def run_standalone_tests(
-    *,
-    cdash: Optional[CDashHandler] = None,
-    fail_fast: bool = False,
-    log_file: Optional[str] = None,
-    job_spec: Optional[spack.spec.Spec] = None,
-    repro_dir: Optional[str] = None,
-    timeout: Optional[int] = None,
-):
+def run_standalone_tests(**kwargs):
     """Run stand-alone tests on the current spec.
 
-    Args:
-        cdash: cdash handler instance
-        fail_fast: terminate tests after the first failure
-        log_file: test log file name if NOT CDash reporting
-        job_spec: spec that was built
-        repro_dir: reproduction directory
-        timeout: maximum time (in seconds) that tests are allowed to run
+    Arguments:
+       kwargs (dict): dictionary of arguments used to run the tests
+
+    List of recognized keys:
+
+    * "cdash" (CDashHandler): (optional) cdash handler instance
+    * "fail_fast" (bool): (optional) terminate tests after the first failure
+    * "log_file" (str): (optional) test log file name if NOT CDash reporting
+    * "job_spec" (Spec): spec that was built
+    * "repro_dir" (str): reproduction directory
     """
+    cdash = kwargs.get("cdash")
+    fail_fast = kwargs.get("fail_fast")
+    log_file = kwargs.get("log_file")
+
     if cdash and log_file:
         tty.msg(f"The test log file {log_file} option is ignored with CDash reporting")
         log_file = None
 
     # Error out but do NOT terminate if there are missing required arguments.
+    job_spec = kwargs.get("job_spec")
     if not job_spec:
         tty.error("Job spec is required to run stand-alone tests")
         return
 
+    repro_dir = kwargs.get("repro_dir")
     if not repro_dir:
         tty.error("Reproduction directory is required for stand-alone tests")
         return
@@ -1341,9 +1331,6 @@ def run_standalone_tests(
     if fail_fast:
         test_args.append("--fail-fast")
 
-    if timeout is not None:
-        test_args.extend(["--timeout", str(timeout)])
-
     if cdash:
         test_args.extend(cdash.args())
     else:

lib/spack/spack/ci/common.py

@@ -2,13 +2,9 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import copy
-import errno
-import glob
-import gzip
 import json
 import os
 import re
-import shutil
 import sys
 import time
 from collections import deque
@@ -29,14 +25,13 @@
 import spack.mirrors.mirror
 import spack.schema
 import spack.spec
-import spack.util.compression as compression
 import spack.util.spack_yaml as syaml
+import spack.util.url as url_util
 import spack.util.web as web_util
 from spack import traverse
 from spack.reporters import CDash, CDashConfiguration
 from spack.reporters.cdash import SPACK_CDASH_TIMEOUT
 from spack.reporters.cdash import build_stamp as cdash_build_stamp
-from spack.url_buildcache import get_url_buildcache_class
 
 IS_WINDOWS = sys.platform == "win32"
 SPACK_RESERVED_TAGS = ["public", "protected", "notary"]
@@ -45,67 +40,22 @@
 _urlopen = web_util.urlopen
 
 
-def copy_gzipped(glob_or_path: str, dest: str) -> None:
-    """Copy all of the files in the source glob/path to the destination.
-
-    Args:
-        glob_or_path: path to file to test
-        dest: destination path to copy to
-    """
-
-    files = glob.glob(glob_or_path)
-    if not files:
-        raise OSError("No such file or directory: '{0}'".format(glob_or_path), errno.ENOENT)
-    if len(files) > 1 and not os.path.isdir(dest):
-        raise ValueError(
-            "'{0}' matches multiple files but '{1}' is not a directory".format(glob_or_path, dest)
-        )
-
-    def is_gzipped(path):
-        with open(path, "rb") as fd:
-            return compression.GZipFileType().matches_magic(fd)
-
-    for src in files:
-        if is_gzipped(src):
-            fs.copy(src, dest)
-        else:
-            # Compress and copy in one step
-            src_name = os.path.basename(src)
-            if os.path.isdir(dest):
-                zipped = os.path.join(dest, f"{src_name}.gz")
-            elif not dest.endswith(".gz"):
-                zipped = f"{dest}.gz"
-            else:
-                zipped = dest
-
-            with open(src, "rb") as fin, gzip.open(zipped, "wb") as fout:
-                shutil.copyfileobj(fin, fout)
-
-
-def copy_files_to_artifacts(
-    src: str, artifacts_dir: str, *, compress_artifacts: bool = False
-) -> None:
+def copy_files_to_artifacts(src, artifacts_dir):
     """
     Copy file(s) to the given artifacts directory
 
-    Args:
+    Parameters:
         src (str): the glob-friendly path expression for the file(s) to copy
         artifacts_dir (str): the destination directory
-        compress_artifacts (bool): option to compress copied artifacts using Gzip
     """
     try:
-
-        if compress_artifacts:
-            copy_gzipped(src, artifacts_dir)
-        else:
-            fs.copy(src, artifacts_dir)
+        fs.copy(src, artifacts_dir)
     except Exception as err:
-        tty.warn(
-            (
-                f"Unable to copy files ({src}) to artifacts {artifacts_dir} due to "
-                f"exception: {str(err)}"
-            )
+        msg = (
+            f"Unable to copy files ({src}) to artifacts {artifacts_dir} due to "
+            f"exception: {str(err)}"
         )
+        tty.warn(msg)
 
 
 def win_quote(quote_str: str) -> str:
@@ -179,13 +129,33 @@ def write_pipeline_manifest(specs, src_prefix, dest_prefix, output_file):
 
     for release_spec in specs:
         release_spec_dag_hash = release_spec.dag_hash()
-        cache_class = get_url_buildcache_class(
-            layout_version=bindist.CURRENT_BUILD_CACHE_LAYOUT_VERSION
-        )
-        buildcache_copies[release_spec_dag_hash] = {
-            "src": cache_class.get_manifest_url(release_spec, src_prefix),
-            "dest": cache_class.get_manifest_url(release_spec, dest_prefix),
-        }
+        # TODO: This assumes signed version of the spec
+        buildcache_copies[release_spec_dag_hash] = [
+            {
+                "src": url_util.join(
+                    src_prefix,
+                    bindist.build_cache_relative_path(),
+                    bindist.tarball_name(release_spec, ".spec.json.sig"),
+                ),
+                "dest": url_util.join(
+                    dest_prefix,
+                    bindist.build_cache_relative_path(),
+                    bindist.tarball_name(release_spec, ".spec.json.sig"),
+                ),
+            },
+            {
+                "src": url_util.join(
+                    src_prefix,
+                    bindist.build_cache_relative_path(),
+                    bindist.tarball_path_name(release_spec, ".spack"),
+                ),
+                "dest": url_util.join(
+                    dest_prefix,
+                    bindist.build_cache_relative_path(),
+                    bindist.tarball_path_name(release_spec, ".spack"),
+                ),
+            },
+        ]
 
     target_dir = os.path.dirname(output_file)
 

lib/spack/spack/ci/gitlab.py

@@ -292,9 +292,6 @@ def main_script_replacements(cmd):
     )
     maybe_generate_manifest(pipeline, options, manifest_path)
 
-    relative_specs_url = bindist.buildcache_relative_specs_url()
-    relative_keys_url = bindist.buildcache_relative_keys_url()
-
     if options.pipeline_type == PipelineType.COPY_ONLY:
         stage_names.append("copy")
         sync_job = copy.deepcopy(spack_ci_ir["jobs"]["copy"]["attributes"])
@@ -304,12 +301,9 @@ def main_script_replacements(cmd):
         if "variables" not in sync_job:
             sync_job["variables"] = {}
 
-        sync_job["variables"].update(
-            {
-                "SPACK_COPY_ONLY_DESTINATION": options.buildcache_destination.fetch_url,
-                "SPACK_BUILDCACHE_RELATIVE_KEYS_URL": relative_keys_url,
-            }
-        )
+        sync_job["variables"][
+            "SPACK_COPY_ONLY_DESTINATION"
+        ] = options.buildcache_destination.fetch_url
 
         pipeline_mirrors = spack.mirrors.mirror.MirrorCollection(binary=True)
         if "buildcache-source" not in pipeline_mirrors:
@@ -339,13 +333,9 @@ def main_script_replacements(cmd):
             signing_job["interruptible"] = True
             if "variables" not in signing_job:
                 signing_job["variables"] = {}
-            signing_job["variables"].update(
-                {
-                    "SPACK_BUILDCACHE_DESTINATION": options.buildcache_destination.push_url,
-                    "SPACK_BUILDCACHE_RELATIVE_SPECS_URL": relative_specs_url,
-                    "SPACK_BUILDCACHE_RELATIVE_KEYS_URL": relative_keys_url,
-                }
-            )
+            signing_job["variables"][
+                "SPACK_BUILDCACHE_DESTINATION"
+            ] = options.buildcache_destination.push_url
             signing_job["dependencies"] = []
 
             output_object["sign-pkgs"] = signing_job

lib/spack/spack/cmd/__init__.py

@@ -436,7 +436,7 @@ def display_specs(specs, args=None, **kwargs):
         all_headers (bool): show headers even when arch/compiler aren't defined
         status_fn (typing.Callable): if provided, prepend install-status info
         output (typing.IO): A file object to write to. Default is ``sys.stdout``
-        specfile_format (bool): specfile format of the current spec
+
     """
 
     def get_arg(name, default=None):
@@ -458,7 +458,6 @@ def get_arg(name, default=None):
     all_headers = get_arg("all_headers", False)
     output = get_arg("output", sys.stdout)
     status_fn = get_arg("status_fn", None)
-    specfile_format = get_arg("specfile_format", False)
 
     decorator = get_arg("decorator", None)
     if decorator is None:
@@ -480,9 +479,6 @@ def get_arg(name, default=None):
         vfmt = "{variants}" if variants else ""
         format_string = nfmt + "{@version}" + vfmt + ffmt
 
-    if specfile_format:
-        format_string = "[{specfile_version}] " + format_string
-
     def fmt(s, depth=0):
         """Formatter function for all output specs"""
         string = ""

lib/spack/spack/cmd/bootstrap.py

@@ -2,7 +2,6 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
-import pathlib
 import shutil
 import sys
 import tempfile
@@ -29,7 +28,7 @@
 
 
 # Tarball to be downloaded if binary packages are requested in a local mirror
-BINARY_TARBALL = "https://github.com/spack/spack-bootstrap-mirrors/releases/download/v0.6/bootstrap-buildcache-v3.tar.gz"
+BINARY_TARBALL = "https://github.com/spack/spack-bootstrap-mirrors/releases/download/v0.6/bootstrap-buildcache.tar.gz"
 
 #: Subdirectory where to create the mirror
 LOCAL_MIRROR_DIR = "bootstrap_cache"
@@ -411,9 +410,8 @@ def _mirror(args):
         stage.create()
         stage.fetch()
         stage.expand_archive()
-        stage_dir = pathlib.Path(stage.source_path)
-        for entry in stage_dir.iterdir():
-            shutil.move(str(entry), mirror_dir)
+        build_cache_dir = os.path.join(stage.source_path, "build_cache")
+        shutil.move(build_cache_dir, mirror_dir)
         llnl.util.tty.set_msg_enabled(True)
 
     def write_metadata(subdir, metadata):
@@ -438,6 +436,7 @@ def write_metadata(subdir, metadata):
         shutil.copy(spack.util.path.canonicalize_path(GNUPG_JSON), abs_directory)
         shutil.copy(spack.util.path.canonicalize_path(PATCHELF_JSON), abs_directory)
         instructions += cmd.format("local-binaries", rel_directory)
+        instructions += "  % spack buildcache update-index <final-path>/bootstrap_cache\n"
     print(instructions)
 
 

lib/spack/spack/cmd/buildcache.py

@@ -4,9 +4,11 @@
 import argparse
 import glob
 import json
+import os
+import shutil
 import sys
 import tempfile
-from typing import List, Optional, Tuple
+from typing import List, Tuple
 
 import llnl.util.tty as tty
 from llnl.string import plural
@@ -25,21 +27,14 @@
 import spack.stage
 import spack.store
 import spack.util.parallel
+import spack.util.url as url_util
 import spack.util.web as web_util
 from spack import traverse
 from spack.cmd import display_specs
 from spack.cmd.common import arguments
 from spack.spec import Spec, save_dependency_specfiles
 
-from ..buildcache_migrate import migrate
 from ..enums import InstallRecordStatus
-from ..url_buildcache import (
-    BuildcacheComponent,
-    BuildcacheEntryError,
-    URLBuildcacheEntry,
-    check_mirror_for_layout,
-    get_url_buildcache_class,
-)
 
 description = "create, download and install binary packages"
 section = "packaging"
@@ -81,6 +76,9 @@ def setup_parser(subparser: argparse.ArgumentParser):
         default=False,
         help="regenerate buildcache index after building package(s)",
     )
+    push.add_argument(
+        "--spec-file", default=None, help="create buildcache entry for spec from json or yaml file"
+    )
     push.add_argument(
         "--only",
         default="package,dependencies",
@@ -194,14 +192,28 @@ def setup_parser(subparser: argparse.ArgumentParser):
         default=lambda: spack.config.default_modify_scope(),
         help="configuration scope containing mirrors to check",
     )
-
+    # Unfortunately there are 3 ways to do the same thing here:
+    check_specs = check.add_mutually_exclusive_group()
+    check_specs.add_argument(
+        "-s", "--spec", help="check single spec instead of release specs file"
+    )
+    check_specs.add_argument(
+        "--spec-file",
+        help="check single spec from json or yaml file instead of release specs file",
+    )
     arguments.add_common_arguments(check, ["specs"])
 
     check.set_defaults(func=check_fn)
 
     # Download tarball and specfile
     download = subparsers.add_parser("download", help=download_fn.__doc__)
-    download.add_argument("-s", "--spec", help="download built tarball for spec from mirror")
+    download_spec_or_specfile = download.add_mutually_exclusive_group(required=True)
+    download_spec_or_specfile.add_argument(
+        "-s", "--spec", help="download built tarball for spec from mirror"
+    )
+    download_spec_or_specfile.add_argument(
+        "--spec-file", help="download built tarball for spec (from json or yaml file) from mirror"
+    )
     download.add_argument(
         "-p",
         "--path",
@@ -211,10 +223,28 @@ def setup_parser(subparser: argparse.ArgumentParser):
     )
     download.set_defaults(func=download_fn)
 
+    # Get buildcache name
+    getbuildcachename = subparsers.add_parser(
+        "get-buildcache-name", help=get_buildcache_name_fn.__doc__
+    )
+    getbuildcachename_spec_or_specfile = getbuildcachename.add_mutually_exclusive_group(
+        required=True
+    )
+    getbuildcachename_spec_or_specfile.add_argument(
+        "-s", "--spec", help="spec string for which buildcache name is desired"
+    )
+    getbuildcachename_spec_or_specfile.add_argument(
+        "--spec-file", help="path to spec json or yaml file for which buildcache name is desired"
+    )
+    getbuildcachename.set_defaults(func=get_buildcache_name_fn)
+
     # Given the root spec, save the yaml of the dependent spec to a file
     savespecfile = subparsers.add_parser("save-specfile", help=save_specfile_fn.__doc__)
     savespecfile_spec_or_specfile = savespecfile.add_mutually_exclusive_group(required=True)
     savespecfile_spec_or_specfile.add_argument("--root-spec", help="root spec of dependent spec")
+    savespecfile_spec_or_specfile.add_argument(
+        "--root-specfile", help="path to json or yaml file containing root spec of dependent spec"
+    )
     savespecfile.add_argument(
         "-s",
         "--specs",
@@ -277,27 +307,6 @@ def setup_parser(subparser: argparse.ArgumentParser):
     )
     update_index.set_defaults(func=update_index_fn)
 
-    # Migrate a buildcache from layout_version 2 to version 3
-    migrate = subparsers.add_parser("migrate", help=migrate_fn.__doc__)
-    migrate.add_argument("mirror", type=arguments.mirror_name, help="name of a configured mirror")
-    migrate.add_argument(
-        "-u",
-        "--unsigned",
-        default=False,
-        action="store_true",
-        help="Ignore signatures and do not resign, default is False",
-    )
-    migrate.add_argument(
-        "-d",
-        "--delete-existing",
-        default=False,
-        action="store_true",
-        help="Delete the previous layout, the default is to keep it.",
-    )
-    arguments.add_common_arguments(migrate, ["yes_to_all"])
-    # TODO: add -y argument to prompt if user really means to delete existing
-    migrate.set_defaults(func=migrate_fn)
-
 
 def _matching_specs(specs: List[Spec]) -> List[Spec]:
     """Disambiguate specs and return a list of matching specs"""
@@ -371,8 +380,14 @@ def _specs_to_be_packaged(
 
 def push_fn(args):
     """create a binary package and push it to a mirror"""
-    if args.specs:
-        roots = _matching_specs(spack.cmd.parse_specs(args.specs))
+    if args.spec_file:
+        tty.warn(
+            "The flag `--spec-file` is deprecated and will be removed in Spack 0.22. "
+            "Use positional arguments instead."
+        )
+
+    if args.specs or args.spec_file:
+        roots = _matching_specs(spack.cmd.parse_specs(args.specs or args.spec_file))
     else:
         roots = spack.cmd.require_active_env(cmd_name="buildcache push").concrete_roots()
 
@@ -423,10 +438,6 @@ def push_fn(args):
                     (s, PackageNotInstalledError("package not installed")) for s in not_installed
                 )
 
-    # Warn about possible old binary mirror layout
-    if not mirror.push_url.startswith("oci://"):
-        check_mirror_for_layout(mirror)
-
     with bindist.make_uploader(
         mirror=mirror,
         force=args.force,
@@ -518,7 +529,22 @@ def check_fn(args: argparse.Namespace):
     this command uses the process exit code to indicate its result, specifically, if the
     exit code is non-zero, then at least one of the indicated specs needs to be rebuilt
     """
-    specs_arg = args.specs
+    if args.spec_file:
+        specs_arg = (
+            args.spec_file if os.path.sep in args.spec_file else os.path.join(".", args.spec_file)
+        )
+        tty.warn(
+            "The flag `--spec-file` is deprecated and will be removed in Spack 0.22. "
+            f"Use `spack buildcache check {specs_arg}` instead."
+        )
+    elif args.spec:
+        specs_arg = args.spec
+        tty.warn(
+            "The flag `--spec` is deprecated and will be removed in Spack 0.23. "
+            f"Use `spack buildcache check {specs_arg}` instead."
+        )
+    else:
+        specs_arg = args.specs
 
     if specs_arg:
         specs = _matching_specs(spack.cmd.parse_specs(specs_arg))
@@ -552,12 +578,28 @@ def download_fn(args):
     code indicates that the command failed to download at least one of the required buildcache
     components
     """
-    specs = _matching_specs(spack.cmd.parse_specs(args.spec))
+    if args.spec_file:
+        tty.warn(
+            "The flag `--spec-file` is deprecated and will be removed in Spack 0.22. "
+            "Use --spec instead."
+        )
+
+    specs = _matching_specs(spack.cmd.parse_specs(args.spec or args.spec_file))
 
     if len(specs) != 1:
         tty.die("a single spec argument is required to download from a buildcache")
 
-    bindist.download_single_spec(specs[0], args.path)
+    if not bindist.download_single_spec(specs[0], args.path):
+        sys.exit(1)
+
+
+def get_buildcache_name_fn(args):
+    """get name (prefix) of buildcache entries for this spec"""
+    tty.warn("This command is deprecated and will be removed in Spack 0.22.")
+    specs = _matching_specs(spack.cmd.parse_specs(args.spec or args.spec_file))
+    if len(specs) != 1:
+        tty.die("a single spec argument is required to get buildcache name")
+    print(bindist.tarball_name(specs[0], ""))
 
 
 def save_specfile_fn(args):
@@ -567,7 +609,13 @@ def save_specfile_fn(args):
     successful. if any errors or exceptions are encountered, or if expected command-line arguments
     are not provided, then the exit code will be non-zero
     """
-    specs = spack.cmd.parse_specs(args.root_spec)
+    if args.root_specfile:
+        tty.warn(
+            "The flag `--root-specfile` is deprecated and will be removed in Spack 0.22. "
+            "Use --root-spec instead."
+        )
+
+    specs = spack.cmd.parse_specs(args.root_spec or args.root_specfile)
 
     if len(specs) != 1:
         tty.die("a single spec argument is required to save specfile")
@@ -582,78 +630,29 @@ def save_specfile_fn(args):
     )
 
 
-def copy_buildcache_entry(cache_entry: URLBuildcacheEntry, destination_url: str):
-    """Download buildcache entry and copy it to the destination_url"""
-    try:
-        spec_dict = cache_entry.fetch_metadata()
-        cache_entry.fetch_archive()
-    except bindist.BuildcacheEntryError as e:
-        tty.warn(f"Failed to retrieve buildcache for copying due to {e}")
-        cache_entry.destroy()
-        return
+def copy_buildcache_file(src_url, dest_url, local_path=None):
+    """Copy from source url to destination url"""
+    tmpdir = None
 
-    spec_blob_record = cache_entry.get_blob_record(BuildcacheComponent.SPEC)
-    local_spec_path = cache_entry.get_local_spec_path()
-    tarball_blob_record = cache_entry.get_blob_record(BuildcacheComponent.TARBALL)
-    local_tarball_path = cache_entry.get_local_archive_path()
-
-    target_spec = spack.spec.Spec.from_dict(spec_dict)
-    spec_label = f"{target_spec.name}/{target_spec.dag_hash()[:7]}"
-
-    if not tarball_blob_record:
-        cache_entry.destroy()
-        raise BuildcacheEntryError(f"No source tarball blob record, failed to sync {spec_label}")
-
-    # Try to push the tarball
-    tarball_dest_url = cache_entry.get_blob_url(destination_url, tarball_blob_record)
+    if not local_path:
+        tmpdir = tempfile.mkdtemp()
+        local_path = os.path.join(tmpdir, os.path.basename(src_url))
 
     try:
-        web_util.push_to_url(local_tarball_path, tarball_dest_url, keep_original=True)
-    except Exception as e:
-        tty.warn(f"Failed to push {local_tarball_path} to {tarball_dest_url} due to {e}")
-        cache_entry.destroy()
-        return
-
-    if not spec_blob_record:
-        cache_entry.destroy()
-        raise BuildcacheEntryError(f"No source spec blob record, failed to sync {spec_label}")
-
-    # Try to push the spec file
-    spec_dest_url = cache_entry.get_blob_url(destination_url, spec_blob_record)
-
-    try:
-        web_util.push_to_url(local_spec_path, spec_dest_url, keep_original=True)
-    except Exception as e:
-        tty.warn(f"Failed to push {local_spec_path} to {spec_dest_url} due to {e}")
-        cache_entry.destroy()
-        return
-
-    # Stage the manifest locally, since if it's signed, we don't want to try to
-    # to reproduce that here. Instead just push the locally staged manifest to
-    # the expected path at the destination url.
-    manifest_src_url = cache_entry.remote_manifest_url
-    manifest_dest_url = cache_entry.get_manifest_url(target_spec, destination_url)
-
-    manifest_stage = spack.stage.Stage(manifest_src_url)
-
-    try:
-        manifest_stage.create()
-        manifest_stage.fetch()
-    except Exception as e:
-        tty.warn(f"Failed to fetch manifest from {manifest_src_url} due to {e}")
-        manifest_stage.destroy()
-        cache_entry.destroy()
-        return
-
-    local_manifest_path = manifest_stage.save_filename
-
-    try:
-        web_util.push_to_url(local_manifest_path, manifest_dest_url, keep_original=True)
-    except Exception as e:
-        tty.warn(f"Failed to push manifest to {manifest_dest_url} due to {e}")
-
-    manifest_stage.destroy()
-    cache_entry.destroy()
+        temp_stage = spack.stage.Stage(src_url, path=os.path.dirname(local_path))
+        try:
+            temp_stage.create()
+            temp_stage.fetch()
+            web_util.push_to_url(local_path, dest_url, keep_original=True)
+        except spack.error.FetchError as e:
+            # Expected, since we have to try all the possible extensions
+            tty.debug("no such file: {0}".format(src_url))
+            tty.debug(e)
+        finally:
+            temp_stage.destroy()
+    finally:
+        if tmpdir and os.path.exists(tmpdir):
+            shutil.rmtree(tmpdir)
 
 
 def sync_fn(args):
@@ -693,21 +692,37 @@ def sync_fn(args):
         )
     )
 
+    build_cache_dir = bindist.build_cache_relative_path()
+    buildcache_rel_paths = []
+
     tty.debug("Syncing the following specs:")
-    specs_to_sync = [s for s in env.all_specs() if not s.external]
-    for s in specs_to_sync:
+    for s in env.all_specs():
         tty.debug("  {0}{1}: {2}".format("* " if s in env.roots() else "  ", s.name, s.dag_hash()))
-        cache_class = get_url_buildcache_class(
-            layout_version=bindist.CURRENT_BUILD_CACHE_LAYOUT_VERSION
+
+        buildcache_rel_paths.extend(
+            [
+                os.path.join(build_cache_dir, bindist.tarball_path_name(s, ".spack")),
+                os.path.join(build_cache_dir, bindist.tarball_name(s, ".spec.json.sig")),
+                os.path.join(build_cache_dir, bindist.tarball_name(s, ".spec.json")),
+                os.path.join(build_cache_dir, bindist.tarball_name(s, ".spec.yaml")),
+            ]
         )
-        src_cache_entry = cache_class(src_mirror_url, s, allow_unsigned=True)
-        src_cache_entry.read_manifest()
-        copy_buildcache_entry(src_cache_entry, dest_mirror_url)
+
+    tmpdir = tempfile.mkdtemp()
+
+    try:
+        for rel_path in buildcache_rel_paths:
+            src_url = url_util.join(src_mirror_url, rel_path)
+            local_path = os.path.join(tmpdir, rel_path)
+            dest_url = url_util.join(dest_mirror_url, rel_path)
+
+            tty.debug("Copying {0} to {1} via {2}".format(src_url, dest_url, local_path))
+            copy_buildcache_file(src_url, dest_url, local_path=local_path)
+    finally:
+        shutil.rmtree(tmpdir)
 
 
-def manifest_copy(
-    manifest_file_list: List[str], dest_mirror: Optional[spack.mirrors.mirror.Mirror] = None
-):
+def manifest_copy(manifest_file_list, dest_mirror=None):
     """Read manifest files containing information about specific specs to copy
     from source to destination, remove duplicates since any binary packge for
     a given hash should be the same as any other, and copy all files specified
@@ -717,24 +732,21 @@ def manifest_copy(
     for manifest_path in manifest_file_list:
         with open(manifest_path, encoding="utf-8") as fd:
             manifest = json.loads(fd.read())
-            for spec_hash, copy_obj in manifest.items():
+            for spec_hash, copy_list in manifest.items():
                 # Last duplicate hash wins
-                deduped_manifest[spec_hash] = copy_obj
+                deduped_manifest[spec_hash] = copy_list
 
-    for spec_hash, copy_obj in deduped_manifest.items():
-        cache_class = get_url_buildcache_class(
-            layout_version=bindist.CURRENT_BUILD_CACHE_LAYOUT_VERSION
-        )
-        src_cache_entry = cache_class(
-            cache_class.get_base_url(copy_obj["src"]), allow_unsigned=True
-        )
-        src_cache_entry.read_manifest(manifest_url=copy_obj["src"])
-        if dest_mirror:
-            destination_url = dest_mirror.push_url
-        else:
-            destination_url = cache_class.get_base_url(copy_obj["dest"])
-        tty.debug("copying {0} to {1}".format(copy_obj["src"], destination_url))
-        copy_buildcache_entry(src_cache_entry, destination_url)
+    build_cache_dir = bindist.build_cache_relative_path()
+    for spec_hash, copy_list in deduped_manifest.items():
+        for copy_file in copy_list:
+            dest = copy_file["dest"]
+            if dest_mirror:
+                src_relative_path = os.path.join(
+                    build_cache_dir, copy_file["src"].rsplit(build_cache_dir, 1)[1].lstrip("/")
+                )
+                dest = url_util.join(dest_mirror.push_url, src_relative_path)
+            tty.debug("copying {0} to {1}".format(copy_file["src"], dest))
+            copy_buildcache_file(copy_file["src"], dest)
 
 
 def update_index(mirror: spack.mirrors.mirror.Mirror, update_keys=False):
@@ -758,9 +770,13 @@ def update_index(mirror: spack.mirrors.mirror.Mirror, update_keys=False):
         bindist._url_generate_package_index(url, tmpdir)
 
     if update_keys:
+        keys_url = url_util.join(
+            url, bindist.build_cache_relative_path(), bindist.build_cache_keys_relative_path()
+        )
+
         try:
             with tempfile.TemporaryDirectory(dir=spack.stage.get_stage_root()) as tmpdir:
-                bindist.generate_key_index(url, tmpdir)
+                bindist.generate_key_index(keys_url, tmpdir)
         except bindist.CannotListKeys as e:
             # Do not error out if listing keys went wrong. This usually means that the _gpg path
             # does not exist. TODO: distinguish between this and other errors.
@@ -772,53 +788,5 @@ def update_index_fn(args):
     return update_index(args.mirror, update_keys=args.keys)
 
 
-def migrate_fn(args):
-    """perform in-place binary mirror migration (2 to 3)
-
-    A mirror can contain both layout version 2 and version 3 simultaneously without
-    interference. This command performs in-place migration of a binary mirror laid
-    out according to version 2, to a binary mirror laid out according to layout
-    version 3.  Only indexed specs will be migrated, so consider updating the mirror
-    index before running this command.  Re-run the command to migrate any missing
-    items.
-
-    The default mode of operation is to perform a signed migration, that is, spack
-    will attempt to verify the signatures on specs, and then re-sign them before
-    migration, using whatever keys are already installed in your key ring.  You can
-    migrate a mirror of unsigned binaries (or convert a mirror of signed binaries
-    to unsigned) by providing the --unsigned argument.
-
-    By default spack will leave the original mirror contents (in the old layout) in
-    place after migration. You can have spack remove the old contents by providing
-    the --delete-existing argument.  Because migrating a mostly-already-migrated
-    mirror should be fast, consider a workflow where you perform a default migration,
-    (i.e. preserve the existing layout rather than deleting it) then evaluate the
-    state of the migrated mirror by attempting to install from it, and finally
-    running the migration again with --delete-existing."""
-    target_mirror = args.mirror
-    unsigned = args.unsigned
-    assert isinstance(target_mirror, spack.mirrors.mirror.Mirror)
-    delete_existing = args.delete_existing
-
-    proceed = True
-    if delete_existing and not args.yes_to_all:
-        msg = (
-            "Using --delete-existing will delete the entire contents \n"
-            "    of the old layout within the mirror. Because migrating a mirror \n"
-            "    that has already been migrated should be fast, consider a workflow \n"
-            "    where you perform a default migration (i.e. preserve the existing \n"
-            "    layout rather than deleting it), then evaluate the state of the \n"
-            "    migrated mirror by attempting to install from it, and finally, \n"
-            "    run the migration again with --delete-existing."
-        )
-        tty.warn(msg)
-        proceed = tty.get_yes_or_no("Do you want to proceed?", default=False)
-
-    if not proceed:
-        tty.die("Migration aborted.")
-
-    migrate(target_mirror, unsigned=unsigned, delete_existing=delete_existing)
-
-
 def buildcache(parser, args):
     return args.func(args)

lib/spack/spack/cmd/ci.py

@@ -160,12 +160,6 @@ def setup_parser(subparser):
         default=False,
         help="stop stand-alone tests after the first failure",
     )
-    rebuild.add_argument(
-        "--timeout",
-        type=int,
-        default=None,
-        help="maximum time (in seconds) that tests are allowed to run",
-    )
     rebuild.set_defaults(func=ci_rebuild)
     spack.cmd.common.arguments.add_common_arguments(rebuild, ["jobs"])
 
@@ -423,7 +417,7 @@ def ci_rebuild(args):
         # jobs in subsequent stages.
         tty.msg("No need to rebuild {0}, found hash match at: ".format(job_spec_pkg_name))
         for match in matches:
-            tty.msg("    {0}".format(match.url_and_version.url))
+            tty.msg("    {0}".format(match["mirror_url"]))
 
         # Now we are done and successful
         return 0
@@ -453,7 +447,7 @@ def ci_rebuild(args):
 
     # Arguments when installing the root from sources
     deps_install_args = install_args + ["--only=dependencies"]
-    root_install_args = install_args + ["--keep-stage", "--only=package"]
+    root_install_args = install_args + ["--only=package"]
 
     if cdash_handler:
         # Add additional arguments to `spack install` for CDash reporting.
@@ -493,9 +487,6 @@ def ci_rebuild(args):
     # Copy logs and archived files from the install metadata (.spack) directory to artifacts now
     spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir)
 
-    # Clear the stage directory
-    spack.stage.purge()
-
     # If the installation succeeded and we're running stand-alone tests for
     # the package, run them and copy the output. Failures of any kind should
     # *not* terminate the build process or preclude creating the build cache.
@@ -530,7 +521,6 @@ def ci_rebuild(args):
                     fail_fast=args.fail_fast,
                     log_file=log_file,
                     repro_dir=repro_dir,
-                    timeout=args.timeout,
                 )
 
             except Exception as err:
@@ -791,9 +781,7 @@ def ci_verify_versions(args):
     """
     # Get a list of all packages that have been changed or added
     # between from_ref and to_ref
-    pkgs = spack.repo.get_all_package_diffs(
-        "AC", spack.repo.builtin_repo(), args.from_ref, args.to_ref
-    )
+    pkgs = spack.repo.get_all_package_diffs("AC", args.from_ref, args.to_ref)
 
     failed_version = False
     for pkg_name in pkgs:

lib/spack/spack/cmd/compiler.py

@@ -63,7 +63,7 @@ def setup_parser(subparser):
     )
 
     # List
-    list_parser = sp.add_parser("list", aliases=["ls"], help="list available compilers")
+    list_parser = sp.add_parser("list", help="list available compilers")
     list_parser.add_argument(
         "--scope", action=arguments.ConfigScope, help="configuration scope to read from"
     )
@@ -216,6 +216,5 @@ def compiler(parser, args):
         "rm": compiler_remove,
         "info": compiler_info,
         "list": compiler_list,
-        "ls": compiler_list,
     }
     action[args.compiler_command](args)

lib/spack/spack/cmd/create.py

@@ -23,7 +23,7 @@
 from spack.util.editor import editor
 from spack.util.executable import which
 from spack.util.format import get_version_lines
-from spack.util.naming import pkg_name_to_class_name, simplify_name
+from spack.util.naming import mod_to_class, simplify_name, valid_fully_qualified_module_name
 
 description = "create a new package file"
 section = "packaging"
@@ -95,7 +95,7 @@ class BundlePackageTemplate:
 
     def __init__(self, name: str, versions, languages: List[str]):
         self.name = name
-        self.class_name = pkg_name_to_class_name(name)
+        self.class_name = mod_to_class(name)
         self.versions = versions
         self.languages = languages
 
@@ -874,7 +874,7 @@ def get_name(name, url):
 
     result = simplify_name(result)
 
-    if not re.match(r"^[a-z0-9-]+$", result):
+    if not valid_fully_qualified_module_name(result):
         tty.die("Package name can only contain a-z, 0-9, and '-'")
 
     return result

lib/spack/spack/cmd/develop.py

@@ -102,7 +102,7 @@ def assure_concrete_spec(env: spack.environment.Environment, spec: spack.spec.Sp
                     )
         else:
             # look up the maximum version so infintiy versions are preferred for develop
-            version = max(spack.repo.PATH.get_pkg_class(spec.fullname).versions.keys())
+            version = max(spec.package_class.versions.keys())
             tty.msg(f"Defaulting to highest version: {spec.name}@{version}")
     spec.versions = spack.version.VersionList([version])
 

lib/spack/spack/cmd/external.py

@@ -62,7 +62,7 @@ def setup_parser(subparser):
         "package Spack knows how to find."
     )
 
-    sp.add_parser("list", aliases=["ls"], help="list detectable packages, by repository and name")
+    sp.add_parser("list", help="list detectable packages, by repository and name")
 
     read_cray_manifest = sp.add_parser(
         "read-cray-manifest",
@@ -259,7 +259,6 @@ def external(parser, args):
     action = {
         "find": external_find,
         "list": external_list,
-        "ls": external_list,
         "read-cray-manifest": external_read_cray_manifest,
     }
     action[args.external_command](args)

lib/spack/spack/cmd/find.py

@@ -51,12 +51,6 @@ def setup_parser(subparser):
         "-I", "--install-status", action="store_true", help="show install status of packages"
     )
 
-    subparser.add_argument(
-        "--specfile-format",
-        action="store_true",
-        help="show the specfile format for installed deps ",
-    )
-
     subparser.add_argument(
         "-d", "--deps", action="store_true", help="output dependencies along with found specs"
     )
@@ -286,7 +280,6 @@ def root_decorator(spec, string):
             show_flags=True,
             decorator=root_decorator,
             variants=True,
-            specfile_format=args.specfile_format,
         )
 
     print()
@@ -308,7 +301,6 @@ def root_decorator(spec, string):
             namespace=True,
             show_flags=True,
             variants=True,
-            specfile_format=args.specfile_format,
         )
         print()
 
@@ -398,12 +390,7 @@ def find(parser, args):
             if args.show_concretized:
                 display_results += concretized_but_not_installed
             cmd.display_specs(
-                display_results,
-                args,
-                decorator=decorator,
-                all_headers=True,
-                status_fn=status_fn,
-                specfile_format=args.specfile_format,
+                display_results, args, decorator=decorator, all_headers=True, status_fn=status_fn
             )
 
         # print number of installed packages last (as the list may be long)

lib/spack/spack/cmd/list.py

@@ -10,13 +10,11 @@
 import re
 import sys
 from html import escape
-from typing import Type
 
 import llnl.util.tty as tty
 from llnl.util.tty.colify import colify
 
 import spack.deptypes as dt
-import spack.package_base
 import spack.repo
 from spack.cmd.common import arguments
 from spack.version import VersionList
@@ -141,10 +139,10 @@ def name_only(pkgs, out):
         tty.msg("%d packages" % len(pkgs))
 
 
-def github_url(pkg: Type[spack.package_base.PackageBase]) -> str:
+def github_url(pkg):
     """Link to a package file on github."""
-    mod_path = pkg.__module__.replace(".", "/")
-    return f"https://github.com/spack/spack/blob/develop/var/spack/{mod_path}.py"
+    url = "https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/{0}/package.py"
+    return url.format(pkg.name)
 
 
 def rows_for_ncols(elts, ncols):

lib/spack/spack/cmd/pkg.py

@@ -89,17 +89,17 @@ def setup_parser(subparser):
 
 def pkg_add(args):
     """add a package to the git stage with `git add`"""
-    spack.repo.add_package_to_git_stage(args.packages, spack.repo.builtin_repo())
+    spack.repo.add_package_to_git_stage(args.packages)
 
 
 def pkg_list(args):
     """list packages associated with a particular spack git revision"""
-    colify(spack.repo.list_packages(args.rev, spack.repo.builtin_repo()))
+    colify(spack.repo.list_packages(args.rev))
 
 
 def pkg_diff(args):
     """compare packages available in two different git revisions"""
-    u1, u2 = spack.repo.diff_packages(args.rev1, args.rev2, spack.repo.builtin_repo())
+    u1, u2 = spack.repo.diff_packages(args.rev1, args.rev2)
 
     if u1:
         print("%s:" % args.rev1)
@@ -114,23 +114,21 @@ def pkg_diff(args):
 
 def pkg_removed(args):
     """show packages removed since a commit"""
-    u1, u2 = spack.repo.diff_packages(args.rev1, args.rev2, spack.repo.builtin_repo())
+    u1, u2 = spack.repo.diff_packages(args.rev1, args.rev2)
     if u1:
         colify(sorted(u1))
 
 
 def pkg_added(args):
     """show packages added since a commit"""
-    u1, u2 = spack.repo.diff_packages(args.rev1, args.rev2, spack.repo.builtin_repo())
+    u1, u2 = spack.repo.diff_packages(args.rev1, args.rev2)
     if u2:
         colify(sorted(u2))
 
 
 def pkg_changed(args):
     """show packages changed since a commit"""
-    packages = spack.repo.get_all_package_diffs(
-        args.type, spack.repo.builtin_repo(), args.rev1, args.rev2
-    )
+    packages = spack.repo.get_all_package_diffs(args.type, args.rev1, args.rev2)
 
     if packages:
         colify(sorted(packages))

lib/spack/spack/cmd/repo.py

@@ -4,7 +4,6 @@
 
 import os
 import sys
-from typing import List
 
 import llnl.util.tty as tty
 
@@ -25,7 +24,9 @@ def setup_parser(subparser):
     create_parser = sp.add_parser("create", help=repo_create.__doc__)
     create_parser.add_argument("directory", help="directory to create the repo in")
     create_parser.add_argument(
-        "namespace", help="name or namespace to identify packages in the repository"
+        "namespace",
+        help="namespace to identify packages in the repository (defaults to the directory name)",
+        nargs="?",
     )
     create_parser.add_argument(
         "-d",
@@ -137,7 +138,7 @@ def repo_remove(args):
 def repo_list(args):
     """show registered repositories and their namespaces"""
     roots = spack.config.get("repos", scope=args.scope)
-    repos: List[spack.repo.Repo] = []
+    repos = []
     for r in roots:
         try:
             repos.append(spack.repo.from_path(r))
@@ -145,14 +146,17 @@ def repo_list(args):
             continue
 
     if sys.stdout.isatty():
-        tty.msg(f"{len(repos)} package repositor" + ("y." if len(repos) == 1 else "ies."))
+        msg = "%d package repositor" % len(repos)
+        msg += "y." if len(repos) == 1 else "ies."
+        tty.msg(msg)
 
     if not repos:
         return
 
     max_ns_len = max(len(r.namespace) for r in repos)
     for repo in repos:
-        print(f"{repo.namespace:<{max_ns_len + 4}}{repo.package_api_str:<8}{repo.root}")
+        fmt = "%%-%ds%%s" % (max_ns_len + 4)
+        print(fmt % (repo.namespace, repo.root))
 
 
 def repo(parser, args):

lib/spack/spack/cmd/solve.py

@@ -136,7 +136,20 @@ def solve(parser, args):
     setup_only = set(show) == {"asp"}
     unify = spack.config.get("concretizer:unify")
     allow_deprecated = spack.config.get("config:deprecated", False)
-    if unify == "when_possible":
+    if unify != "when_possible":
+        # set up solver parameters
+        # Note: reuse and other concretizer prefs are passed as configuration
+        result = solver.solve(
+            specs,
+            out=output,
+            timers=args.timers,
+            stats=args.stats,
+            setup_only=setup_only,
+            allow_deprecated=allow_deprecated,
+        )
+        if not setup_only:
+            _process_result(result, show, required_format, kwargs)
+    else:
         for idx, result in enumerate(
             solver.solve_in_rounds(
                 specs,
@@ -153,29 +166,3 @@ def solve(parser, args):
                 print("% END ROUND {0}\n".format(idx))
             if not setup_only:
                 _process_result(result, show, required_format, kwargs)
-    elif unify:
-        # set up solver parameters
-        # Note: reuse and other concretizer prefs are passed as configuration
-        result = solver.solve(
-            specs,
-            out=output,
-            timers=args.timers,
-            stats=args.stats,
-            setup_only=setup_only,
-            allow_deprecated=allow_deprecated,
-        )
-        if not setup_only:
-            _process_result(result, show, required_format, kwargs)
-    else:
-        for spec in specs:
-            tty.msg("SOLVING SPEC:", spec)
-            result = solver.solve(
-                [spec],
-                out=output,
-                timers=args.timers,
-                stats=args.stats,
-                setup_only=setup_only,
-                allow_deprecated=allow_deprecated,
-            )
-            if not setup_only:
-                _process_result(result, show, required_format, kwargs)

lib/spack/spack/cmd/style.py

@@ -56,10 +56,10 @@ def is_package(f):
     """Whether flake8 should consider a file as a core file or a package.
 
     We run flake8 with different exceptions for the core and for
-    packages, since we allow `from spack.package import *` and poking globals
+    packages, since we allow `from spack import *` and poking globals
     into packages.
     """
-    return f.startswith("var/spack/") and f.endswith("package.py")
+    return f.startswith("var/spack/repos/") and f.endswith("package.py")
 
 
 #: decorator for adding tools to the list
@@ -380,7 +380,7 @@ def run_black(black_cmd, file_list, args):
 def _module_part(root: str, expr: str):
     parts = expr.split(".")
     # spack.pkg is for repositories, don't try to resolve it here.
-    if expr.startswith(spack.repo.PKG_MODULE_PREFIX_V1) or expr == "spack.pkg":
+    if ".".join(parts[:2]) == spack.repo.ROOT_PYTHON_NAMESPACE:
         return None
     while parts:
         f1 = os.path.join(root, "lib", "spack", *parts) + ".py"

lib/spack/spack/cmd/test.py

@@ -65,12 +65,6 @@ def setup_parser(subparser):
     run_parser.add_argument(
         "--help-cdash", action="store_true", help="show usage instructions for CDash reporting"
     )
-    run_parser.add_argument(
-        "--timeout",
-        type=int,
-        default=None,
-        help="maximum time (in seconds) that tests are allowed to run",
-    )
 
     cd_group = run_parser.add_mutually_exclusive_group()
     arguments.add_common_arguments(cd_group, ["clean", "dirty"])
@@ -182,7 +176,7 @@ def test_run(args):
     for spec in specs:
         matching = spack.store.STORE.db.query_local(spec, hashes=hashes, explicit=explicit)
         if spec and not matching:
-            tty.warn(f"No {explicit_str}installed packages match spec {spec}")
+            tty.warn("No {0}installed packages match spec {1}".format(explicit_str, spec))
 
             # TODO: Need to write out a log message and/or CDASH Testing
             #   output that package not installed IF continue to process
@@ -198,7 +192,7 @@ def test_run(args):
     # test_stage_dir
     test_suite = spack.install_test.TestSuite(specs_to_test, args.alias)
     test_suite.ensure_stage()
-    tty.msg(f"Spack test {test_suite.name}")
+    tty.msg("Spack test %s" % test_suite.name)
 
     # Set up reporter
     setattr(args, "package", [s.format() for s in test_suite.specs])
@@ -210,7 +204,6 @@ def test_run(args):
             dirty=args.dirty,
             fail_first=args.fail_first,
             externals=args.externals,
-            timeout=args.timeout,
         )
 
 

lib/spack/spack/compilers/adaptor.py

@@ -18,10 +18,6 @@ class Languages(enum.Enum):
 
 
 class CompilerAdaptor:
-    """Provides access to compiler attributes via `Package.compiler`. Useful for
-    packages which do not yet access compiler properties via `self.spec[language]`.
-    """
-
     def __init__(
         self, compiled_spec: spack.spec.Spec, compilers: Dict[Languages, spack.spec.Spec]
     ) -> None:
@@ -83,14 +79,6 @@ def implicit_rpaths(self) -> List[str]:
             result.extend(CompilerPropertyDetector(compiler).implicit_rpaths())
         return result
 
-    @property
-    def opt_flags(self) -> List[str]:
-        return next(iter(self.compilers.values())).package.opt_flags
-
-    @property
-    def debug_flags(self) -> List[str]:
-        return next(iter(self.compilers.values())).package.debug_flags
-
     @property
     def openmp_flag(self) -> str:
         return next(iter(self.compilers.values())).package.openmp_flag
@@ -152,7 +140,7 @@ def c17_flag(self) -> str:
     @property
     def c23_flag(self) -> str:
         return self.compilers[Languages.C].package.standard_flag(
-            language=Languages.C.value, standard="23"
+            language=Languages.C.value, standard="17"
         )
 
     @property
@@ -202,10 +190,6 @@ def f77(self):
         self._lang_exists_or_raise("f77", lang=Languages.FORTRAN)
         return self.compilers[Languages.FORTRAN].package.fortran
 
-    @property
-    def stdcxx_libs(self):
-        return self._maybe_return_attribute("stdcxx_libs", lang=Languages.CXX)
-
 
 class DeprecatedCompiler(lang.DeprecatedProperty):
     def __init__(self) -> None:

lib/spack/spack/database.py

@@ -149,12 +149,12 @@ def _getfqdn():
     return socket.getfqdn()
 
 
-def reader(version: vn.StandardVersion) -> Type["spack.spec.SpecfileReaderBase"]:
+def reader(version: vn.ConcreteVersion) -> Type["spack.spec.SpecfileReaderBase"]:
     reader_cls = {
-        vn.StandardVersion.from_string("5"): spack.spec.SpecfileV1,
-        vn.StandardVersion.from_string("6"): spack.spec.SpecfileV3,
-        vn.StandardVersion.from_string("7"): spack.spec.SpecfileV4,
-        vn.StandardVersion.from_string("8"): spack.spec.SpecfileV5,
+        vn.Version("5"): spack.spec.SpecfileV1,
+        vn.Version("6"): spack.spec.SpecfileV3,
+        vn.Version("7"): spack.spec.SpecfileV4,
+        vn.Version("8"): spack.spec.SpecfileV5,
     }
     return reader_cls[version]
 
@@ -824,7 +824,7 @@ def check(cond, msg):
         db = fdata["database"]
         check("version" in db, "no 'version' in JSON DB.")
 
-        self.db_version = vn.StandardVersion.from_string(db["version"])
+        self.db_version = vn.Version(db["version"])
         if self.db_version > _DB_VERSION:
             raise InvalidDatabaseVersionError(self, _DB_VERSION, self.db_version)
         elif self.db_version < _DB_VERSION:

lib/spack/spack/detection/common.py

@@ -20,7 +20,7 @@
 import sys
 from typing import Dict, List, Optional, Set, Tuple, Union
 
-from llnl.util import tty
+import llnl.util.tty
 
 import spack.config
 import spack.error
@@ -93,13 +93,14 @@ def _spec_is_valid(spec: spack.spec.Spec) -> bool:
     except spack.error.SpackError:
         # It is assumed here that we can at least extract the package name from the spec so we
         # can look up the implementation of determine_spec_details
-        tty.warn(f"Constructed spec for {spec.name} does not have a string representation")
+        msg = f"Constructed spec for {spec.name} does not have a string representation"
+        llnl.util.tty.warn(msg)
         return False
 
     try:
         spack.spec.Spec(str(spec))
     except spack.error.SpackError:
-        tty.warn(
+        llnl.util.tty.warn(
             "Constructed spec has a string representation but the string"
             " representation does not evaluate to a valid spec: {0}".format(str(spec))
         )
@@ -108,24 +109,20 @@ def _spec_is_valid(spec: spack.spec.Spec) -> bool:
     return True
 
 
-def path_to_dict(search_paths: List[str]) -> Dict[str, str]:
+def path_to_dict(search_paths: List[str]):
     """Return dictionary[fullpath]: basename from list of paths"""
-    path_to_lib: Dict[str, str] = {}
+    path_to_lib = {}
     # Reverse order of search directories so that a lib in the first
     # entry overrides later entries
     for search_path in reversed(search_paths):
         try:
-            dir_iter = os.scandir(search_path)
+            with os.scandir(search_path) as entries:
+                path_to_lib.update(
+                    {entry.path: entry.name for entry in entries if entry.is_file()}
+                )
         except OSError as e:
-            tty.debug(f"cannot scan '{search_path}' for external software: {e}")
-            continue
-        with dir_iter as entries:
-            for entry in entries:
-                try:
-                    if entry.is_file():
-                        path_to_lib[entry.path] = entry.name
-                except OSError as e:
-                    tty.debug(f"cannot scan '{search_path}' for external software: {e}")
+            msg = f"cannot scan '{search_path}' for external software: {str(e)}"
+            llnl.util.tty.debug(msg)
 
     return path_to_lib
 

lib/spack/spack/directives.py

@@ -34,13 +34,11 @@ class OpenMpi(Package):
 import collections.abc
 import os
 import re
-import warnings
 from typing import Any, Callable, List, Optional, Tuple, Type, Union
 
 import llnl.util.tty.color
 
 import spack.deptypes as dt
-import spack.error
 import spack.fetch_strategy
 import spack.package_base
 import spack.patch
@@ -610,7 +608,7 @@ def _execute_patch(
     return _execute_patch
 
 
-def conditional(*values: Union[str, bool], when: Optional[WhenType] = None):
+def conditional(*values: List[Any], when: Optional[WhenType] = None):
     """Conditional values that can be used in variant declarations."""
     # _make_when_spec returns None when the condition is statically false.
     when = _make_when_spec(when)
@@ -622,7 +620,7 @@ def conditional(*values: Union[str, bool], when: Optional[WhenType] = None):
 @directive("variants")
 def variant(
     name: str,
-    default: Optional[Union[bool, str, Tuple[str, ...]]] = None,
+    default: Optional[Any] = None,
     description: str = "",
     values: Optional[Union[collections.abc.Sequence, Callable[[Any], bool]]] = None,
     multi: Optional[bool] = None,
@@ -652,24 +650,6 @@ def variant(
         DirectiveError: If arguments passed to the directive are invalid
     """
 
-    # This validation can be removed at runtime and enforced with an audit in Spack v1.0.
-    # For now it's a warning to let people migrate faster.
-    if not (
-        default is None
-        or type(default) in (bool, str)
-        or (type(default) is tuple and all(type(x) is str for x in default))
-    ):
-        if isinstance(default, (list, tuple)):
-            did_you_mean = f"default={','.join(str(x) for x in default)!r}"
-        else:
-            did_you_mean = f"default={str(default)!r}"
-        warnings.warn(
-            f"default value for variant '{name}' is not a boolean or string: default={default!r}. "
-            f"Did you mean {did_you_mean}?",
-            stacklevel=3,
-            category=spack.error.SpackAPIWarning,
-        )
-
     def format_error(msg, pkg):
         msg += " @*r{{[{0}, variant '{1}']}}"
         return llnl.util.tty.color.colorize(msg.format(pkg.name, name))
@@ -685,11 +665,7 @@ def _raise_reserved_name(pkg):
     # Ensure we have a sequence of allowed variant values, or a
     # predicate for it.
     if values is None:
-        if (
-            default in (True, False)
-            or type(default) is str
-            and default.upper() in ("TRUE", "FALSE")
-        ):
+        if str(default).upper() in ("TRUE", "FALSE"):
             values = (True, False)
         else:
             values = lambda x: True
@@ -722,15 +698,12 @@ def _raise_argument_error(pkg):
     # or the empty string, as the former indicates that a default
     # was not set while the latter will make the variant unparsable
     # from the command line
-    if isinstance(default, tuple):
-        default = ",".join(default)
-
     if default is None or default == "":
 
         def _raise_default_not_set(pkg):
             if default is None:
-                msg = "either a default was not explicitly set, or 'None' was used"
-            else:
+                msg = "either a default was not explicitly set, " "or 'None' was used"
+            elif default == "":
                 msg = "the default cannot be an empty string"
             raise DirectiveError(format_error(msg, pkg))
 

lib/spack/spack/directives_meta.py

@@ -65,7 +65,7 @@ def __init__(cls: "DirectiveMeta", name: str, bases: tuple, attr_dict: dict):
         # The instance is being initialized: if it is a package we must ensure
         # that the directives are called to set it up.
 
-        if spack.repo.is_package_module(cls.__module__):
+        if cls.__module__.startswith(spack.repo.ROOT_PYTHON_NAMESPACE):
             # Ensure the presence of the dictionaries associated with the directives.
             # All dictionaries are defaultdicts that create lists for missing keys.
             for d in DirectiveMeta._directive_dict_names:
@@ -144,6 +144,7 @@ class Foo(Package):
         Package class, and it's how Spack gets information from the
         packages to the core.
         """
+        global directive_names
 
         if isinstance(dicts, str):
             dicts = (dicts,)

lib/spack/spack/environment/__init__.py

@@ -566,7 +566,7 @@
     display_specs,
     environment_dir_from_name,
     environment_from_name_or_dir,
-    environment_path_scope,
+    environment_path_scopes,
     exists,
     initialize_environment_dir,
     installed_specs,
@@ -603,7 +603,7 @@
     "display_specs",
     "environment_dir_from_name",
     "environment_from_name_or_dir",
-    "environment_path_scope",
+    "environment_path_scopes",
     "exists",
     "initialize_environment_dir",
     "installed_specs",

lib/spack/spack/environment/environment.py

@@ -31,6 +31,7 @@
 import spack.repo
 import spack.schema.env
 import spack.spec
+import spack.spec_list
 import spack.store
 import spack.user_environment as uenv
 import spack.util.environment
@@ -43,10 +44,10 @@
 from spack.installer import PackageInstaller
 from spack.schema.env import TOP_LEVEL_KEY
 from spack.spec import Spec
+from spack.spec_list import SpecList
 from spack.util.path import substitute_path_variables
 
 from ..enums import ConfigScopePriority
-from .list import SpecList, SpecListError, SpecListParser
 
 SpecPair = spack.concretize.SpecPair
 
@@ -96,15 +97,16 @@ def environment_name(path: Union[str, pathlib.Path]) -> str:
         return path_str
 
 
-def ensure_no_disallowed_env_config_mods(scope: spack.config.ConfigScope) -> None:
-    config = scope.get_section("config")
-    if config and "environments_root" in config["config"]:
-        raise SpackEnvironmentError(
-            "Spack environments are prohibited from modifying 'config:environments_root' "
-            "because it can make the definition of the environment ill-posed. Please "
-            "remove from your environment and place it in a permanent scope such as "
-            "defaults, system, site, etc."
-        )
+def ensure_no_disallowed_env_config_mods(scopes: List[spack.config.ConfigScope]) -> None:
+    for scope in scopes:
+        config = scope.get_section("config")
+        if config and "environments_root" in config["config"]:
+            raise SpackEnvironmentError(
+                "Spack environments are prohibited from modifying 'config:environments_root' "
+                "because it can make the definition of the environment ill-posed. Please "
+                "remove from your environment and place it in a permanent scope such as "
+                "defaults, system, site, etc."
+            )
 
 
 def default_manifest_yaml():
@@ -931,10 +933,8 @@ def __init__(self, manifest_dir: Union[str, pathlib.Path]) -> None:
         self.new_specs: List[Spec] = []
         self.views: Dict[str, ViewDescriptor] = {}
 
-        #: Parser for spec lists
-        self._spec_lists_parser = SpecListParser()
         #: Specs from "spack.yaml"
-        self.spec_lists: Dict[str, SpecList] = {}
+        self.spec_lists: Dict[str, SpecList] = {user_speclist_name: SpecList()}
         #: User specs from the last concretization
         self.concretized_user_specs: List[Spec] = []
         #: Roots associated with the last concretization, in order
@@ -1002,6 +1002,26 @@ def write_transaction(self):
         """Get a write lock context manager for use in a `with` block."""
         return lk.WriteTransaction(self.txlock, acquire=self._re_read)
 
+    def _process_definition(self, entry):
+        """Process a single spec definition item."""
+        when_string = entry.get("when")
+        if when_string is not None:
+            when = spack.spec.eval_conditional(when_string)
+            assert len([x for x in entry if x != "when"]) == 1
+        else:
+            when = True
+            assert len(entry) == 1
+
+        if when:
+            for name, spec_list in entry.items():
+                if name == "when":
+                    continue
+                user_specs = SpecList(name, spec_list, self.spec_lists.copy())
+                if name in self.spec_lists:
+                    self.spec_lists[name].extend(user_specs)
+                else:
+                    self.spec_lists[name] = user_specs
+
     def _process_view(self, env_view: Optional[Union[bool, str, Dict]]):
         """Process view option(s), which can be boolean, string, or None.
 
@@ -1049,11 +1069,7 @@ def add_view(name, values):
 
     def _process_concrete_includes(self):
         """Extract and load into memory included concrete spec data."""
-        _included_concrete_envs = self.manifest[TOP_LEVEL_KEY].get(included_concrete_name, [])
-        # Expand config and environment variables
-        self.included_concrete_envs = [
-            spack.util.path.canonicalize_path(_env) for _env in _included_concrete_envs
-        ]
+        self.included_concrete_envs = self.manifest[TOP_LEVEL_KEY].get(included_concrete_name, [])
 
         if self.included_concrete_envs:
             if os.path.exists(self.lock_path):
@@ -1067,24 +1083,21 @@ def _process_concrete_includes(self):
 
     def _construct_state_from_manifest(self):
         """Set up user specs and views from the manifest file."""
+        self.spec_lists = collections.OrderedDict()
         self.views = {}
-        self._sync_speclists()
-        self._process_view(spack.config.get("view", True))
-        self._process_concrete_includes()
 
-    def _sync_speclists(self):
-        self.spec_lists = {}
-        self.spec_lists.update(
-            self._spec_lists_parser.parse_definitions(
-                data=spack.config.CONFIG.get("definitions", [])
-            )
-        )
+        for item in spack.config.get("definitions", []):
+            self._process_definition(item)
 
         env_configuration = self.manifest[TOP_LEVEL_KEY]
         spec_list = env_configuration.get(user_speclist_name, [])
-        self.spec_lists[user_speclist_name] = self._spec_lists_parser.parse_user_specs(
-            name=user_speclist_name, yaml_list=spec_list
+        user_specs = SpecList(
+            user_speclist_name, [s for s in spec_list if s], self.spec_lists.copy()
         )
+        self.spec_lists[user_speclist_name] = user_specs
+
+        self._process_view(spack.config.get("view", True))
+        self._process_concrete_includes()
 
     def all_concretized_user_specs(self) -> List[Spec]:
         """Returns all of the concretized user specs of the environment and
@@ -1155,7 +1168,9 @@ def clear(self, re_read=False):
             re_read: If ``True``, do not clear ``new_specs``. This value cannot be read from yaml,
                 and needs to be maintained when re-reading an existing environment.
         """
-        self.spec_lists = {}
+        self.spec_lists = collections.OrderedDict()
+        self.spec_lists[user_speclist_name] = SpecList()
+
         self._dev_specs = {}
         self.concretized_order = []  # roots of last concretize, in order
         self.concretized_user_specs = []  # user specs from last concretize
@@ -1262,6 +1277,22 @@ def destroy(self):
         """Remove this environment from Spack entirely."""
         shutil.rmtree(self.path)
 
+    def update_stale_references(self, from_list=None):
+        """Iterate over spec lists updating references."""
+        if not from_list:
+            from_list = next(iter(self.spec_lists.keys()))
+        index = list(self.spec_lists.keys()).index(from_list)
+
+        # spec_lists is an OrderedDict to ensure lists read from the manifest
+        # are maintainted in order, hence, all list entries after the modified
+        # list may refer to the modified list requiring stale references to be
+        # updated.
+        for i, (name, speclist) in enumerate(
+            list(self.spec_lists.items())[index + 1 :], index + 1
+        ):
+            new_reference = dict((n, self.spec_lists[n]) for n in list(self.spec_lists.keys())[:i])
+            speclist.update_reference(new_reference)
+
     def add(self, user_spec, list_name=user_speclist_name):
         """Add a single user_spec (non-concretized) to the Environment
 
@@ -1281,17 +1312,18 @@ def add(self, user_spec, list_name=user_speclist_name):
             elif not spack.repo.PATH.exists(spec.name) and not spec.abstract_hash:
                 virtuals = spack.repo.PATH.provider_index.providers.keys()
                 if spec.name not in virtuals:
-                    raise SpackEnvironmentError(f"no such package: {spec.name}")
+                    msg = "no such package: %s" % spec.name
+                    raise SpackEnvironmentError(msg)
 
         list_to_change = self.spec_lists[list_name]
         existing = str(spec) in list_to_change.yaml_list
         if not existing:
             list_to_change.add(str(spec))
+            self.update_stale_references(list_name)
             if list_name == user_speclist_name:
                 self.manifest.add_user_spec(str(user_spec))
             else:
                 self.manifest.add_definition(str(user_spec), list_name=list_name)
-            self._sync_speclists()
 
         return bool(not existing)
 
@@ -1335,17 +1367,18 @@ def change_existing_spec(
                 "There are no specs named {0} in {1}".format(match_spec.name, list_name)
             )
         elif len(matches) > 1 and not allow_changing_multiple_specs:
-            raise ValueError(f"{str(match_spec)} matches multiple specs")
+            raise ValueError("{0} matches multiple specs".format(str(match_spec)))
 
         for idx, spec in matches:
             override_spec = Spec.override(spec, change_spec)
+            self.spec_lists[list_name].replace(idx, str(override_spec))
             if list_name == user_speclist_name:
                 self.manifest.override_user_spec(str(override_spec), idx=idx)
             else:
                 self.manifest.override_definition(
                     str(spec), override=str(override_spec), list_name=list_name
                 )
-        self._sync_speclists()
+        self.update_stale_references(from_list=list_name)
 
     def remove(self, query_spec, list_name=user_speclist_name, force=False):
         """Remove specs from an environment that match a query_spec"""
@@ -1373,17 +1406,22 @@ def remove(self, query_spec, list_name=user_speclist_name, force=False):
             raise SpackEnvironmentError(f"{err_msg_header}, no spec matches")
 
         old_specs = set(self.user_specs)
-
-        # Remove specs from the appropriate spec list
+        new_specs = set()
         for spec in matches:
             if spec not in list_to_change:
                 continue
             try:
                 list_to_change.remove(spec)
-            except SpecListError as e:
+                self.update_stale_references(list_name)
+                new_specs = set(self.user_specs)
+            except spack.spec_list.SpecListError as e:
+                # define new specs list
+                new_specs = set(self.user_specs)
                 msg = str(e)
                 if force:
                     msg += " It will be removed from the concrete specs."
+                    # Mock new specs, so we can remove this spec from concrete spec lists
+                    new_specs.remove(spec)
                 tty.warn(msg)
             else:
                 if list_name == user_speclist_name:
@@ -1391,11 +1429,7 @@ def remove(self, query_spec, list_name=user_speclist_name, force=False):
                 else:
                     self.manifest.remove_definition(str(spec), list_name=list_name)
 
-        # Recompute "definitions" and user specs
-        self._sync_speclists()
-        new_specs = set(self.user_specs)
-
-        # If 'force', update stale concretized specs
+        # If force, update stale concretized specs
         for spec in old_specs - new_specs:
             if force and spec in self.concretized_user_specs:
                 i = self.concretized_user_specs.index(spec)
@@ -1609,6 +1643,23 @@ def _concretize_separately(self, tests=False):
 
         # Unify the specs objects, so we get correct references to all parents
         self._read_lockfile_dict(self._to_lockfile_dict())
+
+        # Re-attach information on test dependencies
+        if tests:
+            # This is slow, but the information on test dependency is lost
+            # after unification or when reading from a lockfile.
+            for h in self.specs_by_hash:
+                current_spec, computed_spec = self.specs_by_hash[h], by_hash[h]
+                for node in computed_spec.traverse():
+                    test_edges = node.edges_to_dependencies(depflag=dt.TEST)
+                    for current_edge in test_edges:
+                        test_dependency = current_edge.spec
+                        if test_dependency in current_spec[node.name]:
+                            continue
+                        current_spec[node.name].add_dependency_edge(
+                            test_dependency.copy(), depflag=dt.TEST, virtuals=current_edge.virtuals
+                        )
+
         return concretized_specs
 
     @property
@@ -2316,12 +2367,8 @@ def update_environment_repository(self) -> None:
 
     def _add_to_environment_repository(self, spec_node: Spec) -> None:
         """Add the root node of the spec to the environment repository"""
-        namespace: str = spec_node.namespace
-        repository = spack.repo.create_or_construct(
-            root=os.path.join(self.repos_path, namespace),
-            namespace=namespace,
-            package_api=spack.repo.PATH.get_repo(namespace).package_api,
-        )
+        repository_dir = os.path.join(self.repos_path, spec_node.namespace)
+        repository = spack.repo.create_or_construct(repository_dir, spec_node.namespace)
         pkg_dir = repository.dirname_for_package_name(spec_node.name)
         fs.mkdirp(pkg_dir)
         spack.repo.PATH.dump_provenance(spec_node, pkg_dir)
@@ -2670,9 +2717,9 @@ def __init__(self, manifest_dir: Union[pathlib.Path, str], name: Optional[str] =
         self.scope_name = f"env:{self.name}"
         self.config_stage_dir = os.path.join(env_subdir_path(manifest_dir), "config")
 
-        #: Configuration scope associated with this environment. Note that this is not
+        #: Configuration scopes associated with this environment. Note that these are not
         #: invalidated by a re-read of the manifest file.
-        self._env_config_scope: Optional[spack.config.ConfigScope] = None
+        self._config_scopes: Optional[List[spack.config.ConfigScope]] = None
 
         if not self.manifest_file.exists():
             msg = f"cannot find '{manifest_name}' in {self.manifest_dir}"
@@ -2781,8 +2828,6 @@ def add_definition(self, user_spec: str, list_name: str) -> None:
             item[list_name].append(user_spec)
             break
 
-        # "definitions" can be remote, so we need to update the global config too
-        spack.config.CONFIG.set("definitions", defs, scope=self.scope_name)
         self.changed = True
 
     def remove_definition(self, user_spec: str, list_name: str) -> None:
@@ -2809,8 +2854,6 @@ def remove_definition(self, user_spec: str, list_name: str) -> None:
             except ValueError:
                 pass
 
-        # "definitions" can be remote, so we need to update the global config too
-        spack.config.CONFIG.set("definitions", defs, scope=self.scope_name)
         self.changed = True
 
     def override_definition(self, user_spec: str, *, override: str, list_name: str) -> None:
@@ -2836,8 +2879,6 @@ def override_definition(self, user_spec: str, *, override: str, list_name: str)
             except ValueError:
                 pass
 
-        # "definitions" can be remote, so we need to update the global config too
-        spack.config.CONFIG.set("definitions", defs, scope=self.scope_name)
         self.changed = True
 
     def _iterate_on_definitions(self, definitions, *, list_name, err_msg):
@@ -2916,27 +2957,33 @@ def __str__(self):
         return str(self.manifest_file)
 
     @property
-    def env_config_scope(self) -> spack.config.ConfigScope:
-        """The configuration scope for the environment manifest"""
-        if self._env_config_scope is None:
-            self._env_config_scope = spack.config.SingleFileScope(
+    def env_config_scopes(self) -> List[spack.config.ConfigScope]:
+        """A list of all configuration scopes for the environment manifest. On the first call this
+        instantiates all the scopes, on subsequent calls it returns the cached list."""
+        if self._config_scopes is not None:
+            return self._config_scopes
+
+        scopes: List[spack.config.ConfigScope] = [
+            spack.config.SingleFileScope(
                 self.scope_name,
                 str(self.manifest_file),
                 spack.schema.env.schema,
                 yaml_path=[TOP_LEVEL_KEY],
             )
-            ensure_no_disallowed_env_config_mods(self._env_config_scope)
-        return self._env_config_scope
+        ]
+        ensure_no_disallowed_env_config_mods(scopes)
+        self._config_scopes = scopes
+        return scopes
 
     def prepare_config_scope(self) -> None:
-        """Add the manifest's scope to the global configuration search path."""
-        spack.config.CONFIG.push_scope(
-            self.env_config_scope, priority=ConfigScopePriority.ENVIRONMENT
-        )
+        """Add the manifest's scopes to the global configuration search path."""
+        for scope in self.env_config_scopes:
+            spack.config.CONFIG.push_scope(scope, priority=ConfigScopePriority.ENVIRONMENT)
 
     def deactivate_config_scope(self) -> None:
-        """Remove the manifest's scope from the global config path."""
-        spack.config.CONFIG.remove_scope(self.env_config_scope.name)
+        """Remove any of the manifest's scopes from the global config path."""
+        for scope in self.env_config_scopes:
+            spack.config.CONFIG.remove_scope(scope.name)
 
     @contextlib.contextmanager
     def use_config(self):
@@ -2947,8 +2994,8 @@ def use_config(self):
             self.deactivate_config_scope()
 
 
-def environment_path_scope(name: str, path: str) -> Optional[spack.config.ConfigScope]:
-    """Retrieve the suitably named environment path scope
+def environment_path_scopes(name: str, path: str) -> Optional[List[spack.config.ConfigScope]]:
+    """Retrieve the suitably named environment path scopes
 
     Arguments:
         name: configuration scope name
@@ -2963,9 +3010,11 @@ def environment_path_scope(name: str, path: str) -> Optional[spack.config.Config
     else:
         return None
 
-    manifest.env_config_scope.name = f"{name}:{manifest.env_config_scope.name}"
-    manifest.env_config_scope.writable = False
-    return manifest.env_config_scope
+    for scope in manifest.env_config_scopes:
+        scope.name = f"{name}:{scope.name}"
+        scope.writable = False
+
+    return manifest.env_config_scopes
 
 
 class SpackEnvironmentError(spack.error.SpackError):

lib/spack/spack/environment/list.py

@@ -1,286 +0,0 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import itertools
-from typing import Any, Dict, List, NamedTuple, Optional, Union
-
-import spack.spec
-import spack.util.spack_yaml
-import spack.variant
-from spack.error import SpackError
-from spack.spec import Spec
-
-
-class SpecList:
-    def __init__(self, *, name: str = "specs", yaml_list=None, expanded_list=None):
-        self.name = name
-        self.yaml_list = yaml_list[:] if yaml_list is not None else []
-        # Expansions can be expensive to compute and difficult to keep updated
-        # We cache results and invalidate when self.yaml_list changes
-        self.specs_as_yaml_list = expanded_list or []
-        self._constraints = None
-        self._specs: Optional[List[Spec]] = None
-
-    @property
-    def is_matrix(self):
-        for item in self.specs_as_yaml_list:
-            if isinstance(item, dict):
-                return True
-        return False
-
-    @property
-    def specs_as_constraints(self):
-        if self._constraints is None:
-            constraints = []
-            for item in self.specs_as_yaml_list:
-                if isinstance(item, dict):  # matrix of specs
-                    constraints.extend(_expand_matrix_constraints(item))
-                else:  # individual spec
-                    constraints.append([Spec(item)])
-            self._constraints = constraints
-
-        return self._constraints
-
-    @property
-    def specs(self) -> List[Spec]:
-        if self._specs is None:
-            specs: List[Spec] = []
-            # This could be slightly faster done directly from yaml_list,
-            # but this way is easier to maintain.
-            for constraint_list in self.specs_as_constraints:
-                spec = constraint_list[0].copy()
-                for const in constraint_list[1:]:
-                    spec.constrain(const)
-                specs.append(spec)
-            self._specs = specs
-
-        return self._specs
-
-    def add(self, spec: Spec):
-        spec_str = str(spec)
-        self.yaml_list.append(spec_str)
-
-        # expanded list can be updated without invalidation
-        if self.specs_as_yaml_list is not None:
-            self.specs_as_yaml_list.append(spec_str)
-
-        # Invalidate cache variables when we change the list
-        self._constraints = None
-        self._specs = None
-
-    def remove(self, spec):
-        # Get spec to remove from list
-        remove = [
-            s
-            for s in self.yaml_list
-            if (isinstance(s, str) and not s.startswith("$")) and Spec(s) == Spec(spec)
-        ]
-        if not remove:
-            msg = f"Cannot remove {spec} from SpecList {self.name}.\n"
-            msg += f"Either {spec} is not in {self.name} or {spec} is "
-            msg += "expanded from a matrix and cannot be removed directly."
-            raise SpecListError(msg)
-
-        # Remove may contain more than one string representation of the same spec
-        for item in remove:
-            self.yaml_list.remove(item)
-            self.specs_as_yaml_list.remove(item)
-
-        # invalidate cache variables when we change the list
-        self._constraints = None
-        self._specs = None
-
-    def extend(self, other: "SpecList", copy_reference=True) -> None:
-        self.yaml_list.extend(other.yaml_list)
-        self.specs_as_yaml_list.extend(other.specs_as_yaml_list)
-        self._constraints = None
-        self._specs = None
-
-    def __len__(self):
-        return len(self.specs)
-
-    def __getitem__(self, key):
-        return self.specs[key]
-
-    def __iter__(self):
-        return iter(self.specs)
-
-
-def _expand_matrix_constraints(matrix_config):
-    # recurse so we can handle nested matrices
-    expanded_rows = []
-    for row in matrix_config["matrix"]:
-        new_row = []
-        for r in row:
-            if isinstance(r, dict):
-                # Flatten the nested matrix into a single row of constraints
-                new_row.extend(
-                    [
-                        [" ".join([str(c) for c in expanded_constraint_list])]
-                        for expanded_constraint_list in _expand_matrix_constraints(r)
-                    ]
-                )
-            else:
-                new_row.append([r])
-        expanded_rows.append(new_row)
-
-    excludes = matrix_config.get("exclude", [])  # only compute once
-    sigil = matrix_config.get("sigil", "")
-
-    results = []
-    for combo in itertools.product(*expanded_rows):
-        # Construct a combined spec to test against excludes
-        flat_combo = [Spec(constraint) for constraints in combo for constraint in constraints]
-
-        test_spec = flat_combo[0].copy()
-        for constraint in flat_combo[1:]:
-            test_spec.constrain(constraint)
-
-        # Abstract variants don't have normal satisfaction semantics
-        # Convert all variants to concrete types.
-        # This method is best effort, so all existing variants will be
-        # converted before any error is raised.
-        # Catch exceptions because we want to be able to operate on
-        # abstract specs without needing package information
-        try:
-            spack.spec.substitute_abstract_variants(test_spec)
-        except spack.variant.UnknownVariantError:
-            pass
-
-        # Resolve abstract hashes for exclusion criteria
-        if any(test_spec.lookup_hash().satisfies(x) for x in excludes):
-            continue
-
-        if sigil:
-            flat_combo[0] = Spec(sigil + str(flat_combo[0]))
-
-        # Add to list of constraints
-        results.append(flat_combo)
-
-    return results
-
-
-def _sigilify(item, sigil):
-    if isinstance(item, dict):
-        if sigil:
-            item["sigil"] = sigil
-        return item
-    else:
-        return sigil + item
-
-
-class Definition(NamedTuple):
-    name: str
-    yaml_list: List[Union[str, Dict]]
-    when: Optional[str]
-
-
-class SpecListParser:
-    """Parse definitions and user specs from data in environments"""
-
-    def __init__(self):
-        self.definitions: Dict[str, SpecList] = {}
-
-    def parse_definitions(self, *, data: List[Dict[str, Any]]) -> Dict[str, SpecList]:
-        definitions_from_yaml: Dict[str, List[Definition]] = {}
-        for item in data:
-            value = self._parse_yaml_definition(item)
-            definitions_from_yaml.setdefault(value.name, []).append(value)
-
-        self.definitions = {}
-        self._build_definitions(definitions_from_yaml)
-        return self.definitions
-
-    def parse_user_specs(self, *, name, yaml_list) -> SpecList:
-        definition = Definition(name=name, yaml_list=yaml_list, when=None)
-        return self._speclist_from_definitions(name, [definition])
-
-    def _parse_yaml_definition(self, yaml_entry) -> Definition:
-        when_string = yaml_entry.get("when")
-
-        if (when_string and len(yaml_entry) > 2) or (not when_string and len(yaml_entry) > 1):
-            mark = spack.util.spack_yaml.get_mark_from_yaml_data(yaml_entry)
-            attributes = ", ".join(x for x in yaml_entry if x != "when")
-            error_msg = f"definition must have a single attribute, got many: {attributes}"
-            raise SpecListError(f"{mark.name}:{mark.line + 1}: {error_msg}")
-
-        for name, yaml_list in yaml_entry.items():
-            if name == "when":
-                continue
-            return Definition(name=name, yaml_list=yaml_list, when=when_string)
-
-        # If we are here, it means only "when" is in the entry
-        mark = spack.util.spack_yaml.get_mark_from_yaml_data(yaml_entry)
-        error_msg = "definition must have a single attribute, got none"
-        raise SpecListError(f"{mark.name}:{mark.line + 1}: {error_msg}")
-
-    def _build_definitions(self, definitions_from_yaml: Dict[str, List[Definition]]):
-        for name, definitions in definitions_from_yaml.items():
-            self.definitions[name] = self._speclist_from_definitions(name, definitions)
-
-    def _speclist_from_definitions(self, name, definitions) -> SpecList:
-        combined_yaml_list = []
-        for def_part in definitions:
-            if def_part.when is not None and not spack.spec.eval_conditional(def_part.when):
-                continue
-            combined_yaml_list.extend(def_part.yaml_list)
-        expanded_list = self._expand_yaml_list(combined_yaml_list)
-        return SpecList(name=name, yaml_list=combined_yaml_list, expanded_list=expanded_list)
-
-    def _expand_yaml_list(self, raw_yaml_list):
-        result = []
-        for item in raw_yaml_list:
-            if isinstance(item, str) and item.startswith("$"):
-                result.extend(self._expand_reference(item))
-                continue
-
-            value = item
-            if isinstance(item, dict):
-                value = self._expand_yaml_matrix(item)
-            result.append(value)
-        return result
-
-    def _expand_reference(self, item: str):
-        sigil, name = "", item[1:]
-        if name.startswith("^") or name.startswith("%"):
-            sigil, name = name[0], name[1:]
-
-        if name not in self.definitions:
-            mark = spack.util.spack_yaml.get_mark_from_yaml_data(item)
-            error_msg = f"trying to expand the name '{name}', which is not defined yet"
-            raise UndefinedReferenceError(f"{mark.name}:{mark.line + 1}: {error_msg}")
-
-        value = self.definitions[name].specs_as_yaml_list
-        if not sigil:
-            return value
-        return [_sigilify(x, sigil) for x in value]
-
-    def _expand_yaml_matrix(self, matrix_yaml):
-        extra_attributes = set(matrix_yaml) - {"matrix", "exclude"}
-        if extra_attributes:
-            mark = spack.util.spack_yaml.get_mark_from_yaml_data(matrix_yaml)
-            error_msg = f"extra attributes in spec matrix: {','.join(sorted(extra_attributes))}"
-            raise SpecListError(f"{mark.name}:{mark.line + 1}: {error_msg}")
-
-        if "matrix" not in matrix_yaml:
-            mark = spack.util.spack_yaml.get_mark_from_yaml_data(matrix_yaml)
-            error_msg = "matrix is missing the 'matrix' attribute"
-            raise SpecListError(f"{mark.name}:{mark.line + 1}: {error_msg}")
-
-        # Assume data has been validated against the YAML schema
-        result = {"matrix": [self._expand_yaml_list(row) for row in matrix_yaml["matrix"]]}
-        if "exclude" in matrix_yaml:
-            result["exclude"] = matrix_yaml["exclude"]
-        return result
-
-
-class SpecListError(SpackError):
-    """Error class for all errors related to SpecList objects."""
-
-
-class UndefinedReferenceError(SpecListError):
-    """Error class for undefined references in Spack stacks."""
-
-
-class InvalidSpecConstraintError(SpecListError):
-    """Error class for invalid spec constraints at concretize time."""

lib/spack/spack/environment/shell.py

@@ -49,23 +49,10 @@ def activate_header(env, shell, prompt=None, view: Optional[str] = None):
         cmds += 'set "SPACK_ENV=%s"\n' % env.path
         if view:
             cmds += 'set "SPACK_ENV_VIEW=%s"\n' % view
-        if prompt:
-            old_prompt = os.environ.get("SPACK_OLD_PROMPT")
-            if not old_prompt:
-                old_prompt = os.environ.get("PROMPT")
-            cmds += f'set "SPACK_OLD_PROMPT={old_prompt}"\n'
-            cmds += f'set "PROMPT={prompt} $P$G"\n'
     elif shell == "pwsh":
         cmds += "$Env:SPACK_ENV='%s'\n" % env.path
         if view:
             cmds += "$Env:SPACK_ENV_VIEW='%s'\n" % view
-        if prompt:
-            cmds += (
-                "function global:prompt { $pth = $(Convert-Path $(Get-Location))"
-                ' | Split-Path -leaf; if(!"$Env:SPACK_OLD_PROMPT") '
-                '{$Env:SPACK_OLD_PROMPT="[spack] PS $pth>"}; '
-                '"%s PS $pth>"}\n' % prompt
-            )
     else:
         bash_color_prompt = colorize(f"@G{{{prompt}}}", color=True, enclose=True)
         zsh_color_prompt = colorize(f"@G{{{prompt}}}", color=True, enclose=False, zsh=True)
@@ -120,19 +107,10 @@ def deactivate_header(shell):
         cmds += 'set "SPACK_ENV="\n'
         cmds += 'set "SPACK_ENV_VIEW="\n'
         # TODO: despacktivate
-        old_prompt = os.environ.get("SPACK_OLD_PROMPT")
-        if old_prompt:
-            cmds += f'set "PROMPT={old_prompt}"\n'
-            cmds += 'set "SPACK_OLD_PROMPT="\n'
+        # TODO: prompt
     elif shell == "pwsh":
         cmds += "Set-Item -Path Env:SPACK_ENV\n"
         cmds += "Set-Item -Path Env:SPACK_ENV_VIEW\n"
-        cmds += (
-            "function global:prompt { $pth = $(Convert-Path $(Get-Location))"
-            ' | Split-Path -leaf; $spack_prompt = "[spack] $pth >"; '
-            'if("$Env:SPACK_OLD_PROMPT") {$spack_prompt=$Env:SPACK_OLD_PROMPT};'
-            " $spack_prompt}\n"
-        )
     else:
         cmds += "if [ ! -z ${SPACK_ENV+x} ]; then\n"
         cmds += "unset SPACK_ENV; export SPACK_ENV;\n"

lib/spack/spack/error.py

@@ -202,16 +202,3 @@ class MirrorError(SpackError):
 
     def __init__(self, msg, long_msg=None):
         super().__init__(msg, long_msg)
-
-
-class NoChecksumException(SpackError):
-    """
-    Raised if file fails checksum verification.
-    """
-
-    def __init__(self, path, size, contents, algorithm, expected, computed):
-        super().__init__(
-            f"{algorithm} checksum failed for {path}",
-            f"Expected {expected} but got {computed}. "
-            f"File size = {size} bytes. Contents = {contents!r}",
-        )

lib/spack/spack/fetch_strategy.py

@@ -27,14 +27,11 @@
 import os
 import re
 import shutil
-import sys
-import time
 import urllib.error
 import urllib.parse
 import urllib.request
-import urllib.response
 from pathlib import PurePath
-from typing import Callable, List, Mapping, Optional
+from typing import List, Optional
 
 import llnl.url
 import llnl.util
@@ -222,114 +219,6 @@ def mirror_id(self):
         """BundlePackages don't have a mirror id."""
 
 
-def _format_speed(total_bytes: int, elapsed: float) -> str:
-    """Return a human-readable average download speed string."""
-    elapsed = 1 if elapsed <= 0 else elapsed  # avoid divide by zero
-    speed = total_bytes / elapsed
-    if speed >= 1e9:
-        return f"{speed / 1e9:6.1f} GB/s"
-    elif speed >= 1e6:
-        return f"{speed / 1e6:6.1f} MB/s"
-    elif speed >= 1e3:
-        return f"{speed / 1e3:6.1f} KB/s"
-    return f"{speed:6.1f}  B/s"
-
-
-def _format_bytes(total_bytes: int) -> str:
-    """Return a human-readable total bytes string."""
-    if total_bytes >= 1e9:
-        return f"{total_bytes / 1e9:7.2f} GB"
-    elif total_bytes >= 1e6:
-        return f"{total_bytes / 1e6:7.2f} MB"
-    elif total_bytes >= 1e3:
-        return f"{total_bytes / 1e3:7.2f} KB"
-    return f"{total_bytes:7.2f}  B"
-
-
-class FetchProgress:
-    #: Characters to rotate in the spinner.
-    spinner = ["|", "/", "-", "\\"]
-
-    def __init__(
-        self,
-        total_bytes: Optional[int] = None,
-        enabled: bool = True,
-        get_time: Callable[[], float] = time.time,
-    ) -> None:
-        """Initialize a FetchProgress instance.
-        Args:
-            total_bytes: Total number of bytes to download, if known.
-            enabled: Whether to print progress information.
-            get_time: Function to get the current time."""
-        #: Number of bytes downloaded so far.
-        self.current_bytes = 0
-        #: Delta time between progress prints
-        self.delta = 0.1
-        #: Whether to print progress information.
-        self.enabled = enabled
-        #: Function to get the current time.
-        self.get_time = get_time
-        #: Time of last progress print to limit output
-        self.last_printed = 0.0
-        #: Time of start of download
-        self.start_time = get_time() if enabled else 0.0
-        #: Total number of bytes to download, if known.
-        self.total_bytes = total_bytes if total_bytes and total_bytes > 0 else 0
-        #: Index of spinner character to print (used if total bytes is unknown)
-        self.index = 0
-
-    @classmethod
-    def from_headers(
-        cls,
-        headers: Mapping[str, str],
-        enabled: bool = True,
-        get_time: Callable[[], float] = time.time,
-    ) -> "FetchProgress":
-        """Create a FetchProgress instance from HTTP headers."""
-        # headers.get is case-insensitive if it's from a HTTPResponse object.
-        content_length = headers.get("Content-Length")
-        try:
-            total_bytes = int(content_length) if content_length else None
-        except ValueError:
-            total_bytes = None
-        return cls(total_bytes=total_bytes, enabled=enabled, get_time=get_time)
-
-    def advance(self, num_bytes: int, out=sys.stdout) -> None:
-        if not self.enabled:
-            return
-        self.current_bytes += num_bytes
-        self.print(out=out)
-
-    def print(self, final: bool = False, out=sys.stdout) -> None:
-        if not self.enabled:
-            return
-        current_time = self.get_time()
-        if self.last_printed + self.delta < current_time or final:
-            self.last_printed = current_time
-            # print a newline if this is the final update
-            maybe_newline = "\n" if final else ""
-            # if we know the total bytes, show a percentage, otherwise a spinner
-            if self.total_bytes > 0:
-                percentage = min(100 * self.current_bytes / self.total_bytes, 100.0)
-                percent_or_spinner = f"[{percentage:3.0f}%] "
-            else:
-                # only show the spinner if we are not at 100%
-                if final:
-                    percent_or_spinner = "[100%] "
-                else:
-                    percent_or_spinner = f"[ {self.spinner[self.index]}  ] "
-                self.index = (self.index + 1) % len(self.spinner)
-
-            print(
-                f"\r    {percent_or_spinner}{_format_bytes(self.current_bytes)} "
-                f"@ {_format_speed(self.current_bytes, current_time - self.start_time)}"
-                f"{maybe_newline}",
-                end="",
-                flush=True,
-                file=out,
-            )
-
-
 @fetcher
 class URLFetchStrategy(FetchStrategy):
     """URLFetchStrategy pulls source code from a URL for an archive, check the
@@ -427,7 +316,7 @@ def _check_headers(self, headers):
             tty.warn(msg)
 
     @_needs_stage
-    def _fetch_urllib(self, url, chunk_size=65536):
+    def _fetch_urllib(self, url):
         save_file = self.stage.save_filename
 
         request = urllib.request.Request(url, headers={"User-Agent": web_util.SPACK_USER_AGENT})
@@ -438,15 +327,8 @@ def _fetch_urllib(self, url, chunk_size=65536):
         try:
             response = web_util.urlopen(request)
             tty.msg(f"Fetching {url}")
-            progress = FetchProgress.from_headers(response.headers, enabled=sys.stdout.isatty())
             with open(save_file, "wb") as f:
-                while True:
-                    chunk = response.read(chunk_size)
-                    if not chunk:
-                        break
-                    f.write(chunk)
-                    progress.advance(len(chunk))
-            progress.print(final=True)
+                shutil.copyfileobj(response, f)
         except OSError as e:
             # clean up archive on failure.
             if self.archive_file:

lib/spack/spack/install_test.py

@@ -12,7 +12,7 @@
 import shutil
 import sys
 from collections import Counter, OrderedDict
-from typing import Callable, Iterable, List, Optional, Tuple, Type, TypeVar, Union
+from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
@@ -391,7 +391,7 @@ def phase_tests(self, builder, phase_name: str, method_names: List[str]):
             if self.test_failures:
                 raise TestFailure(self.test_failures)
 
-    def stand_alone_tests(self, kwargs, timeout: Optional[int] = None) -> None:
+    def stand_alone_tests(self, kwargs):
         """Run the package's stand-alone tests.
 
         Args:
@@ -399,9 +399,7 @@ def stand_alone_tests(self, kwargs, timeout: Optional[int] = None) -> None:
         """
         import spack.build_environment  # avoid circular dependency
 
-        spack.build_environment.start_build_process(
-            self.pkg, test_process, kwargs, timeout=timeout
-        )
+        spack.build_environment.start_build_process(self.pkg, test_process, kwargs)
 
     def parts(self) -> int:
         """The total number of (checked) test parts."""
@@ -849,7 +847,7 @@ def write_test_summary(counts: "Counter"):
 class TestSuite:
     """The class that manages specs for ``spack test run`` execution."""
 
-    def __init__(self, specs: Iterable[Spec], alias: Optional[str] = None) -> None:
+    def __init__(self, specs, alias=None):
         # copy so that different test suites have different package objects
         # even if they contain the same spec
         self.specs = [spec.copy() for spec in specs]
@@ -857,43 +855,42 @@ def __init__(self, specs: Iterable[Spec], alias: Optional[str] = None) -> None:
         self.current_base_spec = None  # spec currently running do_test
 
         self.alias = alias
-        self._hash: Optional[str] = None
-        self._stage: Optional[Prefix] = None
+        self._hash = None
+        self._stage = None
 
         self.counts: "Counter" = Counter()
 
     @property
-    def name(self) -> str:
+    def name(self):
         """The name (alias or, if none, hash) of the test suite."""
         return self.alias if self.alias else self.content_hash
 
     @property
-    def content_hash(self) -> str:
+    def content_hash(self):
         """The hash used to uniquely identify the test suite."""
         if not self._hash:
             json_text = sjson.dump(self.to_dict())
-            assert json_text is not None, f"{__name__} unexpected value for 'json_text'"
             sha = hashlib.sha1(json_text.encode("utf-8"))
             b32_hash = base64.b32encode(sha.digest()).lower()
             b32_hash = b32_hash.decode("utf-8")
             self._hash = b32_hash
         return self._hash
 
-    def __call__(
-        self,
-        *,
-        remove_directory: bool = True,
-        dirty: bool = False,
-        fail_first: bool = False,
-        externals: bool = False,
-        timeout: Optional[int] = None,
-    ):
+    def __call__(self, *args, **kwargs):
         self.write_reproducibility_data()
+
+        remove_directory = kwargs.get("remove_directory", True)
+        dirty = kwargs.get("dirty", False)
+        fail_first = kwargs.get("fail_first", False)
+        externals = kwargs.get("externals", False)
+
         for spec in self.specs:
             try:
                 if spec.package.test_suite:
                     raise TestSuiteSpecError(
-                        f"Package {spec.package.name} cannot be run in two test suites at once"
+                        "Package {} cannot be run in two test suites at once".format(
+                            spec.package.name
+                        )
                     )
 
                 # Set up the test suite to know which test is running
@@ -908,7 +905,7 @@ def __call__(
                 fs.mkdirp(test_dir)
 
                 # run the package tests
-                spec.package.do_test(dirty=dirty, externals=externals, timeout=timeout)
+                spec.package.do_test(dirty=dirty, externals=externals)
 
                 # Clean up on success
                 if remove_directory:
@@ -959,12 +956,15 @@ def __call__(
         if failures:
             raise TestSuiteFailure(failures)
 
-    def test_status(self, spec: spack.spec.Spec, externals: bool) -> TestStatus:
-        """Returns the overall test results status for the spec.
+    def test_status(self, spec: spack.spec.Spec, externals: bool) -> Optional[TestStatus]:
+        """Determine the overall test results status for the spec.
 
         Args:
             spec: instance of the spec under test
             externals: ``True`` if externals are to be tested, else ``False``
+
+        Returns:
+            the spec's test status if available or ``None``
         """
         tests_status_file = self.tested_file_for_spec(spec)
         if not os.path.exists(tests_status_file):
@@ -981,84 +981,109 @@ def test_status(self, spec: spack.spec.Spec, externals: bool) -> TestStatus:
             value = (f.read()).strip("\n")
             return TestStatus(int(value)) if value else TestStatus.NO_TESTS
 
-    def ensure_stage(self) -> None:
+    def ensure_stage(self):
         """Ensure the test suite stage directory exists."""
         if not os.path.exists(self.stage):
             fs.mkdirp(self.stage)
 
     @property
-    def stage(self) -> Prefix:
-        """The root test suite stage directory"""
+    def stage(self):
+        """The root test suite stage directory.
+
+        Returns:
+            str: the spec's test stage directory path
+        """
         if not self._stage:
             self._stage = Prefix(fs.join_path(get_test_stage_dir(), self.content_hash))
         return self._stage
 
     @stage.setter
-    def stage(self, value: Union[Prefix, str]) -> None:
+    def stage(self, value):
         """Set the value of a non-default stage directory."""
         self._stage = value if isinstance(value, Prefix) else Prefix(value)
 
     @property
-    def results_file(self) -> Prefix:
+    def results_file(self):
         """The path to the results summary file."""
         return self.stage.join(results_filename)
 
     @classmethod
-    def test_pkg_id(cls, spec: Spec) -> str:
+    def test_pkg_id(cls, spec):
         """The standard install test package identifier.
 
         Args:
             spec: instance of the spec under test
+
+        Returns:
+            str: the install test package identifier
         """
         return spec.format_path("{name}-{version}-{hash:7}")
 
     @classmethod
-    def test_log_name(cls, spec: Spec) -> str:
+    def test_log_name(cls, spec):
         """The standard log filename for a spec.
 
         Args:
-            spec: instance of the spec under test
-        """
-        return f"{cls.test_pkg_id(spec)}-test-out.txt"
+            spec (spack.spec.Spec): instance of the spec under test
 
-    def log_file_for_spec(self, spec: Spec) -> Prefix:
+        Returns:
+            str: the spec's log filename
+        """
+        return "%s-test-out.txt" % cls.test_pkg_id(spec)
+
+    def log_file_for_spec(self, spec):
         """The test log file path for the provided spec.
 
         Args:
-            spec: instance of the spec under test
+            spec (spack.spec.Spec): instance of the spec under test
+
+        Returns:
+            str: the path to the spec's log file
         """
         return self.stage.join(self.test_log_name(spec))
 
-    def test_dir_for_spec(self, spec: Spec) -> Prefix:
+    def test_dir_for_spec(self, spec):
         """The path to the test stage directory for the provided spec.
 
         Args:
-            spec: instance of the spec under test
+            spec (spack.spec.Spec): instance of the spec under test
+
+        Returns:
+            str: the spec's test stage directory path
         """
         return Prefix(self.stage.join(self.test_pkg_id(spec)))
 
     @classmethod
-    def tested_file_name(cls, spec: Spec) -> str:
+    def tested_file_name(cls, spec):
         """The standard test status filename for the spec.
 
         Args:
-            spec: instance of the spec under test
+            spec (spack.spec.Spec): instance of the spec under test
+
+        Returns:
+            str: the spec's test status filename
         """
         return "%s-tested.txt" % cls.test_pkg_id(spec)
 
-    def tested_file_for_spec(self, spec: Spec) -> str:
+    def tested_file_for_spec(self, spec):
         """The test status file path for the spec.
 
         Args:
-            spec: instance of the spec under test
+            spec (spack.spec.Spec): instance of the spec under test
+
+        Returns:
+            str: the spec's test status file path
         """
         return fs.join_path(self.stage, self.tested_file_name(spec))
 
     @property
-    def current_test_cache_dir(self) -> str:
+    def current_test_cache_dir(self):
         """Path to the test stage directory where the current spec's cached
         build-time files were automatically copied.
 
+        Returns:
+            str: path to the current spec's staged, cached build-time files.
+
         Raises:
             TestSuiteSpecError: If there is no spec being tested
         """
@@ -1070,10 +1095,13 @@ def current_test_cache_dir(self) -> str:
         return self.test_dir_for_spec(base_spec).cache.join(test_spec.name)
 
     @property
-    def current_test_data_dir(self) -> str:
+    def current_test_data_dir(self):
         """Path to the test stage directory where the current spec's custom
         package (data) files were automatically copied.
 
+        Returns:
+            str: path to the current spec's staged, custom package (data) files
+
         Raises:
             TestSuiteSpecError: If there is no spec being tested
         """
@@ -1084,17 +1112,17 @@ def current_test_data_dir(self) -> str:
         base_spec = self.current_base_spec
         return self.test_dir_for_spec(base_spec).data.join(test_spec.name)
 
-    def write_test_result(self, spec: Spec, result: TestStatus) -> None:
+    def write_test_result(self, spec, result):
         """Write the spec's test result to the test suite results file.
 
         Args:
-            spec: instance of the spec under test
-            result: result from the spec's test execution (e.g, PASSED)
+            spec (spack.spec.Spec): instance of the spec under test
+            result (str): result from the spec's test execution (e.g, PASSED)
         """
         msg = f"{self.test_pkg_id(spec)} {result}"
         _add_msg_to_file(self.results_file, msg)
 
-    def write_reproducibility_data(self) -> None:
+    def write_reproducibility_data(self):
         for spec in self.specs:
             repo_cache_path = self.stage.repo.join(spec.name)
             spack.repo.PATH.dump_provenance(spec, repo_cache_path)
@@ -1139,12 +1167,12 @@ def from_dict(d):
         return TestSuite(specs, alias)
 
     @staticmethod
-    def from_file(filename: str) -> "TestSuite":
+    def from_file(filename):
         """Instantiate a TestSuite using the specs and optional alias
         provided in the given file.
 
         Args:
-            filename: The path to the JSON file containing the test
+            filename (str): The path to the JSON file containing the test
                 suite specs and optional alias.
 
         Raises:

lib/spack/spack/installer.py

@@ -65,7 +65,6 @@
 import spack.util.executable
 import spack.util.path
 import spack.util.timer as timer
-from spack.url_buildcache import BuildcacheEntryError
 from spack.util.environment import EnvironmentModifications, dump_environment
 from spack.util.executable import which
 
@@ -450,17 +449,17 @@ def _process_binary_cache_tarball(
             else ``False``
     """
     with timer.measure("fetch"):
-        tarball_stage = binary_distribution.download_tarball(
+        download_result = binary_distribution.download_tarball(
             pkg.spec.build_spec, unsigned, mirrors_for_spec
         )
 
-        if tarball_stage is None:
+        if download_result is None:
             return False
 
     tty.msg(f"Extracting {package_id(pkg.spec)} from binary cache")
 
     with timer.measure("install"), spack.util.path.filter_padding():
-        binary_distribution.extract_tarball(pkg.spec, tarball_stage, force=False, timer=timer)
+        binary_distribution.extract_tarball(pkg.spec, download_result, force=False, timer=timer)
 
         if pkg.spec.spliced:  # overwrite old metadata with new
             spack.store.STORE.layout.write_spec(
@@ -567,11 +566,10 @@ def dump_packages(spec: "spack.spec.Spec", path: str) -> None:
                 tty.warn(f"Warning: Couldn't copy in provenance for {node.name}")
 
         # Create a destination repository
-        pkg_api = spack.repo.PATH.get_repo(node.namespace).package_api
-        repo_root = os.path.join(path, node.namespace) if pkg_api < (2, 0) else path
-        repo = spack.repo.create_or_construct(
-            repo_root, namespace=node.namespace, package_api=pkg_api
-        )
+        dest_repo_root = os.path.join(path, node.namespace)
+        if not os.path.exists(dest_repo_root):
+            spack.repo.create_repo(dest_repo_root)
+        repo = spack.repo.from_path(dest_repo_root)
 
         # Get the location of the package in the dest repo.
         dest_pkg_dir = repo.dirname_for_package_name(node.name)
@@ -2178,7 +2176,7 @@ def install(self) -> None:
                 )
                 raise
 
-            except BuildcacheEntryError as exc:
+            except binary_distribution.NoChecksumException as exc:
                 if task.cache_only:
                     raise
 

lib/spack/spack/main.py

@@ -20,7 +20,6 @@
 import signal
 import subprocess as sp
 import sys
-import tempfile
 import traceback
 import warnings
 from typing import List, Tuple
@@ -42,7 +41,6 @@
 import spack.paths
 import spack.platforms
 import spack.repo
-import spack.solver.asp
 import spack.spec
 import spack.store
 import spack.util.debug
@@ -550,6 +548,7 @@ def setup_main_options(args):
         spack.config.CONFIG.scopes["command_line"].sections["repos"] = syaml.syaml_dict(
             [(key, [spack.paths.mock_packages_path])]
         )
+        spack.repo.PATH = spack.repo.create(spack.config.CONFIG)
 
     # If the user asked for it, don't check ssl certs.
     if args.insecure:
@@ -560,8 +559,6 @@ def setup_main_options(args):
     for config_var in args.config_vars or []:
         spack.config.add(fullpath=config_var, scope="command_line")
 
-    spack.repo.enable_repo(spack.repo.create(spack.config.CONFIG))
-
     # On Windows10 console handling for ASCI/VT100 sequences is not
     # on by default. Turn on before we try to write to console
     # with color
@@ -874,8 +871,8 @@ def add_command_line_scopes(
     """
     for i, path in enumerate(command_line_scopes):
         name = f"cmd_scope_{i}"
-        scope = ev.environment_path_scope(name, path)
-        if scope is None:
+        scopes = ev.environment_path_scopes(name, path)
+        if scopes is None:
             if os.path.isdir(path):  # directory with config files
                 cfg.push_scope(
                     spack.config.DirectoryConfigScope(name, path, writable=False),
@@ -888,7 +885,8 @@ def add_command_line_scopes(
             else:
                 raise spack.error.ConfigError(f"Invalid configuration scope: {path}")
 
-        cfg.push_scope(scope, priority=ConfigScopePriority.CUSTOM)
+        for scope in scopes:
+            cfg.push_scope(scope, priority=ConfigScopePriority.CUSTOM)
 
 
 def _main(argv=None):
@@ -1049,10 +1047,6 @@ def main(argv=None):
     try:
         return _main(argv)
 
-    except spack.solver.asp.OutputDoesNotSatisfyInputError as e:
-        _handle_solver_bug(e)
-        return 1
-
     except spack.error.SpackError as e:
         tty.debug(e)
         e.die()  # gracefully die on any SpackErrors
@@ -1076,45 +1070,5 @@ def main(argv=None):
         return 3
 
 
-def _handle_solver_bug(
-    e: spack.solver.asp.OutputDoesNotSatisfyInputError, out=sys.stderr, root=None
-) -> None:
-    # when the solver outputs specs that do not satisfy the input and spack is used as a command
-    # line tool, we dump the incorrect output specs to json so users can upload them in bug reports
-    wrong_output = [(input, output) for input, output in e.input_to_output if output is not None]
-    no_output = [input for input, output in e.input_to_output if output is None]
-    if no_output:
-        tty.error(
-            "internal solver error: the following specs were not solved:\n    - "
-            + "\n    - ".join(str(s) for s in no_output),
-            stream=out,
-        )
-    if wrong_output:
-        msg = (
-            "internal solver error: the following specs were concretized, but do not satisfy the "
-            "input:\n    - "
-            + "\n    - ".join(str(s) for s, _ in wrong_output)
-            + "\n    Please report a bug at https://github.com/spack/spack/issues"
-        )
-        # try to write the input/output specs to a temporary directory for bug reports
-        try:
-            tmpdir = tempfile.mkdtemp(prefix="spack-asp-", dir=root)
-            files = []
-            for i, (input, output) in enumerate(wrong_output, start=1):
-                in_file = os.path.join(tmpdir, f"input-{i}.json")
-                out_file = os.path.join(tmpdir, f"output-{i}.json")
-                files.append(in_file)
-                files.append(out_file)
-                with open(in_file, "w", encoding="utf-8") as f:
-                    input.to_json(f)
-                with open(out_file, "w", encoding="utf-8") as f:
-                    output.to_json(f)
-
-            msg += " and attach the following files:\n    - " + "\n    - ".join(files)
-        except Exception:
-            msg += "."
-        tty.error(msg, stream=out)
-
-
 class SpackCommandError(Exception):
     """Raised when SpackCommand execution fails."""

lib/spack/spack/package.py

@@ -162,7 +162,6 @@ class tty:
 configure: Executable
 make_jobs: int
 make: MakeExecutable
-nmake: Executable
 ninja: MakeExecutable
 python_include: str
 python_platlib: str
@@ -172,5 +171,3 @@ class tty:
 spack_cxx: str
 spack_f77: str
 spack_fc: str
-prefix: Prefix
-dso_suffix: str

lib/spack/spack/package_base.py

@@ -14,6 +14,7 @@
 import functools
 import glob
 import hashlib
+import importlib
 import io
 import os
 import re
@@ -27,7 +28,7 @@
 
 import llnl.util.filesystem as fsys
 import llnl.util.tty as tty
-from llnl.util.lang import ClassProperty, classproperty, memoized
+from llnl.util.lang import classproperty, memoized
 
 import spack.config
 import spack.dependency
@@ -47,7 +48,6 @@
 import spack.url
 import spack.util.environment
 import spack.util.executable
-import spack.util.naming
 import spack.util.path
 import spack.util.web
 import spack.variant
@@ -701,10 +701,10 @@ class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta):
     _verbose = None
 
     #: Package homepage where users can find more information about the package
-    homepage: ClassProperty[Optional[str]] = None
+    homepage: Optional[str] = None
 
     #: Default list URL (place to find available versions)
-    list_url: ClassProperty[Optional[str]] = None
+    list_url: Optional[str] = None
 
     #: Link depth to which list_url should be searched for new versions
     list_depth = 0
@@ -818,12 +818,12 @@ def package_dir(cls):
 
     @classproperty
     def module(cls):
-        """Module instance that this package class is defined in.
+        """Module object (not just the name) that this package is defined in.
 
         We use this to add variables to package modules.  This makes
         install() methods easier to write (e.g., can call configure())
         """
-        return sys.modules[cls.__module__]
+        return importlib.import_module(cls.__module__)
 
     @classproperty
     def namespace(cls):
@@ -839,36 +839,26 @@ def fullname(cls):
     def fullnames(cls):
         """Fullnames for this package and any packages from which it inherits."""
         fullnames = []
-        for base in cls.__mro__:
-            if not spack.repo.is_package_module(base.__module__):
+        for cls in cls.__mro__:
+            namespace = getattr(cls, "namespace", None)
+            if namespace:
+                fullnames.append("%s.%s" % (namespace, cls.name))
+            if namespace == "builtin":
+                # builtin packages cannot inherit from other repos
                 break
-            fullnames.append(base.fullname)
         return fullnames
 
     @classproperty
     def name(cls):
-        """The name of this package."""
+        """The name of this package.
+
+        The name of a package is the name of its Python module, without
+        the containing module names.
+        """
         if cls._name is None:
-            # We cannot know the exact package API version, but we can distinguish between v1
-            # v2 based on the module. We don't want to figure out the exact package API version
-            # since it requires parsing the repo.yaml.
-            module = cls.__module__
-
-            if module.startswith(spack.repo.PKG_MODULE_PREFIX_V1):
-                version = (1, 0)
-            elif module.startswith(spack.repo.PKG_MODULE_PREFIX_V2):
-                version = (2, 0)
-            else:
-                raise ValueError(f"Package {cls.__qualname__} is not a known Spack package")
-
-            if version < (2, 0):
-                # spack.pkg.builtin.package_name.
-                _, _, pkg_module = module.rpartition(".")
-            else:
-                # spack_repo.builtin.packages.package_name.package
-                pkg_module = module.rsplit(".", 2)[-2]
-
-            cls._name = spack.util.naming.pkg_dir_to_pkg_name(pkg_module, version)
+            cls._name = cls.module.__name__
+            if "." in cls._name:
+                cls._name = cls._name[cls._name.rindex(".") + 1 :]
         return cls._name
 
     @classproperty
@@ -1831,7 +1821,7 @@ def _resource_stage(self, resource):
         resource_stage_folder = "-".join(pieces)
         return resource_stage_folder
 
-    def do_test(self, *, dirty=False, externals=False, timeout: Optional[int] = None):
+    def do_test(self, dirty=False, externals=False):
         if self.test_requires_compiler and not any(
             lang in self.spec for lang in ("c", "cxx", "fortran")
         ):
@@ -1849,7 +1839,7 @@ def do_test(self, *, dirty=False, externals=False, timeout: Optional[int] = None
             "verbose": tty.is_verbose(),
         }
 
-        self.tester.stand_alone_tests(kwargs, timeout=timeout)
+        self.tester.stand_alone_tests(kwargs)
 
     def unit_test_check(self):
         """Hook for unit tests to assert things about package internals.

lib/spack/spack/paths.py

@@ -56,9 +56,8 @@
 
 # read-only things in $spack/var/spack
 repos_path = os.path.join(var_path, "repos")
-test_repos_path = os.path.join(var_path, "test_repos")
-packages_path = os.path.join(repos_path, "spack_repo", "builtin")
-mock_packages_path = os.path.join(test_repos_path, "builtin.mock")
+packages_path = os.path.join(repos_path, "builtin")
+mock_packages_path = os.path.join(repos_path, "builtin.mock")
 
 #
 # Writable things in $spack/var/spack

lib/spack/spack/repo.py

@@ -47,34 +47,40 @@
 import spack.util.path
 import spack.util.spack_yaml as syaml
 
-PKG_MODULE_PREFIX_V1 = "spack.pkg."
-PKG_MODULE_PREFIX_V2 = "spack_repo."
+#: Package modules are imported as spack.pkg.<repo-namespace>.<pkg-name>
+ROOT_PYTHON_NAMESPACE = "spack.pkg"
 
 _API_REGEX = re.compile(r"^v(\d+)\.(\d+)$")
 
 
-def is_package_module(fullname: str) -> bool:
-    """Check if the given module is a package module."""
-    return fullname.startswith(PKG_MODULE_PREFIX_V1) or fullname.startswith(PKG_MODULE_PREFIX_V2)
+def python_package_for_repo(namespace):
+    """Returns the full namespace of a repository, given its relative one
+
+    For instance:
+
+        python_package_for_repo('builtin') == 'spack.pkg.builtin'
+
+    Args:
+        namespace (str): repo namespace
+    """
+    return "{0}.{1}".format(ROOT_PYTHON_NAMESPACE, namespace)
 
 
-def namespace_from_fullname(fullname: str) -> str:
+def namespace_from_fullname(fullname):
     """Return the repository namespace only for the full module name.
 
     For instance:
 
-        namespace_from_fullname("spack.pkg.builtin.hdf5") == "builtin"
-        namespace_from_fullname("spack_repo.x.y.z.packages.pkg_name.package") == "x.y.z"
+        namespace_from_fullname('spack.pkg.builtin.hdf5') == 'builtin'
 
     Args:
-        fullname: full name for the Python module
+        fullname (str): full name for the Python module
     """
-    if fullname.startswith(PKG_MODULE_PREFIX_V1):
-        namespace, _, _ = fullname.rpartition(".")
-        return namespace[len(PKG_MODULE_PREFIX_V1) :]
-    elif fullname.startswith(PKG_MODULE_PREFIX_V2) and fullname.endswith(".package"):
-        return ".".join(fullname.split(".")[1:-3])
-    return fullname
+    namespace, dot, module = fullname.rpartition(".")
+    prefix_and_dot = "{0}.".format(ROOT_PYTHON_NAMESPACE)
+    if namespace.startswith(prefix_and_dot):
+        namespace = namespace[len(prefix_and_dot) :]
+    return namespace
 
 
 class SpackNamespaceLoader:
@@ -86,13 +92,34 @@ def exec_module(self, module):
 
 
 class ReposFinder:
-    """MetaPathFinder class that loads a Python module corresponding to an API v1 Spack package.
+    """MetaPathFinder class that loads a Python module corresponding to a Spack package.
 
     Returns a loader based on the inspection of the current repository list.
     """
 
-    #: The current list of repositories.
-    repo_path: "RepoPath"
+    def __init__(self):
+        self._repo_init = _path
+        self._repo = None
+
+    @property
+    def current_repository(self):
+        if self._repo is None:
+            self._repo = self._repo_init()
+        return self._repo
+
+    @current_repository.setter
+    def current_repository(self, value):
+        self._repo = value
+
+    @contextlib.contextmanager
+    def switch_repo(self, substitute: "RepoType"):
+        """Switch the current repository list for the duration of the context manager."""
+        old = self._repo
+        try:
+            self._repo = substitute
+            yield
+        finally:
+            self._repo = old
 
     def find_spec(self, fullname, python_path, target=None):
         # "target" is not None only when calling importlib.reload()
@@ -100,7 +127,7 @@ def find_spec(self, fullname, python_path, target=None):
             raise RuntimeError('cannot reload module "{0}"'.format(fullname))
 
         # Preferred API from https://peps.python.org/pep-0451/
-        if not fullname.startswith(PKG_MODULE_PREFIX_V1) and fullname != "spack.pkg":
+        if not fullname.startswith(ROOT_PYTHON_NAMESPACE):
             return None
 
         loader = self.compute_loader(fullname)
@@ -108,16 +135,18 @@ def find_spec(self, fullname, python_path, target=None):
             return None
         return importlib.util.spec_from_loader(fullname, loader)
 
-    def compute_loader(self, fullname: str):
+    def compute_loader(self, fullname):
         # namespaces are added to repo, and package modules are leaves.
         namespace, dot, module_name = fullname.rpartition(".")
 
         # If it's a module in some repo, or if it is the repo's namespace, let the repo handle it.
+        is_repo_path = isinstance(self.current_repository, RepoPath)
+        if is_repo_path:
+            repos = self.current_repository.repos
+        else:
+            repos = [self.current_repository]
 
-        if not hasattr(self, "repo_path"):
-            return None
-
-        for repo in self.repo_path.repos:
+        for repo in repos:
             # We are using the namespace of the repo and the repo contains the package
             if namespace == repo.full_namespace:
                 # With 2 nested conditionals we can call "repo.real_name" only once
@@ -132,7 +161,7 @@ def compute_loader(self, fullname: str):
 
         # No repo provides the namespace, but it is a valid prefix of
         # something in the RepoPath.
-        if self.repo_path.by_namespace.is_prefix(fullname[len(PKG_MODULE_PREFIX_V1) :]):
+        if is_repo_path and self.current_repository.by_namespace.is_prefix(fullname):
             return SpackNamespaceLoader()
 
         return None
@@ -150,12 +179,12 @@ def compute_loader(self, fullname: str):
 NOT_PROVIDED = object()
 
 
-def builtin_repo() -> "Repo":
+def packages_path():
     """Get the test repo if it is active, otherwise the builtin repo."""
     try:
-        return PATH.get_repo("builtin.mock")
+        return PATH.get_repo("builtin.mock").packages_path
     except UnknownNamespaceError:
-        return PATH.get_repo("builtin")
+        return PATH.get_repo("builtin").packages_path
 
 
 class GitExe:
@@ -163,25 +192,24 @@ class GitExe:
     # invocations.
     #
     # Not using -C as that is not supported for git < 1.8.5.
-    def __init__(self, packages_path: str):
+    def __init__(self):
         self._git_cmd = spack.util.git.git(required=True)
-        self.packages_dir = packages_path
 
-    def __call__(self, *args, **kwargs) -> str:
-        with working_dir(self.packages_dir):
-            return self._git_cmd(*args, **kwargs, output=str)
+    def __call__(self, *args, **kwargs):
+        with working_dir(packages_path()):
+            return self._git_cmd(*args, **kwargs)
 
 
-def list_packages(rev: str, repo: "Repo") -> List[str]:
+def list_packages(rev):
     """List all packages associated with the given revision"""
-    git = GitExe(repo.packages_path)
+    git = GitExe()
 
     # git ls-tree does not support ... merge-base syntax, so do it manually
     if rev.endswith("..."):
         ref = rev.replace("...", "")
-        rev = git("merge-base", ref, "HEAD").strip()
+        rev = git("merge-base", ref, "HEAD", output=str).strip()
 
-    output = git("ls-tree", "-r", "--name-only", rev)
+    output = git("ls-tree", "-r", "--name-only", rev, output=str)
 
     # recursively list the packages directory
     package_paths = [
@@ -189,56 +217,54 @@ def list_packages(rev: str, repo: "Repo") -> List[str]:
     ]
 
     # take the directory names with one-level-deep package files
-    package_names = [
-        nm.pkg_dir_to_pkg_name(line[0], repo.package_api)
-        for line in package_paths
-        if len(line) == 2
-    ]
+    package_names = sorted(set([line[0] for line in package_paths if len(line) == 2]))
 
-    return sorted(set(package_names))
+    return package_names
 
 
-def diff_packages(rev1: str, rev2: str, repo: "Repo") -> Tuple[Set[str], Set[str]]:
+def diff_packages(rev1, rev2):
     """Compute packages lists for the two revisions and return a tuple
     containing all the packages in rev1 but not in rev2 and all the
     packages in rev2 but not in rev1."""
-    p1 = set(list_packages(rev1, repo))
-    p2 = set(list_packages(rev2, repo))
+    p1 = set(list_packages(rev1))
+    p2 = set(list_packages(rev2))
     return p1.difference(p2), p2.difference(p1)
 
 
-def get_all_package_diffs(type: str, repo: "Repo", rev1="HEAD^1", rev2="HEAD") -> Set[str]:
-    """Get packages changed, added, or removed (or any combination of those) since a commit.
+def get_all_package_diffs(type, rev1="HEAD^1", rev2="HEAD"):
+    """Show packages changed, added, or removed (or any combination of those)
+       since a commit.
 
     Arguments:
 
-        type: String containing one or more of 'A', 'R', 'C'
-        rev1: Revision to compare against, default is 'HEAD^'
-        rev2: Revision to compare to rev1, default is 'HEAD'
+        type (str): String containing one or more of 'A', 'R', 'C'
+        rev1 (str): Revision to compare against, default is 'HEAD^'
+        rev2 (str): Revision to compare to rev1, default is 'HEAD'
+
+    Returns:
+
+        A set contain names of affected packages.
     """
     lower_type = type.lower()
     if not re.match("^[arc]*$", lower_type):
         tty.die(
-            f"Invalid change type: '{type}'. "
-            "Can contain only A (added), R (removed), or C (changed)"
+            "Invald change type: '%s'." % type,
+            "Can contain only A (added), R (removed), or C (changed)",
         )
 
-    removed, added = diff_packages(rev1, rev2, repo)
+    removed, added = diff_packages(rev1, rev2)
 
-    git = GitExe(repo.packages_path)
-    out = git("diff", "--relative", "--name-only", rev1, rev2).strip()
+    git = GitExe()
+    out = git("diff", "--relative", "--name-only", rev1, rev2, output=str).strip()
 
     lines = [] if not out else re.split(r"\s+", out)
-    changed: Set[str] = set()
+    changed = set()
     for path in lines:
-        dir_name, _, _ = path.partition("/")
-        if not nm.valid_module_name(dir_name, repo.package_api):
-            continue
-        pkg_name = nm.pkg_dir_to_pkg_name(dir_name, repo.package_api)
+        pkg_name, _, _ = path.partition("/")
         if pkg_name not in added and pkg_name not in removed:
             changed.add(pkg_name)
 
-    packages: Set[str] = set()
+    packages = set()
     if "a" in lower_type:
         packages |= added
     if "r" in lower_type:
@@ -249,14 +275,14 @@ def get_all_package_diffs(type: str, repo: "Repo", rev1="HEAD^1", rev2="HEAD") -
     return packages
 
 
-def add_package_to_git_stage(packages: List[str], repo: "Repo") -> None:
+def add_package_to_git_stage(packages):
     """add a package to the git stage with `git add`"""
-    git = GitExe(repo.packages_path)
+    git = GitExe()
 
     for pkg_name in packages:
         filename = PATH.filename_for_package_name(pkg_name)
         if not os.path.isfile(filename):
-            tty.die(f"No such package: {pkg_name}.  Path does not exist:", filename)
+            tty.die("No such package: %s.  Path does not exist:" % pkg_name, filename)
 
         git("add", filename)
 
@@ -326,10 +352,9 @@ class FastPackageChecker(collections.abc.Mapping):
     #: Global cache, reused by every instance
     _paths_cache: Dict[str, Dict[str, os.stat_result]] = {}
 
-    def __init__(self, packages_path: str, package_api: Tuple[int, int]):
+    def __init__(self, packages_path):
         # The path of the repository managed by this instance
         self.packages_path = packages_path
-        self.package_api = package_api
 
         # If the cache we need is not there yet, then build it appropriately
         if packages_path not in self._paths_cache:
@@ -354,38 +379,41 @@ def _create_new_cache(self) -> Dict[str, os.stat_result]:
         # Create a dictionary that will store the mapping between a
         # package name and its stat info
         cache: Dict[str, os.stat_result] = {}
-        with os.scandir(self.packages_path) as entries:
-            for entry in entries:
-                # Construct the file name from the directory
-                pkg_file = os.path.join(entry.path, package_file_name)
+        for pkg_name in os.listdir(self.packages_path):
+            # Skip non-directories in the package root.
+            pkg_dir = os.path.join(self.packages_path, pkg_name)
 
-                try:
-                    sinfo = os.stat(pkg_file)
-                except OSError as e:
-                    if e.errno in (errno.ENOENT, errno.ENOTDIR):
-                        # No package.py file here.
-                        continue
-                    elif e.errno == errno.EACCES:
-                        tty.warn(f"Can't read package file {pkg_file}.")
-                        continue
-                    raise e
-
-                # If it's not a file, skip it.
-                if not stat.S_ISREG(sinfo.st_mode):
-                    continue
-
-                # Only consider package.py files in directories that are valid module names under
-                # the current package API
-                if not nm.valid_module_name(entry.name, self.package_api):
-                    x, y = self.package_api
+            # Warn about invalid names that look like packages.
+            if not nm.valid_module_name(pkg_name):
+                if not pkg_name.startswith(".") and pkg_name != "repo.yaml":
                     tty.warn(
-                        f"Package {pkg_file} cannot be used because `{entry.name}` is not a valid "
-                        f"Spack package module name for Package API v{x}.{y}."
+                        'Skipping package at {0}. "{1}" is not '
+                        "a valid Spack module name.".format(pkg_dir, pkg_name)
                     )
-                    continue
+                continue
 
-                # Store the stat info by package name.
-                cache[nm.pkg_dir_to_pkg_name(entry.name, self.package_api)] = sinfo
+            # Construct the file name from the directory
+            pkg_file = os.path.join(self.packages_path, pkg_name, package_file_name)
+
+            # Use stat here to avoid lots of calls to the filesystem.
+            try:
+                sinfo = os.stat(pkg_file)
+            except OSError as e:
+                if e.errno == errno.ENOENT:
+                    # No package.py file here.
+                    continue
+                elif e.errno == errno.EACCES:
+                    tty.warn("Can't read package file %s." % pkg_file)
+                    continue
+                raise e
+
+            # If it's not a file, skip it.
+            if stat.S_ISDIR(sinfo.st_mode):
+                continue
+
+            # If it is a file, then save the stats under the
+            # appropriate key
+            cache[pkg_name] = sinfo
 
         return cache
 
@@ -638,6 +666,7 @@ def __init__(
                 if isinstance(repo, str):
                     assert cache is not None, "cache must hold a value, when repo is a string"
                     repo = Repo(repo, cache=cache, overrides=overrides)
+                repo.finder(self)
                 self.put_last(repo)
             except RepoError as e:
                 tty.warn(
@@ -647,20 +676,6 @@ def __init__(
                     f"    spack repo rm {repo}",
                 )
 
-    def enable(self) -> None:
-        """Set the relevant search paths for package module loading"""
-        REPOS_FINDER.repo_path = self
-        for p in reversed(self.python_paths()):
-            if p not in sys.path:
-                sys.path.insert(0, p)
-
-    def disable(self) -> None:
-        """Disable the search paths for package module loading"""
-        del REPOS_FINDER.repo_path
-        for p in self.python_paths():
-            if p in sys.path:
-                sys.path.remove(p)
-
     def ensure_unwrapped(self) -> "RepoPath":
         """Ensure we unwrap this object from any dynamic wrapper (like Singleton)"""
         return self
@@ -673,7 +688,7 @@ def put_first(self, repo: "Repo") -> None:
             return
 
         self.repos.insert(0, repo)
-        self.by_namespace[repo.namespace] = repo
+        self.by_namespace[repo.full_namespace] = repo
 
     def put_last(self, repo):
         """Add repo last in the search path."""
@@ -685,8 +700,8 @@ def put_last(self, repo):
         self.repos.append(repo)
 
         # don't mask any higher-precedence repos with same namespace
-        if repo.namespace not in self.by_namespace:
-            self.by_namespace[repo.namespace] = repo
+        if repo.full_namespace not in self.by_namespace:
+            self.by_namespace[repo.full_namespace] = repo
 
     def remove(self, repo):
         """Remove a repo from the search path."""
@@ -695,9 +710,10 @@ def remove(self, repo):
 
     def get_repo(self, namespace: str) -> "Repo":
         """Get a repository by namespace."""
-        if namespace not in self.by_namespace:
+        full_namespace = python_package_for_repo(namespace)
+        if full_namespace not in self.by_namespace:
             raise UnknownNamespaceError(namespace)
-        return self.by_namespace[namespace]
+        return self.by_namespace[full_namespace]
 
     def first_repo(self) -> Optional["Repo"]:
         """Get the first repo in precedence order."""
@@ -805,9 +821,10 @@ def repo_for_pkg(self, spec: Union[str, "spack.spec.Spec"]) -> "Repo":
         # If the spec already has a namespace, then return the
         # corresponding repo if we know about it.
         if namespace:
-            if namespace not in self.by_namespace:
+            fullspace = python_package_for_repo(namespace)
+            if fullspace not in self.by_namespace:
                 raise UnknownNamespaceError(namespace, name=name)
-            return self.by_namespace[namespace]
+            return self.by_namespace[fullspace]
 
         # If there's no namespace, search in the RepoPath.
         for repo in self.repos:
@@ -828,10 +845,6 @@ def get(self, spec: "spack.spec.Spec") -> "spack.package_base.PackageBase":
         assert isinstance(spec, spack.spec.Spec) and spec.concrete, msg
         return self.repo_for_pkg(spec).get(spec)
 
-    def python_paths(self) -> List[str]:
-        """Return a list of all the Python paths in the repos."""
-        return [repo.python_path for repo in self.repos if repo.python_path]
-
     def get_pkg_class(self, pkg_name: str) -> Type["spack.package_base.PackageBase"]:
         """Find a class for the spec's package and return the class object."""
         return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name)
@@ -929,30 +942,6 @@ def _parse_package_api_version(
     )
 
 
-def _validate_and_normalize_subdir(subdir: Any, root: str, package_api: Tuple[int, int]) -> str:
-    if not isinstance(subdir, str):
-        raise BadRepoError(f"Invalid subdirectory '{subdir}' in '{root}'. Must be a string")
-
-    if package_api < (2, 0):
-        return subdir  # In v1.x we did not validate subdir names
-
-    if subdir in (".", ""):
-        raise BadRepoError(
-            f"Invalid subdirectory '{subdir}' in '{root}'. Use a symlink packages -> . instead"
-        )
-
-    # Otherwise we expect a directory name (not path) that can be used as a Python module.
-    if os.sep in subdir:
-        raise BadRepoError(
-            f"Invalid subdirectory '{subdir}' in '{root}'. Expected a directory name, not a path"
-        )
-    if not nm.valid_module_name(subdir, package_api):
-        raise BadRepoError(
-            f"Invalid subdirectory '{subdir}' in '{root}'. Must be a valid Python module name"
-        )
-    return subdir
-
-
 class Repo:
     """Class representing a package repository in the filesystem.
 
@@ -973,8 +962,6 @@ class Repo:
         :py:data:`spack.package_api_version`.
     """
 
-    namespace: str
-
     def __init__(
         self,
         root: str,
@@ -1004,82 +991,38 @@ def check(condition, msg):
 
         # Read configuration and validate namespace
         config = self._read_config()
-
-        self.package_api = _parse_package_api_version(config)
-        self.subdirectory = _validate_and_normalize_subdir(
-            config.get("subdirectory", packages_dir_name), root, self.package_api
-        )
-        self.packages_path = os.path.join(self.root, self.subdirectory)
-
         check(
-            os.path.isdir(self.packages_path),
-            f"No directory '{self.subdirectory}' found in '{root}'",
+            "namespace" in config,
+            f"{os.path.join(root, repo_config_name)} must define a namespace.",
         )
 
-        # The parent dir of spack_repo/ which should be added to sys.path for api v2.x
-        self.python_path: Optional[str] = None
-
-        if self.package_api < (2, 0):
-            check(
-                "namespace" in config,
-                f"{os.path.join(root, repo_config_name)} must define a namespace.",
-            )
-            self.namespace = config["namespace"]
-            # Note: for Package API v1.x the namespace validation always had bugs, which won't be
-            # fixed for compatibility reasons. The regex is missing "$" at the end, and it claims
-            # to test for valid identifiers, but fails to split on `.` first.
-            check(
-                isinstance(self.namespace, str)
-                and re.match(r"[a-zA-Z][a-zA-Z0-9_.]+", self.namespace),
-                f"Invalid namespace '{self.namespace}' in repo '{self.root}'. "
-                "Namespaces must be valid python identifiers separated by '.'",
-            )
-        else:
-            # From Package API v2.0 the namespace follows from the directory structure.
-            check(
-                f"{os.sep}spack_repo{os.sep}" in self.root,
-                f"Invalid repository path '{self.root}'. "
-                f"Path must contain 'spack_repo{os.sep}'",
-            )
-            derived_namespace = self.root.rpartition(f"spack_repo{os.sep}")[2].replace(os.sep, ".")
-            if "namespace" in config:
-                self.namespace = config["namespace"]
-
-                check(
-                    isinstance(self.namespace, str) and self.namespace == derived_namespace,
-                    f"Namespace '{self.namespace}' should be {derived_namespace} or omitted in "
-                    f"{os.path.join(root, repo_config_name)}",
-                )
-            else:
-                self.namespace = derived_namespace
-
-            # strip the namespace directories from the root path to get the python path
-            # e.g. /my/pythonpath/spack_repo/x/y/z -> /my/pythonpath
-            python_path = self.root
-            for _ in self.namespace.split("."):
-                python_path = os.path.dirname(python_path)
-            self.python_path = os.path.dirname(python_path)
-
-            # check that all subdirectories are valid module names
-            check(
-                all(nm.valid_module_name(x, self.package_api) for x in self.namespace.split(".")),
-                f"Invalid namespace '{self.namespace}' in repo '{self.root}'",
-            )
+        self.namespace: str = config["namespace"]
+        check(
+            re.match(r"[a-zA-Z][a-zA-Z0-9_.]+", self.namespace),
+            f"Invalid namespace '{self.namespace}' in repo '{self.root}'. "
+            "Namespaces must be valid python identifiers separated by '.'",
+        )
 
         # Set up 'full_namespace' to include the super-namespace
-        if self.package_api < (2, 0):
-            self.full_namespace = f"{PKG_MODULE_PREFIX_V1}{self.namespace}"
-        elif self.subdirectory == ".":
-            self.full_namespace = f"{PKG_MODULE_PREFIX_V2}{self.namespace}"
-        else:
-            self.full_namespace = f"{PKG_MODULE_PREFIX_V2}{self.namespace}.{self.subdirectory}"
+        self.full_namespace = python_package_for_repo(self.namespace)
 
         # Keep name components around for checking prefixes.
         self._names = self.full_namespace.split(".")
 
+        packages_dir: str = config.get("subdirectory", packages_dir_name)
+        self.packages_path = os.path.join(self.root, packages_dir)
+        check(
+            os.path.isdir(self.packages_path), f"No directory '{packages_dir}' found in '{root}'"
+        )
+
+        self.package_api = _parse_package_api_version(config)
+
         # Class attribute overrides by package name
         self.overrides = overrides or {}
 
+        # Optional reference to a RepoPath to influence module import from spack.pkg
+        self._finder: Optional[RepoPath] = None
+
         # Maps that goes from package name to corresponding file stat
         self._fast_package_checker: Optional[FastPackageChecker] = None
 
@@ -1087,33 +1030,27 @@ def check(condition, msg):
         self._repo_index: Optional[RepoIndex] = None
         self._cache = cache
 
-    @property
-    def package_api_str(self) -> str:
-        return f"v{self.package_api[0]}.{self.package_api[1]}"
+    def finder(self, value: RepoPath) -> None:
+        self._finder = value
 
     def real_name(self, import_name: str) -> Optional[str]:
         """Allow users to import Spack packages using Python identifiers.
 
-        In Package API v1.x, there was no canonical module name for a package, and package's dir
-        was not necessarily a valid Python module name. For that case we have to guess the actual
-        package directory. From Package API v2.0 there is a one-to-one mapping between Spack
-        package names and Python module names, so there is no guessing.
+        A python identifier might map to many different Spack package
+        names due to hyphen/underscore ambiguity.
 
-        For Packge API v1.x we support the following one-to-many mappings:
-            num3proxy -> 3proxy
+        Easy example:
+            num3proxy   -> 3proxy
+
+        Ambiguous:
             foo_bar -> foo_bar, foo-bar
+
+        More ambiguous:
             foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz
         """
-        if self.package_api >= (2, 0):
-            if nm.pkg_dir_to_pkg_name(import_name, package_api=self.package_api) in self:
-                return import_name
-            return None
-
         if import_name in self:
             return import_name
 
-        # For v1 generate the possible package names from a module name, and return the first
-        # package name that exists in this repo.
         options = nm.possible_spack_module_names(import_name)
         try:
             options.remove(import_name)
@@ -1246,9 +1183,7 @@ def extensions_for(
     def dirname_for_package_name(self, pkg_name: str) -> str:
         """Given a package name, get the directory containing its package.py file."""
         _, unqualified_name = self.partition_package_name(pkg_name)
-        return os.path.join(
-            self.packages_path, nm.pkg_name_to_pkg_dir(unqualified_name, self.package_api)
-        )
+        return os.path.join(self.packages_path, unqualified_name)
 
     def filename_for_package_name(self, pkg_name: str) -> str:
         """Get the filename for the module we should load for a particular
@@ -1265,7 +1200,7 @@ def filename_for_package_name(self, pkg_name: str) -> str:
     @property
     def _pkg_checker(self) -> FastPackageChecker:
         if self._fast_package_checker is None:
-            self._fast_package_checker = FastPackageChecker(self.packages_path, self.package_api)
+            self._fast_package_checker = FastPackageChecker(self.packages_path)
         return self._fast_package_checker
 
     def all_package_names(self, include_virtuals: bool = False) -> List[str]:
@@ -1277,9 +1212,7 @@ def all_package_names(self, include_virtuals: bool = False) -> List[str]:
 
     def package_path(self, name: str) -> str:
         """Get path to package.py file for this repo."""
-        return os.path.join(
-            self.packages_path, nm.pkg_name_to_pkg_dir(name, self.package_api), package_file_name
-        )
+        return os.path.join(self.packages_path, name, package_file_name)
 
     def all_package_paths(self) -> Generator[str, None, None]:
         for name in self.all_package_names():
@@ -1337,17 +1270,15 @@ def get_pkg_class(self, pkg_name: str) -> Type["spack.package_base.PackageBase"]
         package. Then extracts the package class from the module
         according to Spack's naming convention.
         """
-        _, pkg_name = self.partition_package_name(pkg_name)
-        fullname = f"{self.full_namespace}.{nm.pkg_name_to_pkg_dir(pkg_name, self.package_api)}"
-        if self.package_api >= (2, 0):
-            fullname += ".package"
-
-        class_name = nm.pkg_name_to_class_name(pkg_name)
+        namespace, pkg_name = self.partition_package_name(pkg_name)
+        class_name = nm.mod_to_class(pkg_name)
+        fullname = f"{self.full_namespace}.{pkg_name}"
 
         try:
-            module = importlib.import_module(fullname)
-        except ImportError as e:
-            raise UnknownPackageError(fullname) from e
+            with REPOS_FINDER.switch_repo(self._finder or self):
+                module = importlib.import_module(fullname)
+        except ImportError:
+            raise UnknownPackageError(fullname)
         except Exception as e:
             msg = f"cannot load package '{pkg_name}' from the '{self.namespace}' repository: {e}"
             raise RepoError(msg) from e
@@ -1438,71 +1369,46 @@ def partition_package_name(pkg_name: str) -> Tuple[str, str]:
     return namespace, pkg_name
 
 
-def get_repo_yaml_dir(
-    root: str, namespace: Optional[str], package_api: Tuple[int, int]
-) -> Tuple[str, str]:
-    """Returns the directory where repo.yaml is located and the effective namespace."""
-    if package_api < (2, 0):
-        namespace = namespace or os.path.basename(root)
-        # This ad-hoc regex is left for historical reasons, and should not have a breaking change.
-        if not re.match(r"\w[\.\w-]*", namespace):
-            raise InvalidNamespaceError(f"'{namespace}' is not a valid namespace.")
-        return root, namespace
-
-    # Package API v2 has <root>/spack_repo/<namespace>/<subdir> structure and requires a namespace
-    if namespace is None:
-        raise InvalidNamespaceError("Namespace must be provided.")
-
-    # if namespace has dots those translate to subdirs of further namespace packages.
-    namespace_components = namespace.split(".")
-
-    if not all(nm.valid_module_name(n, package_api=package_api) for n in namespace_components):
-        raise InvalidNamespaceError(f"'{namespace}' is not a valid namespace." % namespace)
-
-    return os.path.join(root, "spack_repo", *namespace_components), namespace
-
-
-def create_repo(
-    root,
-    namespace: Optional[str] = None,
-    subdir: str = packages_dir_name,
-    package_api: Tuple[int, int] = spack.package_api_version,
-) -> Tuple[str, str]:
+def create_repo(root, namespace=None, subdir=packages_dir_name):
     """Create a new repository in root with the specified namespace.
 
     If the namespace is not provided, use basename of root.
     Return the canonicalized path and namespace of the created repository.
     """
     root = spack.util.path.canonicalize_path(root)
-    repo_yaml_dir, namespace = get_repo_yaml_dir(os.path.abspath(root), namespace, package_api)
+    if not namespace:
+        namespace = os.path.basename(root)
 
-    existed = True
-    try:
-        dir_entry = next(os.scandir(repo_yaml_dir), None)
-    except OSError as e:
-        if e.errno == errno.ENOENT:
-            existed = False
-            dir_entry = None
-        else:
-            raise BadRepoError(f"Cannot create new repo in {root}: {e}")
+    if not re.match(r"\w[\.\w-]*", namespace):
+        raise InvalidNamespaceError("'%s' is not a valid namespace." % namespace)
 
-    if dir_entry is not None:
-        raise BadRepoError(f"Cannot create new repo in {root}: directory is not empty.")
+    existed = False
+    if os.path.exists(root):
+        if os.path.isfile(root):
+            raise BadRepoError("File %s already exists and is not a directory" % root)
+        elif os.path.isdir(root):
+            if not os.access(root, os.R_OK | os.W_OK):
+                raise BadRepoError("Cannot create new repo in %s: cannot access directory." % root)
+            if os.listdir(root):
+                raise BadRepoError("Cannot create new repo in %s: directory is not empty." % root)
+        existed = True
 
-    config_path = os.path.join(repo_yaml_dir, repo_config_name)
-
-    subdir = _validate_and_normalize_subdir(subdir, root, package_api)
-
-    packages_path = os.path.join(repo_yaml_dir, subdir)
+    full_path = os.path.realpath(root)
+    parent = os.path.dirname(full_path)
+    if not os.access(parent, os.R_OK | os.W_OK):
+        raise BadRepoError("Cannot create repository in %s: can't access parent!" % root)
 
     try:
+        config_path = os.path.join(root, repo_config_name)
+        packages_path = os.path.join(root, subdir)
+
         fs.mkdirp(packages_path)
         with open(config_path, "w", encoding="utf-8") as config:
             config.write("repo:\n")
             config.write(f"  namespace: '{namespace}'\n")
             if subdir != packages_dir_name:
                 config.write(f"  subdirectory: '{subdir}'\n")
-            x, y = package_api
+            x, y = spack.package_api_version
             config.write(f"  api: v{x}.{y}\n")
 
     except OSError as e:
@@ -1515,27 +1421,28 @@ def create_repo(
 
         raise BadRepoError(
             "Failed to create new repository in %s." % root, "Caused by %s: %s" % (type(e), e)
-        ) from e
+        )
 
-    return repo_yaml_dir, namespace
+    return full_path, namespace
 
 
-def from_path(path: str) -> Repo:
+def from_path(path: str) -> "Repo":
     """Returns a repository from the path passed as input. Injects the global misc cache."""
     return Repo(path, cache=spack.caches.MISC_CACHE)
 
 
-def create_or_construct(
-    root: str,
-    namespace: Optional[str] = None,
-    package_api: Tuple[int, int] = spack.package_api_version,
-) -> Repo:
+def create_or_construct(path, namespace=None):
     """Create a repository, or just return a Repo if it already exists."""
-    repo_yaml_dir, _ = get_repo_yaml_dir(root, namespace, package_api)
-    if not os.path.exists(repo_yaml_dir):
-        fs.mkdirp(root)
-        create_repo(root, namespace=namespace, package_api=package_api)
-    return from_path(repo_yaml_dir)
+    if not os.path.exists(path):
+        fs.mkdirp(path)
+        create_repo(path, namespace)
+    return from_path(path)
+
+
+def _path(configuration=None):
+    """Get the singleton RepoPath instance for Spack."""
+    configuration = configuration or spack.config.CONFIG
+    return create(configuration=configuration)
 
 
 def create(configuration: spack.config.Configuration) -> RepoPath:
@@ -1560,10 +1467,8 @@ def create(configuration: spack.config.Configuration) -> RepoPath:
     return RepoPath(*repo_dirs, cache=spack.caches.MISC_CACHE, overrides=overrides)
 
 
-#: Global package repository instance.
-PATH: RepoPath = llnl.util.lang.Singleton(
-    lambda: create(configuration=spack.config.CONFIG)
-)  # type: ignore[assignment]
+#: Singleton repo path instance
+PATH: RepoPath = llnl.util.lang.Singleton(_path)  # type: ignore
 
 # Add the finder to sys.meta_path
 REPOS_FINDER = ReposFinder()
@@ -1589,37 +1494,28 @@ def use_repositories(
     Returns:
         Corresponding RepoPath object
     """
+    global PATH
     paths = [getattr(x, "root", x) for x in paths_and_repos]
-    scope_name = f"use-repo-{uuid.uuid4()}"
+    scope_name = "use-repo-{}".format(uuid.uuid4())
     repos_key = "repos:" if override else "repos"
     spack.config.CONFIG.push_scope(
         spack.config.InternalConfigScope(name=scope_name, data={repos_key: paths})
     )
-    old_repo, new_repo = PATH, create(configuration=spack.config.CONFIG)
-    old_repo.disable()
-    enable_repo(new_repo)
+    PATH, saved = create(configuration=spack.config.CONFIG), PATH
     try:
-        yield new_repo
+        with REPOS_FINDER.switch_repo(PATH):  # type: ignore
+            yield PATH
     finally:
         spack.config.CONFIG.remove_scope(scope_name=scope_name)
-        enable_repo(old_repo)
-
-
-def enable_repo(repo_path: RepoPath) -> None:
-    """Set the global package repository and make them available in module search paths."""
-    global PATH
-    PATH = repo_path
-    PATH.enable()
+        PATH = saved
 
 
 class MockRepositoryBuilder:
     """Build a mock repository in a directory"""
 
     def __init__(self, root_directory, namespace=None):
-        namespace = namespace or "".join(random.choice(string.ascii_lowercase) for _ in range(10))
-        repo_root = os.path.join(root_directory, namespace)
-        os.mkdir(repo_root)
-        self.root, self.namespace = create_repo(repo_root, namespace)
+        namespace = namespace or "".join(random.choice(string.ascii_uppercase) for _ in range(10))
+        self.root, self.namespace = create_repo(str(root_directory), namespace)
 
     def add_package(self, name, dependencies=None):
         """Create a mock package in the repository, using a Jinja2 template.
@@ -1631,7 +1527,7 @@ def add_package(self, name, dependencies=None):
                 ``spack.dependency.default_deptype`` and ``spack.spec.Spec()`` are used.
         """
         dependencies = dependencies or []
-        context = {"cls_name": nm.pkg_name_to_class_name(name), "dependencies": dependencies}
+        context = {"cls_name": nm.mod_to_class(name), "dependencies": dependencies}
         template = spack.tengine.make_environment().get_template("mock-repository/package.pyt")
         text = template.render(context)
         package_py = self.recipe_filename(name)
@@ -1643,10 +1539,8 @@ def remove(self, name):
         package_py = self.recipe_filename(name)
         shutil.rmtree(os.path.dirname(package_py))
 
-    def recipe_filename(self, name: str):
-        return os.path.join(
-            self.root, "packages", nm.pkg_name_to_pkg_dir(name, package_api=(2, 0)), "package.py"
-        )
+    def recipe_filename(self, name):
+        return os.path.join(self.root, "packages", name, "package.py")
 
 
 class RepoError(spack.error.SpackError):
@@ -1696,10 +1590,7 @@ def __init__(self, name, repo=None):
 
                 # We need to compare the base package name
                 pkg_name = name.rsplit(".", 1)[-1]
-                try:
-                    similar = difflib.get_close_matches(pkg_name, repo.all_package_names())
-                except Exception:
-                    similar = []
+                similar = difflib.get_close_matches(pkg_name, repo.all_package_names())
 
                 if 1 <= len(similar) <= 5:
                     long_msg += "\n\nDid you mean one of the following packages?\n  "

lib/spack/spack/schema/buildcache_spec.py

@@ -19,6 +19,10 @@
         "additionalProperties": True,
         "items": spack.schema.spec.properties,
     },
+    "binary_cache_checksum": {
+        "type": "object",
+        "properties": {"hash_algorithm": {"type": "string"}, "hash": {"type": "string"}},
+    },
     "buildcache_layout_version": {"type": "number"},
 }
 
@@ -26,6 +30,6 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Spack buildcache specfile schema",
     "type": "object",
-    "additionalProperties": True,
+    "additionalProperties": False,
     "properties": properties,
 }

lib/spack/spack/schema/url_buildcache_manifest.py

@@ -1,45 +0,0 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-"""Schema for buildcache entry manifest file
-
-.. literalinclude:: _spack_root/lib/spack/spack/schema/url_buildcache_manifest.py
-   :lines: 11-
-"""
-from typing import Any, Dict
-
-properties: Dict[str, Any] = {
-    "version": {"type": "integer"},
-    "data": {
-        "type": "array",
-        "items": {
-            "type": "object",
-            "required": [
-                "contentLength",
-                "mediaType",
-                "compression",
-                "checksumAlgorithm",
-                "checksum",
-            ],
-            "properties": {
-                "contentLength": {"type": "integer"},
-                "mediaType": {"type": "string"},
-                "compression": {"type": "string"},
-                "checksumAlgorithm": {"type": "string"},
-                "checksum": {"type": "string"},
-            },
-            "additionalProperties": True,
-        },
-    },
-}
-
-#: Full schema with metadata
-schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "Buildcache manifest schema",
-    "type": "object",
-    "required": ["version", "data"],
-    "additionalProperties": True,
-    "properties": properties,
-}

lib/spack/spack/solver/asp.py

@@ -287,33 +287,9 @@ def specify(spec):
     return spack.spec.Spec(spec)
 
 
-def remove_facts(
-    *to_be_removed: str,
-) -> Callable[[spack.spec.Spec, List[AspFunction]], List[AspFunction]]:
-    """Returns a transformation function that removes facts from the input list of facts."""
-
-    def _remove(spec: spack.spec.Spec, facts: List[AspFunction]) -> List[AspFunction]:
-        return list(filter(lambda x: x.args[0] not in to_be_removed, facts))
-
-    return _remove
-
-
-def remove_build_deps(spec: spack.spec.Spec, facts: List[AspFunction]) -> List[AspFunction]:
-    build_deps = {x.args[2]: x.args[1] for x in facts if x.args[0] == "depends_on"}
-    result = []
-    for x in facts:
-        current_name = x.args[1]
-        if current_name in build_deps:
-            x.name = "build_requirement"
-            result.append(fn.attr("build_requirement", build_deps[current_name], x))
-            continue
-
-        if x.args[0] == "depends_on":
-            continue
-
-        result.append(x)
-
-    return result
+def remove_node(spec: spack.spec.Spec, facts: List[AspFunction]) -> List[AspFunction]:
+    """Transformation that removes all "node" and "virtual_node" from the input list of facts."""
+    return list(filter(lambda x: x.args[0] not in ("node", "virtual_node"), facts))
 
 
 def all_libcs() -> Set[spack.spec.Spec]:
@@ -1311,8 +1287,12 @@ def on_model(model):
             result.raise_if_unsat()
 
             if result.satisfiable and result.unsolved_specs and setup.concretize_everything:
-                raise OutputDoesNotSatisfyInputError(result.unsolved_specs)
-
+                unsolved_str = Result.format_unsolved(result.unsolved_specs)
+                raise InternalConcretizerError(
+                    "Internal Spack error: the solver completed but produced specs"
+                    " that do not satisfy the request. Please report a bug at "
+                    f"https://github.com/spack/spack/issues\n\t{unsolved_str}"
+                )
             if conc_cache_enabled:
                 CONC_CACHE.store(problem_repr, result, self.control.statistics, test=setup.tests)
             concretization_stats = self.control.statistics
@@ -1755,17 +1735,15 @@ def define_variant(
             pkg_fact(fn.variant_condition(name, vid, cond_id))
 
         # record type so we can construct the variant when we read it back in
-        self.gen.fact(fn.variant_type(vid, variant_def.variant_type.string))
+        self.gen.fact(fn.variant_type(vid, variant_def.variant_type.value))
 
         if variant_def.sticky:
             pkg_fact(fn.variant_sticky(vid))
 
         # define defaults for this variant definition
-        if variant_def.multi:
-            for val in sorted(variant_def.make_default().values):
-                pkg_fact(fn.variant_default_value_from_package_py(vid, val))
-        else:
-            pkg_fact(fn.variant_default_value_from_package_py(vid, variant_def.default))
+        defaults = variant_def.make_default().value if variant_def.multi else [variant_def.default]
+        for val in sorted(defaults):
+            pkg_fact(fn.variant_default_value_from_package_py(vid, val))
 
         # define possible values for this variant definition
         values = variant_def.values
@@ -1793,9 +1771,7 @@ def define_variant(
 
             # make a spec indicating whether the variant has this conditional value
             variant_has_value = spack.spec.Spec()
-            variant_has_value.variants[name] = vt.VariantValue(
-                vt.VariantType.MULTI, name, (value.value,)
-            )
+            variant_has_value.variants[name] = spack.variant.AbstractVariant(name, value.value)
 
             if value.when:
                 # the conditional value is always "possible", but it imposes its when condition as
@@ -1908,7 +1884,7 @@ def condition(
 
         if not context:
             context = ConditionContext()
-            context.transform_imposed = remove_facts("node", "virtual_node")
+            context.transform_imposed = remove_node
 
         if imposed_spec:
             imposed_name = imposed_spec.name or imposed_name
@@ -2008,7 +1984,7 @@ def track_dependencies(input_spec, requirements):
                     return requirements + [fn.attr("track_dependencies", input_spec.name)]
 
                 def dependency_holds(input_spec, requirements):
-                    result = remove_facts("node", "virtual_node")(input_spec, requirements) + [
+                    result = remove_node(input_spec, requirements) + [
                         fn.attr(
                             "dependency_holds", pkg.name, input_spec.name, dt.flag_to_string(t)
                         )
@@ -2198,10 +2174,7 @@ def emit_facts_from_requirement_rules(self, rules: List[RequirementRule]):
                         pkg_name, ConstraintOrigin.REQUIRE
                     )
                     if not virtual:
-                        context.transform_required = remove_build_deps
-                        context.transform_imposed = remove_facts(
-                            "node", "virtual_node", "depends_on"
-                        )
+                        context.transform_imposed = remove_node
                     # else: for virtuals we want to emit "node" and
                     # "virtual_node" in imposed specs
 
@@ -2263,18 +2236,16 @@ def external_packages(self):
             if pkg_name not in self.pkgs:
                 continue
 
+            self.gen.h2(f"External package: {pkg_name}")
             # Check if the external package is buildable. If it is
             # not then "external(<pkg>)" is a fact, unless we can
             # reuse an already installed spec.
             external_buildable = data.get("buildable", True)
-            externals = data.get("externals", [])
-            if not external_buildable or externals:
-                self.gen.h2(f"External package: {pkg_name}")
-
             if not external_buildable:
                 self.gen.fact(fn.buildable_false(pkg_name))
 
             # Read a list of all the specs for this package
+            externals = data.get("externals", [])
             candidate_specs = [
                 spack.spec.parse_with_version_concrete(x["spec"]) for x in externals
             ]
@@ -2363,8 +2334,6 @@ def preferred_variants(self, pkg_name):
         if not preferred_variants:
             return
 
-        self.gen.h2(f"Package preferences: {pkg_name}")
-
         for variant_name in sorted(preferred_variants):
             variant = preferred_variants[variant_name]
 
@@ -2377,7 +2346,7 @@ def preferred_variants(self, pkg_name):
                 )
                 continue
 
-            for value in variant.values:
+            for value in variant.value_as_tuple:
                 for variant_def in variant_defs:
                     self.variant_values_from_specs.add((pkg_name, id(variant_def), value))
                 self.gen.fact(
@@ -2492,10 +2461,10 @@ def _spec_clauses(
             # TODO: variant="*" means 'variant is defined to something', which used to
             # be meaningless in concretization, as all variants had to be defined. But
             # now that variants can be conditional, it should force a variant to exist.
-            if not variant.values:
+            if variant.value == ("*",):
                 continue
 
-            for value in variant.values:
+            for value in variant.value_as_tuple:
                 # ensure that the value *can* be valid for the spec
                 if spec.name and not spec.concrete and not spack.repo.PATH.is_virtual(spec.name):
                     variant_defs = vt.prevalidate_variant_value(
@@ -2605,16 +2574,6 @@ def _spec_clauses(
                     # already-installed concrete specs.
                     if concrete_build_deps or dspec.depflag != dt.BUILD:
                         clauses.append(fn.attr("hash", dep.name, dep.dag_hash()))
-                    elif not concrete_build_deps and dspec.depflag:
-                        clauses.append(
-                            fn.attr(
-                                "concrete_build_dependency", spec.name, dep.name, dep.dag_hash()
-                            )
-                        )
-                        for virtual_name in dspec.virtuals:
-                            clauses.append(
-                                fn.attr("virtual_on_build_edge", spec.name, dep.name, virtual_name)
-                            )
 
                 # if the spec is abstract, descend into dependencies.
                 # if it's concrete, then the hashes above take care of dependency
@@ -3169,6 +3128,7 @@ def setup(
         for pkg in sorted(self.pkgs):
             self.gen.h2("Package rules: %s" % pkg)
             self.pkg_rules(pkg, tests=self.tests)
+            self.gen.h2("Package preferences: %s" % pkg)
             self.preferred_variants(pkg)
 
         self.gen.h1("Special variants")
@@ -3240,13 +3200,12 @@ def define_runtime_constraints(self) -> List[spack.spec.Spec]:
 
             # FIXME (compiler as nodes): think of using isinstance(compiler_cls, WrappedCompiler)
             # Add a dependency on the compiler wrapper
-            for language in ("c", "cxx", "fortran"):
-                recorder("*").depends_on(
-                    "compiler-wrapper",
-                    when=f"%[virtuals={language}] {compiler.name}@{compiler.versions}",
-                    type="build",
-                    description=f"Add the compiler wrapper when using {compiler} for {language}",
-                )
+            recorder("*").depends_on(
+                "compiler-wrapper",
+                when=f"%{compiler.name}@{compiler.versions}",
+                type="build",
+                description=f"Add the compiler wrapper when using {compiler}",
+            )
 
             if not using_libc_compatibility():
                 continue
@@ -3308,13 +3267,15 @@ def literal_specs(self, specs):
                     # These facts are needed to compute the "condition_set" of the root
                     pkg_name = clause.args[1]
                     self.gen.fact(fn.mentioned_in_literal(trigger_id, root_name, pkg_name))
+                elif clause_name == "depends_on":
+                    pkg_name = clause.args[2]
+                    self.gen.fact(fn.mentioned_in_literal(trigger_id, root_name, pkg_name))
 
             requirements.append(
                 fn.attr(
                     "virtual_root" if spack.repo.PATH.is_virtual(spec.name) else "root", spec.name
                 )
             )
-            requirements = [x for x in requirements if x.args[0] != "depends_on"]
             cache[imposed_spec_key] = (effect_id, requirements)
             self.gen.fact(fn.pkg_fact(spec.name, fn.condition_effect(condition_id, effect_id)))
 
@@ -3639,9 +3600,11 @@ def rule_body_from(self, when_spec: "spack.spec.Spec") -> Tuple[str, str]:
                 # (avoid adding virtuals everywhere, if a single edge needs it)
                 _, provider, virtual = clause.args
                 clause.args = "virtual_on_edge", node_placeholder, provider, virtual
-        body_str = ",\n".join(f"  {x}" for x in body_clauses)
-        body_str += f",\n  not external({node_variable})"
-        body_str = body_str.replace(f'"{node_placeholder}"', f"{node_variable}")
+        body_str = (
+            f"  {f',{os.linesep}  '.join(str(x) for x in body_clauses)},\n"
+            f"  not external({node_variable}),\n"
+            f"  not runtime(Package)"
+        ).replace(f'"{node_placeholder}"', f"{node_variable}")
         for old, replacement in when_substitutions.items():
             body_str = body_str.replace(old, replacement)
         return body_str, node_variable
@@ -3832,13 +3795,13 @@ def node_os(self, node, os):
     def node_target(self, node, target):
         self._arch(node).target = target
 
-    def variant_selected(self, node, name: str, value: str, variant_type: str, variant_id):
+    def variant_selected(self, node, name, value, variant_type, variant_id):
         spec = self._specs[node]
         variant = spec.variants.get(name)
         if not variant:
-            spec.variants[name] = vt.VariantValue.from_concretizer(name, value, variant_type)
+            spec.variants[name] = vt.VariantType(variant_type).variant_class(name, value)
         else:
-            assert variant_type == "multi", (
+            assert variant_type == vt.VariantType.MULTI.value, (
                 f"Can't have multiple values for single-valued variant: "
                 f"{node}, {name}, {value}, {variant_type}, {variant_id}"
             )
@@ -3862,17 +3825,6 @@ def external_spec_selected(self, node, idx):
         )
         self._specs[node].extra_attributes = spec_info.get("extra_attributes", {})
 
-        # Annotate compiler specs from externals
-        external_spec = spack.spec.Spec(spec_info["spec"])
-        external_spec_deps = external_spec.dependencies()
-        if len(external_spec_deps) > 1:
-            raise InvalidExternalError(
-                f"external spec {spec_info['spec']} cannot have more than one dependency"
-            )
-        elif len(external_spec_deps) == 1:
-            compiler_str = external_spec_deps[0]
-            self._specs[node].annotations.with_compiler(spack.spec.Spec(compiler_str))
-
         # If this is an extension, update the dependencies to include the extendee
         package = spack.repo.PATH.get_pkg_class(self._specs[node].fullname)(self._specs[node])
         extendee_spec = package.extendee_spec
@@ -4228,10 +4180,10 @@ def _inject_patches_variant(root: spack.spec.Spec) -> None:
             continue
 
         patches = list(spec_to_patches[id(spec)])
-        variant: vt.VariantValue = spec.variants.setdefault(
+        variant: vt.MultiValuedVariant = spec.variants.setdefault(
             "patches", vt.MultiValuedVariant("patches", ())
         )
-        variant.set(*(p.sha256 for p in patches))
+        variant.value = tuple(p.sha256 for p in patches)
         # FIXME: Monkey patches variant to store patches order
         ordered_hashes = [(*p.ordering_key, p.sha256) for p in patches if p.ordering_key]
         ordered_hashes.sort()
@@ -4699,9 +4651,13 @@ def solve_in_rounds(
                 break
 
             if not result.specs:
-                # This is also a problem: no specs were solved for, which means we would be in a
-                # loop if we tried again
-                raise OutputDoesNotSatisfyInputError(result.unsolved_specs)
+                # This is also a problem: no specs were solved for, which
+                # means we would be in a loop if we tried again
+                unsolved_str = Result.format_unsolved(result.unsolved_specs)
+                raise InternalConcretizerError(
+                    "Internal Spack error: a subset of input specs could not"
+                    f" be solved for.\n\t{unsolved_str}"
+                )
 
             input_specs = list(x for (x, y) in result.unsolved_specs)
             for spec in result.specs:
@@ -4731,19 +4687,6 @@ def __init__(self, msg):
         self.constraint_type = None
 
 
-class OutputDoesNotSatisfyInputError(InternalConcretizerError):
-
-    def __init__(
-        self, input_to_output: List[Tuple[spack.spec.Spec, Optional[spack.spec.Spec]]]
-    ) -> None:
-        self.input_to_output = input_to_output
-        super().__init__(
-            "internal solver error: the solver completed but produced specs"
-            " that do not satisfy the request. Please report a bug at "
-            f"https://github.com/spack/spack/issues\n\t{Result.format_unsolved(input_to_output)}"
-        )
-
-
 class SolverError(InternalConcretizerError):
     """For cases where the solver is unable to produce a solution.
 
@@ -4776,7 +4719,3 @@ class InvalidSpliceError(spack.error.SpackError):
 
 class NoCompilerFoundError(spack.error.SpackError):
     """Raised when there is no possible compiler"""
-
-
-class InvalidExternalError(spack.error.SpackError):
-    """Raised when there is no possible compiler"""

lib/spack/spack/solver/concretize.lp

@@ -175,24 +175,12 @@ trigger_node(TriggerID, Node, Node) :-
 
 % Since we trigger the existence of literal nodes from a condition, we need to construct the condition_set/2
 mentioned_in_literal(Root, Mentioned) :- mentioned_in_literal(TriggerID, Root, Mentioned), solve_literal(TriggerID).
-literal_node(Root, node(min_dupe_id, Root)) :-  mentioned_in_literal(Root, Root).
+condition_set(node(min_dupe_id, Root), node(min_dupe_id, Root)) :-  mentioned_in_literal(Root, Root).
 
-1 { literal_node(Root, node(0..Y-1, Mentioned)) : max_dupes(Mentioned, Y) } 1 :-
+1 { condition_set(node(min_dupe_id, Root), node(0..Y-1, Mentioned)) : max_dupes(Mentioned, Y) } 1 :-
   mentioned_in_literal(Root, Mentioned), Mentioned != Root,
   internal_error("must have exactly one condition_set for literals").
 
-1 { build_dependency_of_literal_node(LiteralNode, node(0..Y-1, BuildDependency)) : max_dupes(BuildDependency, Y) } 1 :-
-  literal_node(Root, LiteralNode),
-  build(LiteralNode),
-  not external(LiteralNode),
-  attr("build_requirement", LiteralNode, build_requirement("node", BuildDependency)).
-
-condition_set(node(min_dupe_id, Root), LiteralNode) :- literal_node(Root, LiteralNode).
-condition_set(LiteralNode, BuildNode) :- build_dependency_of_literal_node(LiteralNode, BuildNode).
-
-:- build_dependency_of_literal_node(LiteralNode, BuildNode),
-   not attr("depends_on", LiteralNode, BuildNode, "build").
-
 % Discriminate between "roots" that have been explicitly requested, and roots that are deduced from "virtual roots"
 explicitly_requested_root(node(min_dupe_id, Package)) :-
   solve_literal(TriggerID),
@@ -484,53 +472,10 @@ provider(ProviderNode, VirtualNode) :- attr("provider_set", ProviderNode, Virtua
      imposed_constraint(ID, "depends_on", A1, A2, A3),
      internal_error("Build deps must land in exactly one duplicate").
 
-% If the parent is built, then we have a build_requirement on another node. For concrete nodes,
-% or external nodes, we don't since we are trimming their build dependencies.
-1 { attr("depends_on",  node(X, Parent), node(0..Y-1, BuildDependency), "build") : max_dupes(BuildDependency, Y) } 1
+1 { build_requirement(node(X, Parent), node(0..Y-1, BuildDependency)) : max_dupes(BuildDependency, Y) } 1
   :- attr("build_requirement", node(X, Parent), build_requirement("node", BuildDependency)),
-     build(node(X, Parent)),
-     not external(node(X, Parent)).
-
-% Concrete nodes
-:- attr("build_requirement", ParentNode, build_requirement("node", BuildDependency)),
-   concrete(ParentNode),
-   not attr("concrete_build_dependency", ParentNode, BuildDependency, _).
-
-:- attr("build_requirement", ParentNode, build_requirement("node_version_satisfies", BuildDependency, Constraint)),
-   attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash),
-   not 1 { pkg_fact(BuildDependency, version_satisfies(Constraint, Version)) : hash_attr(BuildDependencyHash, "version", BuildDependency, Version) } 1.
-
-:- attr("build_requirement", ParentNode, build_requirement("provider_set", BuildDependency, Virtual)),
-   attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash),
-   attr("virtual_on_build_edge", ParentNode, BuildDependency, Virtual),
-   not 1 { pkg_fact(BuildDependency, version_satisfies(Constraint, Version)) : hash_attr(BuildDependencyHash, "version", BuildDependency, Version) } 1.
-
-% External nodes
-:- attr("build_requirement", ParentNode, build_requirement("node", BuildDependency)),
-   external(ParentNode),
-   not attr("external_build_requirement", ParentNode, build_requirement("node", BuildDependency)).
-
-candidate_external_version(Constraint, BuildDependency, Version)
-  :- attr("build_requirement", ParentNode, build_requirement("node_version_satisfies", BuildDependency, Constraint)),
-     external(ParentNode),
-     pkg_fact(BuildDependency, version_satisfies(Constraint, Version)).
-
-error(100, "External {0} cannot satisfy both {1} and {2}", BuildDependency, LiteralConstraint, ExternalConstraint)
-  :- attr("build_requirement", ParentNode, build_requirement("node_version_satisfies", BuildDependency, LiteralConstraint)),
-     external(ParentNode),
-     attr("external_build_requirement", ParentNode, build_requirement("node_version_satisfies", BuildDependency, ExternalConstraint)),
-     not 1 { pkg_fact(BuildDependency, version_satisfies(ExternalConstraint, Version)) : candidate_external_version(LiteralConstraint, BuildDependency, Version) }.
-
-
-% Asking for gcc@10 %gcc@9 shouldn't give us back an external gcc@10, just because of the hack
-% we have on externals
-:- attr("build_requirement", node(X, Parent), build_requirement("node", BuildDependency)),
-   Parent == BuildDependency,
-   external(node(X, Parent)).
-
-build_requirement(node(X, Parent), node(Y, BuildDependency)) :-
-  attr("depends_on", node(X, Parent), node(Y, BuildDependency), "build"),
-  attr("build_requirement", node(X, Parent), build_requirement("node", BuildDependency)).
+     impose(ID, node(X, Parent)),
+     imposed_constraint(ID,"build_requirement",Parent,_).
 
 1 { virtual_build_requirement(ParentNode, node(0..Y-1, Virtual)) : max_dupes(Virtual, Y) } 1
   :- attr("dependency_holds", ParentNode, Virtual, "build"),
@@ -551,6 +496,7 @@ attr("node_version_satisfies", node(X, BuildDependency), Constraint) :-
   attr("build_requirement", ParentNode, build_requirement("node_version_satisfies", BuildDependency, Constraint)),
   build_requirement(ParentNode, node(X, BuildDependency)).
 
+attr("depends_on",  node(X, Parent), node(Y, BuildDependency), "build") :- build_requirement(node(X, Parent), node(Y, BuildDependency)).
 
 1 { attr("provider_set", node(X, BuildDependency), node(0..Y-1, Virtual)) : max_dupes(Virtual, Y) } 1 :-
   attr("build_requirement", ParentNode, build_requirement("provider_set", BuildDependency, Virtual)),
@@ -936,12 +882,6 @@ requirement_weight(node(ID, Package), Group, W) :-
   requirement_policy(Package, Group, "one_of"),
   requirement_group_satisfied(node(ID, Package), Group).
 
- { attr("build_requirement", node(ID, Package), BuildRequirement) : condition_requirement(TriggerID, "build_requirement", Package, BuildRequirement) } :-
-  pkg_fact(Package, condition_trigger(ConditionID, TriggerID)),
-  requirement_group_member(ConditionID, Package, Group),
-  activate_requirement(node(ID, Package), Group),
-  requirement_group(Package, Group).
-
 requirement_group_satisfied(node(ID, Package), X) :-
   1 { condition_holds(Y, node(ID, Package)) : requirement_group_member(Y, Package, X) } ,
   requirement_policy(Package, X, "any_of"),

lib/spack/spack/solver/input_analysis.py

@@ -85,10 +85,8 @@ def is_virtual(self, name: str) -> bool:
     def is_allowed_on_this_platform(self, *, pkg_name: str) -> bool:
         """Returns true if a package is allowed on the current host"""
         pkg_cls = self.repo.get_pkg_class(pkg_name)
-        no_condition = spack.spec.Spec()
         for when_spec, conditions in pkg_cls.requirements.items():
-            # Restrict analysis to unconditional requirements
-            if when_spec != no_condition:
+            if not when_spec.intersects(self._platform_condition):
                 continue
             for requirements, _, _ in conditions:
                 if not any(x.intersects(self._platform_condition) for x in requirements):

lib/spack/spack/spec.py

@@ -111,14 +111,22 @@
 __all__ = [
     "CompilerSpec",
     "Spec",
+    "SpecParseError",
     "UnsupportedPropagationError",
     "DuplicateDependencyError",
+    "DuplicateCompilerSpecError",
     "UnsupportedCompilerError",
     "DuplicateArchitectureError",
+    "InconsistentSpecError",
     "InvalidDependencyError",
+    "NoProviderError",
+    "MultipleProviderError",
     "UnsatisfiableSpecNameError",
     "UnsatisfiableVersionSpecError",
+    "UnsatisfiableCompilerSpecError",
+    "UnsatisfiableCompilerFlagSpecError",
     "UnsatisfiableArchitectureSpecError",
+    "UnsatisfiableProviderSpecError",
     "UnsatisfiableDependencySpecError",
     "AmbiguousHashError",
     "InvalidHashError",
@@ -837,7 +845,7 @@ def _shared_subset_pair_iterate(container1, container2):
                 b_idx += 1
 
 
-class FlagMap(lang.HashableMap[str, List[CompilerFlag]]):
+class FlagMap(lang.HashableMap):
     __slots__ = ("spec",)
 
     def __init__(self, spec):
@@ -1429,7 +1437,7 @@ def with_compiler(self, compiler: "Spec") -> "SpecAnnotations":
     def __repr__(self) -> str:
         result = f"SpecAnnotations().with_spec_format({self.original_spec_format})"
         if self.compiler_node_attribute:
-            result += f".with_compiler({str(self.compiler_node_attribute)})"
+            result += f"with_compiler({str(self.compiler_node_attribute)})"
         return result
 
 
@@ -1698,10 +1706,10 @@ def _dependencies_dict(self, depflag: dt.DepFlag = dt.ALL):
             result[key] = list(group)
         return result
 
-    def _add_flag(
-        self, name: str, value: Union[str, bool], propagate: bool, concrete: bool
-    ) -> None:
-        """Called by the parser to add a known flag"""
+    def _add_flag(self, name, value, propagate):
+        """Called by the parser to add a known flag.
+        Known flags currently include "arch"
+        """
 
         if propagate and name in vt.RESERVED_NAMES:
             raise UnsupportedPropagationError(
@@ -1710,7 +1718,6 @@ def _add_flag(
 
         valid_flags = FlagMap.valid_compiler_flags()
         if name == "arch" or name == "architecture":
-            assert type(value) is str, "architecture have a string value"
             parts = tuple(value.split("-"))
             plat, os, tgt = parts if len(parts) == 3 else (None, None, value)
             self._set_architecture(platform=plat, os=os, target=tgt)
@@ -1724,15 +1731,19 @@ def _add_flag(
             self.namespace = value
         elif name in valid_flags:
             assert self.compiler_flags is not None
-            assert type(value) is str, f"{name} must have a string value"
             flags_and_propagation = spack.compilers.flags.tokenize_flags(value, propagate)
             flag_group = " ".join(x for (x, y) in flags_and_propagation)
             for flag, propagation in flags_and_propagation:
                 self.compiler_flags.add_flag(name, flag, propagation, flag_group)
         else:
-            self.variants[name] = vt.VariantValue.from_string_or_bool(
-                name, value, propagate=propagate, concrete=concrete
-            )
+            # FIXME:
+            # All other flags represent variants. 'foo=true' and 'foo=false'
+            # map to '+foo' and '~foo' respectively. As such they need a
+            # BoolValuedVariant instance.
+            if str(value).upper() == "TRUE" or str(value).upper() == "FALSE":
+                self.variants[name] = vt.BoolValuedVariant(name, value, propagate)
+            else:
+                self.variants[name] = vt.AbstractVariant(name, value, propagate)
 
     def _set_architecture(self, **kwargs):
         """Called by the parser to set the architecture."""
@@ -1861,7 +1872,9 @@ def add_dependency_edge(
     @property
     def fullname(self):
         return (
-            f"{self.namespace}.{self.name}" if self.namespace else (self.name if self.name else "")
+            ("%s.%s" % (self.namespace, self.name))
+            if self.namespace
+            else (self.name if self.name else "")
         )
 
     @property
@@ -2338,7 +2351,6 @@ def to_node_dict(self, hash=ht.dag_hash):
                     [v.name for v in self.variants.values() if v.propagate], flag_names
                 )
             )
-            d["abstract"] = sorted(v.name for v in self.variants.values() if not v.concrete)
 
         if self.external:
             d["external"] = {
@@ -3065,7 +3077,7 @@ def constrain(self, other, deps=True):
             raise UnsatisfiableVersionSpecError(self.versions, other.versions)
 
         for v in [x for x in other.variants if x in self.variants]:
-            if not self.variants[v].intersects(other.variants[v]):
+            if not self.variants[v].compatible(other.variants[v]):
                 raise vt.UnsatisfiableVariantSpecError(self.variants[v], other.variants[v])
 
         sarch, oarch = self.architecture, other.architecture
@@ -3392,7 +3404,7 @@ def satisfies(self, other: Union[str, "Spec"], deps: bool = True) -> bool:
             return True
 
         # If we have no dependencies, we can't satisfy any constraints.
-        if not self._dependencies and self.original_spec_format() >= 5 and not self.external:
+        if not self._dependencies:
             return False
 
         # If we arrived here, the lhs root node satisfies the rhs root node. Now we need to check
@@ -3403,7 +3415,6 @@ def satisfies(self, other: Union[str, "Spec"], deps: bool = True) -> bool:
         # verify the edge properties, cause everything is encoded in the hash of the nodes that
         # will be verified later.
         lhs_edges: Dict[str, Set[DependencySpec]] = collections.defaultdict(set)
-        mock_nodes_from_old_specfiles = set()
         for rhs_edge in other.traverse_edges(root=False, cover="edges"):
             # If we are checking for ^mpi we need to verify if there is any edge
             if spack.repo.PATH.is_virtual(rhs_edge.spec.name):
@@ -3425,27 +3436,13 @@ def satisfies(self, other: Union[str, "Spec"], deps: bool = True) -> bool:
                     except KeyError:
                         return False
 
-                if current_node.original_spec_format() < 5 or (
-                    current_node.original_spec_format() >= 5 and current_node.external
-                ):
-                    compiler_spec = current_node.annotations.compiler_node_attribute
-                    if compiler_spec is None:
-                        return False
-
-                    mock_nodes_from_old_specfiles.add(compiler_spec)
-                    # This checks that the single node compiler spec satisfies the request
-                    # of a direct dependency. The check is not perfect, but based on heuristic.
-                    if not compiler_spec.satisfies(rhs_edge.spec):
-                        return False
-
-                else:
-                    candidates = current_node.dependencies(
-                        name=rhs_edge.spec.name,
-                        deptype=rhs_edge.depflag,
-                        virtuals=rhs_edge.virtuals or None,
-                    )
-                    if not candidates or not any(x.satisfies(rhs_edge.spec) for x in candidates):
-                        return False
+                candidates = current_node.dependencies(
+                    name=rhs_edge.spec.name,
+                    deptype=rhs_edge.depflag,
+                    virtuals=rhs_edge.virtuals or None,
+                )
+                if not candidates or not any(x.satisfies(rhs_edge.spec) for x in candidates):
+                    return False
 
                 continue
 
@@ -3485,9 +3482,8 @@ def satisfies(self, other: Union[str, "Spec"], deps: bool = True) -> bool:
                     return False
 
         # Edges have been checked above already, hence deps=False
-        lhs_nodes = [x for x in self.traverse(root=False)] + sorted(mock_nodes_from_old_specfiles)
         return all(
-            any(lhs.satisfies(rhs, deps=False) for lhs in lhs_nodes)
+            any(lhs.satisfies(rhs, deps=False) for lhs in self.traverse(root=False))
             for rhs in other.traverse(root=False)
         )
 
@@ -3961,8 +3957,6 @@ def format_attribute(match_object: Match) -> str:
                     except AttributeError:
                         if part == "compiler":
                             return "none"
-                        elif part == "specfile_version":
-                            return f"v{current.original_spec_format()}"
 
                         raise SpecFormatStringError(
                             f"Attempted to format attribute {attribute}. "
@@ -4488,7 +4482,7 @@ def has_virtual_dependency(self, virtual: str) -> bool:
         return bool(self.dependencies(virtuals=(virtual,)))
 
 
-class VariantMap(lang.HashableMap[str, vt.VariantValue]):
+class VariantMap(lang.HashableMap):
     """Map containing variant instances. New values can be added only
     if the key is not already present."""
 
@@ -4498,7 +4492,7 @@ def __init__(self, spec: Spec):
 
     def __setitem__(self, name, vspec):
         # Raise a TypeError if vspec is not of the right type
-        if not isinstance(vspec, vt.VariantValue):
+        if not isinstance(vspec, vt.AbstractVariant):
             raise TypeError(
                 "VariantMap accepts only values of variant types "
                 f"[got {type(vspec).__name__} instead]"
@@ -4608,7 +4602,8 @@ def constrain(self, other: "VariantMap") -> bool:
         changed = False
         for k in other:
             if k in self:
-                if not self[k].intersects(other[k]):
+                # If they are not compatible raise an error
+                if not self[k].compatible(other[k]):
                     raise vt.UnsatisfiableVariantSpecError(self[k], other[k])
                 # If they are compatible merge them
                 changed |= self[k].constrain(other[k])
@@ -4638,7 +4633,7 @@ def __str__(self):
         bool_keys = []
         kv_keys = []
         for key in sorted_keys:
-            if self[key].type == vt.VariantType.BOOL:
+            if isinstance(self[key].value, bool):
                 bool_keys.append(key)
             else:
                 kv_keys.append(key)
@@ -4671,8 +4666,7 @@ def substitute_abstract_variants(spec: Spec):
     unknown = []
     for name, v in spec.variants.items():
         if name == "dev_path":
-            v.type = vt.VariantType.SINGLE
-            v.concrete = True
+            spec.variants.substitute(vt.SingleValuedVariant(name, v._original_value))
             continue
         elif name in vt.RESERVED_NAMES:
             continue
@@ -4695,7 +4689,7 @@ def substitute_abstract_variants(spec: Spec):
         if rest:
             continue
 
-        new_variant = pkg_variant.make_variant(*v.values)
+        new_variant = pkg_variant.make_variant(v._original_value)
         pkg_variant.validate_or_raise(new_variant, spec.name)
         spec.variants.substitute(new_variant)
 
@@ -4813,7 +4807,6 @@ def from_node_dict(cls, node):
             spec.architecture = ArchSpec.from_dict(node)
 
         propagated_names = node.get("propagate", [])
-        abstract_variants = set(node.get("abstract", ()))
         for name, values in node.get("parameters", {}).items():
             propagate = name in propagated_names
             if name in _valid_compiler_flags:
@@ -4821,8 +4814,8 @@ def from_node_dict(cls, node):
                 for val in values:
                     spec.compiler_flags.add_flag(name, val, propagate)
             else:
-                spec.variants[name] = vt.VariantValue.from_node_dict(
-                    name, values, propagate=propagate, abstract=name in abstract_variants
+                spec.variants[name] = vt.MultiValuedVariant.from_node_dict(
+                    name, values, propagate=propagate
                 )
 
         spec.external_path = None
@@ -4847,7 +4840,7 @@ def from_node_dict(cls, node):
             patches = node["patches"]
             if len(patches) > 0:
                 mvar = spec.variants.setdefault("patches", vt.MultiValuedVariant("patches", ()))
-                mvar.set(*patches)
+                mvar.value = patches
                 # FIXME: Monkey patches mvar to store patches order
                 mvar._patches_in_order_of_appearance = patches
 
@@ -5172,6 +5165,25 @@ def eval_conditional(string):
     return eval(string, valid_variables)
 
 
+class SpecParseError(spack.error.SpecError):
+    """Wrapper for ParseError for when we're parsing specs."""
+
+    def __init__(self, parse_error):
+        super().__init__(parse_error.message)
+        self.string = parse_error.string
+        self.pos = parse_error.pos
+
+    @property
+    def long_message(self):
+        return "\n".join(
+            [
+                "  Encountered when parsing spec:",
+                "    %s" % self.string,
+                "    %s^" % (" " * self.pos),
+            ]
+        )
+
+
 class InvalidVariantForSpecError(spack.error.SpecError):
     """Raised when an invalid conditional variant is specified."""
 
@@ -5189,6 +5201,14 @@ class DuplicateDependencyError(spack.error.SpecError):
     """Raised when the same dependency occurs in a spec twice."""
 
 
+class MultipleVersionError(spack.error.SpecError):
+    """Raised when version constraints occur in a spec twice."""
+
+
+class DuplicateCompilerSpecError(spack.error.SpecError):
+    """Raised when the same compiler occurs in a spec twice."""
+
+
 class UnsupportedCompilerError(spack.error.SpecError):
     """Raised when the user asks for a compiler spack doesn't know about."""
 
@@ -5197,6 +5217,11 @@ class DuplicateArchitectureError(spack.error.SpecError):
     """Raised when the same architecture occurs in a spec twice."""
 
 
+class InconsistentSpecError(spack.error.SpecError):
+    """Raised when two nodes in the same spec DAG have inconsistent
+    constraints."""
+
+
 class InvalidDependencyError(spack.error.SpecError):
     """Raised when a dependency in a spec is not actually a dependency
     of the package."""
@@ -5208,6 +5233,30 @@ def __init__(self, pkg, deps):
         )
 
 
+class NoProviderError(spack.error.SpecError):
+    """Raised when there is no package that provides a particular
+    virtual dependency.
+    """
+
+    def __init__(self, vpkg):
+        super().__init__("No providers found for virtual package: '%s'" % vpkg)
+        self.vpkg = vpkg
+
+
+class MultipleProviderError(spack.error.SpecError):
+    """Raised when there is no package that provides a particular
+    virtual dependency.
+    """
+
+    def __init__(self, vpkg, providers):
+        """Takes the name of the vpkg"""
+        super().__init__(
+            "Multiple providers found for '%s': %s" % (vpkg, [str(s) for s in providers])
+        )
+        self.vpkg = vpkg
+        self.providers = providers
+
+
 class UnsatisfiableSpecNameError(spack.error.UnsatisfiableSpecError):
     """Raised when two specs aren't even for the same package."""
 
@@ -5222,6 +5271,20 @@ def __init__(self, provided, required):
         super().__init__(provided, required, "version")
 
 
+class UnsatisfiableCompilerSpecError(spack.error.UnsatisfiableSpecError):
+    """Raised when a spec compiler conflicts with package constraints."""
+
+    def __init__(self, provided, required):
+        super().__init__(provided, required, "compiler")
+
+
+class UnsatisfiableCompilerFlagSpecError(spack.error.UnsatisfiableSpecError):
+    """Raised when a spec variant conflicts with package constraints."""
+
+    def __init__(self, provided, required):
+        super().__init__(provided, required, "compiler_flags")
+
+
 class UnsatisfiableArchitectureSpecError(spack.error.UnsatisfiableSpecError):
     """Raised when a spec architecture conflicts with package constraints."""
 
@@ -5229,6 +5292,14 @@ def __init__(self, provided, required):
         super().__init__(provided, required, "architecture")
 
 
+class UnsatisfiableProviderSpecError(spack.error.UnsatisfiableSpecError):
+    """Raised when a provider is supplied but constraints don't match
+    a vpkg requirement"""
+
+    def __init__(self, provided, required):
+        super().__init__(provided, required, "provider")
+
+
 # TODO: get rid of this and be more specific about particular incompatible
 # dep constraints
 class UnsatisfiableDependencySpecError(spack.error.UnsatisfiableSpecError):

lib/spack/spack/spec_list.py

@@ -0,0 +1,263 @@
+# Copyright Spack Project Developers. See COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import itertools
+from typing import List
+
+import spack.spec
+import spack.variant
+from spack.error import SpackError
+from spack.spec import Spec
+
+
+class SpecList:
+    def __init__(self, name="specs", yaml_list=None, reference=None):
+        # Normalize input arguments
+        yaml_list = yaml_list or []
+        reference = reference or {}
+
+        self.name = name
+        self._reference = reference  # TODO: Do we need defensive copy here?
+
+        # Validate yaml_list before assigning
+        if not all(isinstance(s, str) or isinstance(s, (list, dict)) for s in yaml_list):
+            raise ValueError(
+                "yaml_list can contain only valid YAML types!  Found:\n  %s"
+                % [type(s) for s in yaml_list]
+            )
+        self.yaml_list = yaml_list[:]
+
+        # Expansions can be expensive to compute and difficult to keep updated
+        # We cache results and invalidate when self.yaml_list changes
+        self._expanded_list = None
+        self._constraints = None
+        self._specs = None
+
+    @property
+    def is_matrix(self):
+        for item in self.specs_as_yaml_list:
+            if isinstance(item, dict):
+                return True
+        return False
+
+    @property
+    def specs_as_yaml_list(self):
+        if self._expanded_list is None:
+            self._expanded_list = self._expand_references(self.yaml_list)
+        return self._expanded_list
+
+    @property
+    def specs_as_constraints(self):
+        if self._constraints is None:
+            constraints = []
+            for item in self.specs_as_yaml_list:
+                if isinstance(item, dict):  # matrix of specs
+                    constraints.extend(_expand_matrix_constraints(item))
+                else:  # individual spec
+                    constraints.append([Spec(item)])
+            self._constraints = constraints
+
+        return self._constraints
+
+    @property
+    def specs(self) -> List[Spec]:
+        if self._specs is None:
+            specs = []
+            # This could be slightly faster done directly from yaml_list,
+            # but this way is easier to maintain.
+            for constraint_list in self.specs_as_constraints:
+                spec = constraint_list[0].copy()
+                for const in constraint_list[1:]:
+                    spec.constrain(const)
+                specs.append(spec)
+            self._specs = specs
+
+        return self._specs
+
+    def add(self, spec):
+        self.yaml_list.append(str(spec))
+
+        # expanded list can be updated without invalidation
+        if self._expanded_list is not None:
+            self._expanded_list.append(str(spec))
+
+        # Invalidate cache variables when we change the list
+        self._constraints = None
+        self._specs = None
+
+    def remove(self, spec):
+        # Get spec to remove from list
+        remove = [
+            s
+            for s in self.yaml_list
+            if (isinstance(s, str) and not s.startswith("$")) and Spec(s) == Spec(spec)
+        ]
+        if not remove:
+            msg = f"Cannot remove {spec} from SpecList {self.name}.\n"
+            msg += f"Either {spec} is not in {self.name} or {spec} is "
+            msg += "expanded from a matrix and cannot be removed directly."
+            raise SpecListError(msg)
+
+        # Remove may contain more than one string representation of the same spec
+        for item in remove:
+            self.yaml_list.remove(item)
+
+        # invalidate cache variables when we change the list
+        self._expanded_list = None
+        self._constraints = None
+        self._specs = None
+
+    def replace(self, idx: int, spec: str):
+        """Replace the existing spec at the index with the new one.
+
+        Args:
+            idx: index of the spec to replace in the speclist
+            spec: new spec
+        """
+        self.yaml_list[idx] = spec
+
+        # invalidate cache variables when we change the list
+        self._expanded_list = None
+        self._constraints = None
+        self._specs = None
+
+    def extend(self, other, copy_reference=True):
+        self.yaml_list.extend(other.yaml_list)
+        self._expanded_list = None
+        self._constraints = None
+        self._specs = None
+
+        if copy_reference:
+            self._reference = other._reference
+
+    def update_reference(self, reference):
+        self._reference = reference
+        self._expanded_list = None
+        self._constraints = None
+        self._specs = None
+
+    def _parse_reference(self, name):
+        sigil = ""
+        name = name[1:]
+
+        # Parse specs as constraints
+        if name.startswith("^") or name.startswith("%"):
+            sigil = name[0]
+            name = name[1:]
+
+        # Make sure the reference is valid
+        if name not in self._reference:
+            msg = f"SpecList '{self.name}' refers to named list '{name}'"
+            msg += " which does not appear in its reference dict."
+            raise UndefinedReferenceError(msg)
+
+        return (name, sigil)
+
+    def _expand_references(self, yaml):
+        if isinstance(yaml, list):
+            ret = []
+
+            for item in yaml:
+                # if it's a reference, expand it
+                if isinstance(item, str) and item.startswith("$"):
+                    # replace the reference and apply the sigil if needed
+                    name, sigil = self._parse_reference(item)
+
+                    referent = [
+                        _sigilify(item, sigil) for item in self._reference[name].specs_as_yaml_list
+                    ]
+                    ret.extend(referent)
+                else:
+                    # else just recurse
+                    ret.append(self._expand_references(item))
+            return ret
+        elif isinstance(yaml, dict):
+            # There can't be expansions in dicts
+            return dict((name, self._expand_references(val)) for (name, val) in yaml.items())
+        else:
+            # Strings are just returned
+            return yaml
+
+    def __len__(self):
+        return len(self.specs)
+
+    def __getitem__(self, key):
+        return self.specs[key]
+
+    def __iter__(self):
+        return iter(self.specs)
+
+
+def _expand_matrix_constraints(matrix_config):
+    # recurse so we can handle nested matrices
+    expanded_rows = []
+    for row in matrix_config["matrix"]:
+        new_row = []
+        for r in row:
+            if isinstance(r, dict):
+                # Flatten the nested matrix into a single row of constraints
+                new_row.extend(
+                    [
+                        [" ".join([str(c) for c in expanded_constraint_list])]
+                        for expanded_constraint_list in _expand_matrix_constraints(r)
+                    ]
+                )
+            else:
+                new_row.append([r])
+        expanded_rows.append(new_row)
+
+    excludes = matrix_config.get("exclude", [])  # only compute once
+    sigil = matrix_config.get("sigil", "")
+
+    results = []
+    for combo in itertools.product(*expanded_rows):
+        # Construct a combined spec to test against excludes
+        flat_combo = [Spec(constraint) for constraints in combo for constraint in constraints]
+
+        test_spec = flat_combo[0].copy()
+        for constraint in flat_combo[1:]:
+            test_spec.constrain(constraint)
+
+        # Abstract variants don't have normal satisfaction semantics
+        # Convert all variants to concrete types.
+        # This method is best effort, so all existing variants will be
+        # converted before any error is raised.
+        # Catch exceptions because we want to be able to operate on
+        # abstract specs without needing package information
+        try:
+            spack.spec.substitute_abstract_variants(test_spec)
+        except spack.variant.UnknownVariantError:
+            pass
+
+        # Resolve abstract hashes for exclusion criteria
+        if any(test_spec.lookup_hash().satisfies(x) for x in excludes):
+            continue
+
+        if sigil:
+            flat_combo[0] = Spec(sigil + str(flat_combo[0]))
+
+        # Add to list of constraints
+        results.append(flat_combo)
+
+    return results
+
+
+def _sigilify(item, sigil):
+    if isinstance(item, dict):
+        if sigil:
+            item["sigil"] = sigil
+        return item
+    else:
+        return sigil + item
+
+
+class SpecListError(SpackError):
+    """Error class for all errors related to SpecList objects."""
+
+
+class UndefinedReferenceError(SpecListError):
+    """Error class for undefined references in Spack stacks."""
+
+
+class InvalidSpecConstraintError(SpecListError):
+    """Error class for invalid spec constraints at concretize time."""