autotools.py: order props / methods / callbacks

.github/workflows/build-containers.yml

@@ -40,17 +40,17 @@ jobs:
         # 1: Platforms to build for
         # 2: Base image (e.g. ubuntu:22.04)
         dockerfile: [[amazon-linux, 'linux/amd64,linux/arm64', 'amazonlinux:2'],
-                     [centos-stream9, 'linux/amd64,linux/arm64', 'centos:stream9'],
-                     [leap15, 'linux/amd64,linux/arm64', 'opensuse/leap:15'],
-                     [ubuntu-focal, 'linux/amd64,linux/arm64', 'ubuntu:20.04'],
-                     [ubuntu-jammy, 'linux/amd64,linux/arm64', 'ubuntu:22.04'],
-                     [ubuntu-noble, 'linux/amd64,linux/arm64', 'ubuntu:24.04'],
-                     [almalinux8, 'linux/amd64,linux/arm64', 'almalinux:8'],
-                     [almalinux9, 'linux/amd64,linux/arm64', 'almalinux:9'],
+                     [centos-stream9, 'linux/amd64,linux/arm64,linux/ppc64le', 'centos:stream9'],
+                     [leap15, 'linux/amd64,linux/arm64,linux/ppc64le', 'opensuse/leap:15'],
+                     [ubuntu-focal, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:20.04'],
+                     [ubuntu-jammy, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:22.04'],
+                     [ubuntu-noble, 'linux/amd64,linux/arm64,linux/ppc64le', 'ubuntu:24.04'],
+                     [almalinux8, 'linux/amd64,linux/arm64,linux/ppc64le', 'almalinux:8'],
+                     [almalinux9, 'linux/amd64,linux/arm64,linux/ppc64le', 'almalinux:9'],
                      [rockylinux8, 'linux/amd64,linux/arm64', 'rockylinux:8'],
                      [rockylinux9, 'linux/amd64,linux/arm64', 'rockylinux:9'],
-                     [fedora39, 'linux/amd64,linux/arm64', 'fedora:39'],
-                     [fedora40, 'linux/amd64,linux/arm64', 'fedora:40']]
+                     [fedora39, 'linux/amd64,linux/arm64,linux/ppc64le', 'fedora:39'],
+                     [fedora40, 'linux/amd64,linux/arm64,linux/ppc64le', 'fedora:40']]
     name: Build ${{ matrix.dockerfile[0] }}
     if: github.repository == 'spack/spack'
     steps:

.github/workflows/ci.yaml

@@ -81,10 +81,6 @@ jobs:
     with:
       with_coverage: ${{ needs.changes.outputs.core }}
 
-  import-check:
-    needs: [ changes ]
-    uses: ./.github/workflows/import-check.yaml
-
   all-prechecks:
     needs: [ prechecks ]
     if: ${{ always() }}

.github/workflows/coverage.yml

@@ -33,4 +33,3 @@ jobs:
       with:
         verbose: true
         fail_ci_if_error: false
-        token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/import-check.yaml

@@ -1,49 +0,0 @@
-name: import-check
-
-on:
-  workflow_call:
-
-jobs:
-  # Check we don't make the situation with circular imports worse
-  import-check:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: julia-actions/setup-julia@v2
-      with:
-        version: '1.10'
-    - uses: julia-actions/cache@v2
-
-    # PR: use the base of the PR as the old commit
-    - name: Checkout PR base commit
-      if: github.event_name == 'pull_request'
-      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      with:
-        ref: ${{ github.event.pull_request.base.sha }}
-        path: old
-    # not a PR: use the previous commit as the old commit
-    - name: Checkout previous commit
-      if: github.event_name != 'pull_request'
-      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      with:
-        fetch-depth: 2
-        path: old
-    - name: Checkout previous commit
-      if: github.event_name != 'pull_request'
-      run: git -C old reset --hard HEAD^
-
-    - name: Checkout new commit
-      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      with:
-        path: new
-    - name: Install circular import checker
-      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      with:
-        repository: haampie/circular-import-fighter
-        ref: e38bcd0aa46368e30648b61b7f0d8c1ca68aadff
-        path: circular-import-fighter
-    - name: Install dependencies
-      working-directory: circular-import-fighter
-      run: make -j dependencies
-    - name: Circular import check
-      working-directory: circular-import-fighter
-      run: make -j compare "SPACK_ROOT=../old ../new"

.github/workflows/valid-style.yml

@@ -86,6 +86,66 @@ jobs:
           spack -d bootstrap now --dev
           spack -d style -t black
           spack unit-test -V
+  # Check we don't make the situation with circular imports worse
+  import-check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: julia-actions/setup-julia@v2
+      with:
+        version: '1.10'
+    - uses: julia-actions/cache@v2
+
+    # PR: use the base of the PR as the old commit
+    - name: Checkout PR base commit
+      if: github.event_name == 'pull_request'
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      with:
+        ref: ${{ github.event.pull_request.base.sha }}
+        path: old
+    # not a PR: use the previous commit as the old commit
+    - name: Checkout previous commit
+      if: github.event_name != 'pull_request'
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      with:
+        fetch-depth: 2
+        path: old
+    - name: Checkout previous commit
+      if: github.event_name != 'pull_request'
+      run: git -C old reset --hard HEAD^
+
+    - name: Checkout new commit
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      with:
+        path: new
+    - name: Install circular import checker
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      with:
+        repository: haampie/circular-import-fighter
+        ref: b5d6ce9be35f602cca7d5a6aa0259fca10639cca
+        path: circular-import-fighter
+    - name: Install dependencies
+      working-directory: circular-import-fighter
+      run: make -j dependencies
+    - name: Problematic imports before
+      working-directory: circular-import-fighter
+      run: make SPACK_ROOT=../old SUFFIX=.old
+    - name: Problematic imports after
+      working-directory: circular-import-fighter
+      run: make SPACK_ROOT=../new SUFFIX=.new
+    - name: Compare import cycles
+      working-directory: circular-import-fighter
+      run: |
+        edges_before="$(head -n1 solution.old)"
+        edges_after="$(head -n1 solution.new)"
+        if [ "$edges_after" -gt "$edges_before" ]; then
+          printf '\033[1;31mImport check failed: %s imports need to be deleted, ' "$edges_after"
+          printf 'previously this was %s\033[0m\n'  "$edges_before"
+          printf 'Compare \033[1;97m"Problematic imports before"\033[0m and '
+          printf '\033[1;97m"Problematic imports after"\033[0m.\n'
+          exit 1
+        else
+          printf '\033[1;32mImport check passed: %s <= %s\033[0m\n' "$edges_after" "$edges_before"
+        fi
 
   # Further style checks from pylint
   pylint:

bin/spack

@@ -25,6 +25,7 @@ exit 1
 # The code above runs this file with our preferred python interpreter.
 
 import os
+import os.path
 import sys
 
 min_python3 = (3, 6)

etc/spack/defaults/packages.yaml

@@ -36,7 +36,7 @@ packages:
       go-or-gccgo-bootstrap: [go-bootstrap, gcc]
       iconv: [libiconv]
       ipp: [intel-oneapi-ipp]
-      java: [openjdk, jdk]
+      java: [openjdk, jdk, ibm-java]
       jpeg: [libjpeg-turbo, libjpeg]
       lapack: [openblas, amdlibflame]
       libc: [glibc, musl]
@@ -73,27 +73,15 @@ packages:
     permissions:
       read: world
       write: user
-  cray-fftw:
-    buildable: false
-  cray-libsci:
-    buildable: false
   cray-mpich:
     buildable: false
   cray-mvapich2:
     buildable: false
-  cray-pmi:
-    buildable: false
   egl:
     buildable: false
-  essl:
-    buildable: false
   fujitsu-mpi:
     buildable: false
-  fujitsu-ssl2:
-    buildable: false
   hpcx-mpi:
     buildable: false
-  mpt:
-    buildable: false
   spectrum-mpi:
     buildable: false

lib/spack/docs/bootstrapping.rst

@@ -170,7 +170,7 @@ bootstrapping.
    To register the mirror on the platform where it's supposed to be used run the following command(s):
      % spack bootstrap add --trust local-sources /opt/bootstrap/metadata/sources
      % spack bootstrap add --trust local-binaries /opt/bootstrap/metadata/binaries
-     % spack buildcache update-index /opt/bootstrap/bootstrap_cache
+
 
 This command needs to be run on a machine with internet access and the resulting folder
 has to be moved over to the air-gapped system. Once the local sources are added using the

lib/spack/docs/build_systems/autotoolspackage.rst

@@ -272,9 +272,9 @@ often lists dependencies and the flags needed to locate them. The
 "environment variables" section lists environment variables that the
 build system uses to pass flags to the compiler and linker.
 
-^^^^^^^^^^^^^^^^^^^^^^^^^
-Adding flags to configure
-^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+Addings flags to configure
+^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 For most of the flags you encounter, you will want a variant to
 optionally enable/disable them. You can then optionally pass these
@@ -285,7 +285,7 @@ function like so:
 
    def configure_args(self):
        args = []
-       ...
+
        if self.spec.satisfies("+mpi"):
            args.append("--enable-mpi")
        else:
@@ -299,10 +299,7 @@ Alternatively, you can use the :ref:`enable_or_disable  <autotools_enable_or_dis
 .. code-block:: python
 
    def configure_args(self):
-       args = []
-       ...
-       args.extend(self.enable_or_disable("mpi"))
-       return args
+       return [self.enable_or_disable("mpi")]
 
 
 Note that we are explicitly disabling MPI support if it is not
@@ -347,14 +344,7 @@ typically used to enable or disable some feature within the package.
        default=False,
        description="Memchecker support for debugging [degrades performance]"
    )
-   ...
-
-   def configure_args(self):
-       args = []
-       ...
-       args.extend(self.enable_or_disable("memchecker"))
-
-       return args
+   config_args.extend(self.enable_or_disable("memchecker"))
 
 In this example, specifying the variant ``+memchecker`` will generate
 the following configuration options:

lib/spack/docs/contribution_guide.rst

@@ -361,6 +361,7 @@ and the tags associated with the class of runners to build on.
     * ``.linux_neoverse_n1``
     * ``.linux_neoverse_v1``
     * ``.linux_neoverse_v2``
+    * ``.linux_power``
     * ``.linux_skylake``
     * ``.linux_x86_64``
     * ``.linux_x86_64_v4``

lib/spack/docs/environments.rst

@@ -112,19 +112,6 @@ the original but may concretize differently in the presence of different
 explicit or default configuration settings (e.g., a different version of
 Spack or for a different user account).
 
-Environments created from a manifest will copy any included configs
-from relative paths inside the environment. Relative paths from
-outside the environment will cause errors, and absolute paths will be
-kept absolute. For example, if ``spack.yaml`` includes:
-
-.. code-block:: yaml
-
-   spack:
-     include: [./config.yaml]
-
-then the created environment will have its own copy of the file
-``config.yaml`` copied from the location in the original environment.
-
 Create an environment from a ``spack.lock`` file using:
 
 .. code-block:: console
@@ -173,7 +160,7 @@ accepts.  If an environment already exists then spack will simply activate it
 and ignore the create-specific flags.
 
 .. code-block:: console
-
+   
    $ spack env activate --create -p myenv
    # ...
    # [creates if myenv does not exist yet]
@@ -437,8 +424,8 @@ Developing Packages in a Spack Environment
 
 The ``spack develop`` command allows one to develop Spack packages in
 an environment. It requires a spec containing a concrete version, and
-will configure Spack to install the package from local source.
-If a version is not provided from the command line interface then spack
+will configure Spack to install the package from local source. 
+If a version is not provided from the command line interface then spack 
 will automatically pick the highest version the package has defined.
 This means any infinity versions (``develop``, ``main``, ``stable``) will be
 preferred in this selection process.
@@ -448,9 +435,9 @@ set, and Spack will ensure the package and its dependents are rebuilt
 any time the environment is installed if the package's local source
 code has been modified. Spack's native implementation to check for modifications
 is to check if ``mtime`` is newer than the installation.
-A custom check can be created by overriding the ``detect_dev_src_change`` method
-in your package class. This is particularly useful for projects using custom spack repo's
-to drive development and want to optimize performance.
+A custom check can be created by overriding the ``detect_dev_src_change`` method 
+in your package class. This is particularly useful for projects using custom spack repo's 
+to drive development and want to optimize performance. 
 
 Spack ensures that all instances of a
 developed package in the environment are concretized to match the
@@ -466,7 +453,7 @@ Further development on ``foo`` can be tested by re-installing the environment,
 and eventually committed and pushed to the upstream git repo.
 
 If the package being developed supports out-of-source builds then users can use the
-``--build_directory`` flag to control the location and name of the build directory.
+``--build_directory`` flag to control the location and name of the build directory. 
 This is a shortcut to set the ``package_attributes:build_directory`` in the
 ``packages`` configuration (see :ref:`assigning-package-attributes`).
 The supplied location will become the build-directory for that package in all future builds.

lib/spack/llnl/url.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """URL primitives that just require Python standard library."""
 import itertools
-import os
+import os.path
 import re
 from typing import Optional, Set, Tuple
 from urllib.parse import urlsplit, urlunsplit

lib/spack/llnl/util/filesystem.py

@@ -668,7 +668,7 @@ def copy(src, dest, _permissions=False):
         _permissions (bool): for internal use only
 
     Raises:
-        OSError: if *src* does not match any files or directories
+        IOError: if *src* does not match any files or directories
         ValueError: if *src* matches multiple files but *dest* is
             not a directory
     """
@@ -679,7 +679,7 @@ def copy(src, dest, _permissions=False):
 
     files = glob.glob(src)
     if not files:
-        raise OSError("No such file or directory: '{0}'".format(src))
+        raise IOError("No such file or directory: '{0}'".format(src))
     if len(files) > 1 and not os.path.isdir(dest):
         raise ValueError(
             "'{0}' matches multiple files but '{1}' is not a directory".format(src, dest)
@@ -710,7 +710,7 @@ def install(src, dest):
         dest (str): the destination file or directory
 
     Raises:
-        OSError: if *src* does not match any files or directories
+        IOError: if *src* does not match any files or directories
         ValueError: if *src* matches multiple files but *dest* is
             not a directory
     """
@@ -748,7 +748,7 @@ def copy_tree(
         _permissions (bool): for internal use only
 
     Raises:
-        OSError: if *src* does not match any files or directories
+        IOError: if *src* does not match any files or directories
         ValueError: if *src* is a parent directory of *dest*
     """
     if _permissions:
@@ -762,7 +762,7 @@ def copy_tree(
 
     files = glob.glob(src)
     if not files:
-        raise OSError("No such file or directory: '{0}'".format(src))
+        raise IOError("No such file or directory: '{0}'".format(src))
 
     # For Windows hard-links and junctions, the source path must exist to make a symlink. Add
     # all symlinks to this list while traversing the tree, then when finished, make all
@@ -843,7 +843,7 @@ def install_tree(src, dest, symlinks=True, ignore=None):
         ignore (typing.Callable): function indicating which files to ignore
 
     Raises:
-        OSError: if *src* does not match any files or directories
+        IOError: if *src* does not match any files or directories
         ValueError: if *src* is a parent directory of *dest*
     """
     copy_tree(src, dest, symlinks=symlinks, ignore=ignore, _permissions=True)
@@ -1472,7 +1472,7 @@ def set_executable(path):
 def recursive_mtime_greater_than(path: str, time: float) -> bool:
     """Returns true if any file or dir recursively under `path` has mtime greater than `time`."""
     # use bfs order to increase likelihood of early return
-    queue: Deque[str] = collections.deque([path])
+    queue: Deque[str] = collections.deque()
 
     if os.stat(path).st_mtime > time:
         return True

lib/spack/llnl/util/link_tree.py

@@ -308,7 +308,7 @@ class LinkTree:
 
     def __init__(self, source_root):
         if not os.path.exists(source_root):
-            raise OSError("No such file or directory: '%s'", source_root)
+            raise IOError("No such file or directory: '%s'", source_root)
 
         self._root = source_root
 

lib/spack/llnl/util/lock.py

@@ -391,7 +391,7 @@ def _poll_lock(self, op: int) -> bool:
 
             return True
 
-        except OSError as e:
+        except IOError as e:
             # EAGAIN and EACCES == locked by another process (so try again)
             if e.errno not in (errno.EAGAIN, errno.EACCES):
                 raise

lib/spack/llnl/util/tty/log.py

@@ -918,7 +918,7 @@ def _writer_daemon(
                         try:
                             if stdin_file.read(1) == "v":
                                 echo = not echo
-                        except OSError as e:
+                        except IOError as e:
                             # If SIGTTIN is ignored, the system gives EIO
                             # to let the caller know the read failed b/c it
                             # was in the bg. Ignore that too.
@@ -1013,7 +1013,7 @@ def wrapped(*args, **kwargs):
         while True:
             try:
                 return function(*args, **kwargs)
-            except OSError as e:
+            except IOError as e:
                 if e.errno == errno.EINTR:
                     continue
                 raise

lib/spack/spack/__init__.py

@@ -10,7 +10,7 @@
 import spack.util.git
 
 #: PEP440 canonical <major>.<minor>.<micro>.<devN> string
-__version__ = "1.0.0.dev0"
+__version__ = "0.24.0.dev0"
 spack_version = __version__
 
 

lib/spack/spack/binary_distribution.py

@@ -5,7 +5,6 @@
 import codecs
 import collections
 import concurrent.futures
-import contextlib
 import copy
 import hashlib
 import io
@@ -802,7 +801,7 @@ def url_read_method(url):
         try:
             _, _, spec_file = web_util.read_from_url(url)
             contents = codecs.getreader("utf-8")(spec_file).read()
-        except (web_util.SpackWebError, OSError) as e:
+        except web_util.SpackWebError as e:
             tty.error(f"Error reading specfile: {url}: {e}")
         return contents
 
@@ -2010,7 +2009,7 @@ def fetch_url_to_mirror(url):
 
                 # Download the config = spec.json and the relevant tarball
                 try:
-                    manifest = json.load(response)
+                    manifest = json.loads(response.read())
                     spec_digest = spack.oci.image.Digest.from_string(manifest["config"]["digest"])
                     tarball_digest = spack.oci.image.Digest.from_string(
                         manifest["layers"][-1]["digest"]
@@ -2271,24 +2270,6 @@ def relocate_package(spec: spack.spec.Spec) -> None:
             with fsys.edit_in_place_through_temporary_file(binary) as tmp_binary:
                 codesign("-fs-", tmp_binary)
 
-    install_manifest = os.path.join(
-        spec.prefix,
-        spack.store.STORE.layout.metadata_dir,
-        spack.store.STORE.layout.manifest_file_name,
-    )
-    if not os.path.exists(install_manifest):
-        spec_id = spec.format("{name}/{hash:7}")
-        tty.warn("No manifest file in tarball for spec %s" % spec_id)
-
-    # overwrite old metadata with new
-    if spec.spliced:
-        # rewrite spec on disk
-        spack.store.STORE.layout.write_spec(spec, spack.store.STORE.layout.spec_file_path(spec))
-
-        # de-cache the install manifest
-        with contextlib.suppress(FileNotFoundError):
-            os.unlink(install_manifest)
-
 
 def _extract_inner_tarball(spec, filename, extract_to, signature_required: bool, remote_checksum):
     stagepath = os.path.dirname(filename)
@@ -2455,6 +2436,15 @@ def extract_tarball(spec, download_result, force=False, timer=timer.NULL_TIMER):
     except Exception as e:
         shutil.rmtree(spec.prefix, ignore_errors=True)
         raise e
+    else:
+        manifest_file = os.path.join(
+            spec.prefix,
+            spack.store.STORE.layout.metadata_dir,
+            spack.store.STORE.layout.manifest_file_name,
+        )
+        if not os.path.exists(manifest_file):
+            spec_id = spec.format("{name}/{hash:7}")
+            tty.warn("No manifest file in tarball for spec %s" % spec_id)
     finally:
         if tmpdir:
             shutil.rmtree(tmpdir, ignore_errors=True)
@@ -2559,6 +2549,10 @@ def install_root_node(
         tty.msg('Installing "{0}" from a buildcache'.format(spec.format()))
         extract_tarball(spec, download_result, force)
         spec.package.windows_establish_runtime_linkage()
+        if spec.spliced:  # overwrite old metadata with new
+            spack.store.STORE.layout.write_spec(
+                spec, spack.store.STORE.layout.spec_file_path(spec)
+            )
         spack.hooks.post_install(spec, False)
         spack.store.STORE.db.add(spec, allow_missing=allow_missing)
 
@@ -2596,14 +2590,11 @@ def try_direct_fetch(spec, mirrors=None):
         )
         try:
             _, _, fs = web_util.read_from_url(buildcache_fetch_url_signed_json)
-            specfile_contents = codecs.getreader("utf-8")(fs).read()
             specfile_is_signed = True
-        except (web_util.SpackWebError, OSError) as e1:
+        except web_util.SpackWebError as e1:
             try:
                 _, _, fs = web_util.read_from_url(buildcache_fetch_url_json)
-                specfile_contents = codecs.getreader("utf-8")(fs).read()
-                specfile_is_signed = False
-            except (web_util.SpackWebError, OSError) as e2:
+            except web_util.SpackWebError as e2:
                 tty.debug(
                     f"Did not find {specfile_name} on {buildcache_fetch_url_signed_json}",
                     e1,
@@ -2613,6 +2604,7 @@ def try_direct_fetch(spec, mirrors=None):
                     f"Did not find {specfile_name} on {buildcache_fetch_url_json}", e2, level=2
                 )
                 continue
+        specfile_contents = codecs.getreader("utf-8")(fs).read()
 
         # read the spec from the build cache file. All specs in build caches
         # are concrete (as they are built) so we need to mark this spec
@@ -2706,9 +2698,8 @@ def get_keys(install=False, trust=False, force=False, mirrors=None):
 
         try:
             _, _, json_file = web_util.read_from_url(keys_index)
-            json_index = sjson.load(json_file)
-        except (web_util.SpackWebError, OSError, ValueError) as url_err:
-            # TODO: avoid repeated request
+            json_index = sjson.load(codecs.getreader("utf-8")(json_file))
+        except web_util.SpackWebError as url_err:
             if web_util.url_exists(keys_index):
                 tty.error(
                     f"Unable to find public keys in {url_util.format(fetch_url)},"
@@ -2958,11 +2949,11 @@ def get_remote_hash(self):
         url_index_hash = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, INDEX_HASH_FILE)
         try:
             response = self.urlopen(urllib.request.Request(url_index_hash, headers=self.headers))
-            remote_hash = response.read(64)
-        except OSError:
+        except (TimeoutError, urllib.error.URLError):
             return None
 
         # Validate the hash
+        remote_hash = response.read(64)
         if not re.match(rb"[a-f\d]{64}$", remote_hash):
             return None
         return remote_hash.decode("utf-8")
@@ -2980,13 +2971,13 @@ def conditional_fetch(self) -> FetchIndexResult:
 
         try:
             response = self.urlopen(urllib.request.Request(url_index, headers=self.headers))
-        except OSError as e:
-            raise FetchIndexError(f"Could not fetch index from {url_index}", e) from e
+        except (TimeoutError, urllib.error.URLError) as e:
+            raise FetchIndexError("Could not fetch index from {}".format(url_index), e) from e
 
         try:
             result = codecs.getreader("utf-8")(response).read()
-        except (ValueError, OSError) as e:
-            raise FetchIndexError(f"Remote index {url_index} is invalid") from e
+        except ValueError as e:
+            raise FetchIndexError("Remote index {} is invalid".format(url_index), e) from e
 
         computed_hash = compute_hash(result)
 
@@ -3030,12 +3021,12 @@ def conditional_fetch(self) -> FetchIndexResult:
                 # Not modified; that means fresh.
                 return FetchIndexResult(etag=None, hash=None, data=None, fresh=True)
             raise FetchIndexError(f"Could not fetch index {url}", e) from e
-        except OSError as e:  # URLError, socket.timeout, etc.
+        except (TimeoutError, urllib.error.URLError) as e:
             raise FetchIndexError(f"Could not fetch index {url}", e) from e
 
         try:
             result = codecs.getreader("utf-8")(response).read()
-        except (ValueError, OSError) as e:
+        except ValueError as e:
             raise FetchIndexError(f"Remote index {url} is invalid", e) from e
 
         headers = response.headers
@@ -3067,11 +3058,11 @@ def conditional_fetch(self) -> FetchIndexResult:
                     headers={"Accept": "application/vnd.oci.image.manifest.v1+json"},
                 )
             )
-        except OSError as e:
+        except (TimeoutError, urllib.error.URLError) as e:
             raise FetchIndexError(f"Could not fetch manifest from {url_manifest}", e) from e
 
         try:
-            manifest = json.load(response)
+            manifest = json.loads(response.read())
         except Exception as e:
             raise FetchIndexError(f"Remote index {url_manifest} is invalid", e) from e
 
@@ -3086,16 +3077,14 @@ def conditional_fetch(self) -> FetchIndexResult:
             return FetchIndexResult(etag=None, hash=None, data=None, fresh=True)
 
         # Otherwise fetch the blob / index.json
-        try:
-            response = self.urlopen(
-                urllib.request.Request(
-                    url=self.ref.blob_url(index_digest),
-                    headers={"Accept": "application/vnd.oci.image.layer.v1.tar+gzip"},
-                )
+        response = self.urlopen(
+            urllib.request.Request(
+                url=self.ref.blob_url(index_digest),
+                headers={"Accept": "application/vnd.oci.image.layer.v1.tar+gzip"},
             )
-            result = codecs.getreader("utf-8")(response).read()
-        except (OSError, ValueError) as e:
-            raise FetchIndexError(f"Remote index {url_manifest} is invalid", e) from e
+        )
+
+        result = codecs.getreader("utf-8")(response).read()
 
         # Make sure the blob we download has the advertised hash
         if compute_hash(result) != index_digest.digest:

lib/spack/spack/bootstrap/_common.py

@@ -5,7 +5,7 @@
 import fnmatch
 import glob
 import importlib
-import os
+import os.path
 import re
 import sys
 import sysconfig

lib/spack/spack/bootstrap/clingo.py

@@ -27,9 +27,9 @@
 class ClingoBootstrapConcretizer:
     def __init__(self, configuration):
         self.host_platform = spack.platforms.host()
-        self.host_os = self.host_platform.default_operating_system()
+        self.host_os = self.host_platform.operating_system("frontend")
         self.host_target = archspec.cpu.host().family
-        self.host_architecture = spack.spec.ArchSpec.default_arch()
+        self.host_architecture = spack.spec.ArchSpec.frontend_arch()
         self.host_architecture.target = str(self.host_target)
         self.host_compiler = self._valid_compiler_or_raise()
         self.host_python = self.python_external_spec()

lib/spack/spack/bootstrap/config.py

@@ -4,7 +4,7 @@
 """Manage configuration swapping for bootstrapping purposes"""
 
 import contextlib
-import os
+import os.path
 import sys
 from typing import Any, Dict, Generator, MutableSequence, Sequence
 
@@ -141,7 +141,7 @@ def _bootstrap_config_scopes() -> Sequence["spack.config.ConfigScope"]:
 
 
 def _add_compilers_if_missing() -> None:
-    arch = spack.spec.ArchSpec.default_arch()
+    arch = spack.spec.ArchSpec.frontend_arch()
     if not spack.compilers.compilers_for_arch(arch):
         spack.compilers.find_compilers()
 

lib/spack/spack/bootstrap/core.py

@@ -25,6 +25,7 @@
 import functools
 import json
 import os
+import os.path
 import sys
 import uuid
 from typing import Any, Callable, Dict, List, Optional, Tuple
@@ -45,7 +46,6 @@
 import spack.util.executable
 import spack.util.path
 import spack.util.spack_yaml
-import spack.util.url
 import spack.version
 from spack.installer import PackageInstaller
 
@@ -97,12 +97,8 @@ def __init__(self, conf: ConfigDictionary) -> None:
         self.name = conf["name"]
         self.metadata_dir = spack.util.path.canonicalize_path(conf["metadata"])
 
-        # Check for relative paths, and turn them into absolute paths
-        # root is the metadata_dir
-        maybe_url = conf["info"]["url"]
-        if spack.util.url.is_path_instead_of_url(maybe_url) and not os.path.isabs(maybe_url):
-            maybe_url = os.path.join(self.metadata_dir, maybe_url)
-        self.url = spack.mirrors.mirror.Mirror(maybe_url).fetch_url
+        # Promote (relative) paths to file urls
+        self.url = spack.mirrors.mirror.Mirror(conf["info"]["url"]).fetch_url
 
     @property
     def mirror_scope(self) -> spack.config.InternalConfigScope:

lib/spack/spack/build_systems/autotools.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
+import os.path
 import stat
 import subprocess
 from typing import Callable, List, Optional, Set, Tuple, Union
@@ -190,6 +191,177 @@ def archive_files(self) -> List[str]:
             files.append(self._removed_la_files_log)
         return files
 
+    @property
+    def configure_directory(self) -> str:
+        """Return the directory where 'configure' resides."""
+        return self.pkg.stage.source_path
+
+    @property
+    def configure_abs_path(self) -> str:
+        # Absolute path to configure
+        configure_abs_path = os.path.join(os.path.abspath(self.configure_directory), "configure")
+        return configure_abs_path
+
+    @property
+    def build_directory(self) -> str:
+        """Override to provide another place to build the package"""
+        # Handle the case where the configure directory is set to a non-absolute path
+        # Non-absolute paths are always relative to the staging source path
+        build_dir = self.configure_directory
+        if not os.path.isabs(build_dir):
+            build_dir = os.path.join(self.pkg.stage.source_path, build_dir)
+        return build_dir
+
+    @property
+    def autoreconf_search_path_args(self) -> List[str]:
+        """Search path includes for autoreconf. Add an -I flag for all `aclocal` dirs
+        of build deps, skips the default path of automake, move external include
+        flags to the back, since they might pull in unrelated m4 files shadowing
+        spack dependencies."""
+        return _autoreconf_search_path_args(self.spec)
+
+    def autoreconf(
+        self, pkg: AutotoolsPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
+        """Not needed usually, configure should be already there"""
+
+        # If configure exists nothing needs to be done
+        if os.path.exists(self.configure_abs_path):
+            return
+
+        # Else try to regenerate it, which requires a few build dependencies
+        ensure_build_dependencies_or_raise(
+            spec=spec,
+            dependencies=["autoconf", "automake", "libtool"],
+            error_msg="Cannot generate configure",
+        )
+
+        tty.msg("Configure script not found: trying to generate it")
+        tty.warn("*********************************************************")
+        tty.warn("* If the default procedure fails, consider implementing *")
+        tty.warn("*        a custom AUTORECONF phase in the package       *")
+        tty.warn("*********************************************************")
+        with fs.working_dir(self.configure_directory):
+            # This line is what is needed most of the time
+            # --install, --verbose, --force
+            autoreconf_args = ["-ivf"]
+            autoreconf_args += self.autoreconf_search_path_args
+            autoreconf_args += self.autoreconf_extra_args
+            self.pkg.module.autoreconf(*autoreconf_args)
+
+    def configure(
+        self, pkg: AutotoolsPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
+        """Run "configure", with the arguments specified by the builder and an
+        appropriately set prefix.
+        """
+        options = getattr(self.pkg, "configure_flag_args", [])
+        options += ["--prefix={0}".format(prefix)]
+        options += self.configure_args()
+
+        with fs.working_dir(self.build_directory, create=True):
+            pkg.module.configure(*options)
+
+    def build(
+        self, pkg: AutotoolsPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
+        """Run "make" on the build targets specified by the builder."""
+        # See https://autotools.io/automake/silent.html
+        params = ["V=1"]
+        params += self.build_targets
+        with fs.working_dir(self.build_directory):
+            pkg.module.make(*params)
+
+    def install(
+        self, pkg: AutotoolsPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
+    ) -> None:
+        """Run "make" on the install targets specified by the builder."""
+        with fs.working_dir(self.build_directory):
+            pkg.module.make(*self.install_targets)
+
+    def check(self) -> None:
+        """Run "make" on the ``test`` and ``check`` targets, if found."""
+        with fs.working_dir(self.build_directory):
+            self.pkg._if_make_target_execute("test")
+            self.pkg._if_make_target_execute("check")
+
+    def installcheck(self) -> None:
+        """Run "make" on the ``installcheck`` target, if found."""
+        with fs.working_dir(self.build_directory):
+            self.pkg._if_make_target_execute("installcheck")
+
+    def setup_build_environment(self, env):
+        if self.spec.platform == "darwin" and macos_version() >= Version("11"):
+            # Many configure files rely on matching '10.*' for macOS version
+            # detection and fail to add flags if it shows as version 11.
+            env.set("MACOSX_DEPLOYMENT_TARGET", "10.16")
+
+    def with_or_without(
+        self,
+        name: str,
+        activation_value: Optional[Union[Callable, str]] = None,
+        variant: Optional[str] = None,
+    ) -> List[str]:
+        """Inspects a variant and returns the arguments that activate
+        or deactivate the selected feature(s) for the configure options.
+
+        This function works on all type of variants. For bool-valued variants
+        it will return by default ``--with-{name}`` or ``--without-{name}``.
+        For other kinds of variants it will cycle over the allowed values and
+        return either ``--with-{value}`` or ``--without-{value}``.
+
+        If activation_value is given, then for each possible value of the
+        variant, the option ``--with-{value}=activation_value(value)`` or
+        ``--without-{value}`` will be added depending on whether or not
+        ``variant=value`` is in the spec.
+
+        Args:
+            name: name of a valid multi-valued variant
+            activation_value: callable that accepts a single value and returns the parameter to be
+                used leading to an entry of the type ``--with-{name}={parameter}``.
+
+                The special value "prefix" can also be assigned and will return
+                ``spec[name].prefix`` as activation parameter.
+
+        Returns:
+            list of arguments to configure
+        """
+        return self._activate_or_not(name, "with", "without", activation_value, variant)
+
+    def enable_or_disable(
+        self,
+        name: str,
+        activation_value: Optional[Union[Callable, str]] = None,
+        variant: Optional[str] = None,
+    ) -> List[str]:
+        """Same as
+        :meth:`~spack.build_systems.autotools.AutotoolsBuilder.with_or_without`
+        but substitute ``with`` with ``enable`` and ``without`` with ``disable``.
+
+        Args:
+            name: name of a valid multi-valued variant
+            activation_value: if present accepts a single value and returns the parameter to be
+                used leading to an entry of the type ``--enable-{name}={parameter}``
+
+                The special value "prefix" can also be assigned and will return
+                ``spec[name].prefix`` as activation parameter.
+
+        Returns:
+            list of arguments to configure
+        """
+        return self._activate_or_not(name, "enable", "disable", activation_value, variant)
+
+    def configure_args(self) -> List[str]:
+        """Return the list of all the arguments that must be passed to configure,
+        except ``--prefix`` which will be pre-pended to the list.
+        """
+        return []
+
+    @spack.phase_callbacks.run_before("autoreconf")
+    def _delete_configure_to_force_update(self) -> None:
+        if self.force_autoreconf:
+            fs.force_remove(self.configure_abs_path)
+
     @spack.phase_callbacks.run_after("autoreconf")
     def _do_patch_config_files(self) -> None:
         """Some packages ship with older config.guess/config.sub files and need to
@@ -302,6 +474,24 @@ def runs_ok(script_abs_path):
             fs.copy(substitutes[name], abs_path)
             os.chmod(abs_path, mode)
 
+    @spack.phase_callbacks.run_after("autoreconf")
+    def _set_configure_or_die(self) -> None:
+        """Ensure the presence of a "configure" script, or raise. If the "configure"
+        is found, a module level attribute is set.
+
+        Raises:
+             RuntimeError: if the "configure" script is not found
+        """
+        # Check if the "configure" script is there. If not raise a RuntimeError.
+        if not os.path.exists(self.configure_abs_path):
+            msg = "configure script not found in {0}"
+            raise RuntimeError(msg.format(self.configure_directory))
+
+        # Monkey-patch the configure script in the corresponding module
+        globals_for_pkg = spack.build_environment.ModuleChangePropagator(self.pkg)
+        globals_for_pkg.configure = Executable(self.configure_abs_path)
+        globals_for_pkg.propagate_changes_to_mro()
+
     @spack.phase_callbacks.run_before("configure")
     def _patch_usr_bin_file(self) -> None:
         """On NixOS file is not available in /usr/bin/file. Patch configure
@@ -511,130 +701,27 @@ def _do_patch_libtool(self) -> None:
                 stop_at=stop_at,
             )
 
-    @property
-    def configure_directory(self) -> str:
-        """Return the directory where 'configure' resides."""
-        return self.pkg.stage.source_path
+    spack.phase_callbacks.run_after("build")(execute_build_time_tests)
+    spack.phase_callbacks.run_after("install")(execute_install_time_tests)
 
-    @property
-    def configure_abs_path(self) -> str:
-        # Absolute path to configure
-        configure_abs_path = os.path.join(os.path.abspath(self.configure_directory), "configure")
-        return configure_abs_path
-
-    @property
-    def build_directory(self) -> str:
-        """Override to provide another place to build the package"""
-        # Handle the case where the configure directory is set to a non-absolute path
-        # Non-absolute paths are always relative to the staging source path
-        build_dir = self.configure_directory
-        if not os.path.isabs(build_dir):
-            build_dir = os.path.join(self.pkg.stage.source_path, build_dir)
-        return build_dir
-
-    @spack.phase_callbacks.run_before("autoreconf")
-    def _delete_configure_to_force_update(self) -> None:
-        if self.force_autoreconf:
-            fs.force_remove(self.configure_abs_path)
-
-    @property
-    def autoreconf_search_path_args(self) -> List[str]:
-        """Search path includes for autoreconf. Add an -I flag for all `aclocal` dirs
-        of build deps, skips the default path of automake, move external include
-        flags to the back, since they might pull in unrelated m4 files shadowing
-        spack dependencies."""
-        return _autoreconf_search_path_args(self.spec)
-
-    @spack.phase_callbacks.run_after("autoreconf")
-    def _set_configure_or_die(self) -> None:
-        """Ensure the presence of a "configure" script, or raise. If the "configure"
-        is found, a module level attribute is set.
-
-        Raises:
-             RuntimeError: if the "configure" script is not found
+    @spack.phase_callbacks.run_after("install")
+    def _remove_libtool_archives(self) -> None:
+        """Remove all .la files in prefix sub-folders if the package sets
+        ``install_libtool_archives`` to be False.
         """
-        # Check if the "configure" script is there. If not raise a RuntimeError.
-        if not os.path.exists(self.configure_abs_path):
-            msg = "configure script not found in {0}"
-            raise RuntimeError(msg.format(self.configure_directory))
-
-        # Monkey-patch the configure script in the corresponding module
-        globals_for_pkg = spack.build_environment.ModuleChangePropagator(self.pkg)
-        globals_for_pkg.configure = Executable(self.configure_abs_path)
-        globals_for_pkg.propagate_changes_to_mro()
-
-    def configure_args(self) -> List[str]:
-        """Return the list of all the arguments that must be passed to configure,
-        except ``--prefix`` which will be pre-pended to the list.
-        """
-        return []
-
-    def autoreconf(
-        self, pkg: AutotoolsPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
-    ) -> None:
-        """Not needed usually, configure should be already there"""
-
-        # If configure exists nothing needs to be done
-        if os.path.exists(self.configure_abs_path):
+        # If .la files are to be installed there's nothing to do
+        if self.install_libtool_archives:
             return
 
-        # Else try to regenerate it, which requires a few build dependencies
-        ensure_build_dependencies_or_raise(
-            spec=spec,
-            dependencies=["autoconf", "automake", "libtool"],
-            error_msg="Cannot generate configure",
-        )
+        # Remove the files and create a log of what was removed
+        libtool_files = fs.find(str(self.pkg.prefix), "*.la", recursive=True)
+        with fs.safe_remove(*libtool_files):
+            fs.mkdirp(os.path.dirname(self._removed_la_files_log))
+            with open(self._removed_la_files_log, mode="w", encoding="utf-8") as f:
+                f.write("\n".join(libtool_files))
 
-        tty.msg("Configure script not found: trying to generate it")
-        tty.warn("*********************************************************")
-        tty.warn("* If the default procedure fails, consider implementing *")
-        tty.warn("*        a custom AUTORECONF phase in the package       *")
-        tty.warn("*********************************************************")
-        with fs.working_dir(self.configure_directory):
-            # This line is what is needed most of the time
-            # --install, --verbose, --force
-            autoreconf_args = ["-ivf"]
-            autoreconf_args += self.autoreconf_search_path_args
-            autoreconf_args += self.autoreconf_extra_args
-            self.pkg.module.autoreconf(*autoreconf_args)
-
-    def configure(
-        self, pkg: AutotoolsPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
-    ) -> None:
-        """Run "configure", with the arguments specified by the builder and an
-        appropriately set prefix.
-        """
-        options = getattr(self.pkg, "configure_flag_args", [])
-        options += ["--prefix={0}".format(prefix)]
-        options += self.configure_args()
-
-        with fs.working_dir(self.build_directory, create=True):
-            pkg.module.configure(*options)
-
-    def build(
-        self, pkg: AutotoolsPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
-    ) -> None:
-        """Run "make" on the build targets specified by the builder."""
-        # See https://autotools.io/automake/silent.html
-        params = ["V=1"]
-        params += self.build_targets
-        with fs.working_dir(self.build_directory):
-            pkg.module.make(*params)
-
-    def install(
-        self, pkg: AutotoolsPackage, spec: spack.spec.Spec, prefix: spack.util.prefix.Prefix
-    ) -> None:
-        """Run "make" on the install targets specified by the builder."""
-        with fs.working_dir(self.build_directory):
-            pkg.module.make(*self.install_targets)
-
-    spack.phase_callbacks.run_after("build")(execute_build_time_tests)
-
-    def check(self) -> None:
-        """Run "make" on the ``test`` and ``check`` targets, if found."""
-        with fs.working_dir(self.build_directory):
-            self.pkg._if_make_target_execute("test")
-            self.pkg._if_make_target_execute("check")
+    # On macOS, force rpaths for shared library IDs and remove duplicate rpaths
+    spack.phase_callbacks.run_after("install", when="platform=darwin")(apply_macos_rpath_fixups)
 
     def _activate_or_not(
         self,
@@ -756,93 +843,6 @@ def _default_generator(is_activated):
             args.append(line_generator(activated))
         return args
 
-    def with_or_without(
-        self,
-        name: str,
-        activation_value: Optional[Union[Callable, str]] = None,
-        variant: Optional[str] = None,
-    ) -> List[str]:
-        """Inspects a variant and returns the arguments that activate
-        or deactivate the selected feature(s) for the configure options.
-
-        This function works on all type of variants. For bool-valued variants
-        it will return by default ``--with-{name}`` or ``--without-{name}``.
-        For other kinds of variants it will cycle over the allowed values and
-        return either ``--with-{value}`` or ``--without-{value}``.
-
-        If activation_value is given, then for each possible value of the
-        variant, the option ``--with-{value}=activation_value(value)`` or
-        ``--without-{value}`` will be added depending on whether or not
-        ``variant=value`` is in the spec.
-
-        Args:
-            name: name of a valid multi-valued variant
-            activation_value: callable that accepts a single value and returns the parameter to be
-                used leading to an entry of the type ``--with-{name}={parameter}``.
-
-                The special value "prefix" can also be assigned and will return
-                ``spec[name].prefix`` as activation parameter.
-
-        Returns:
-            list of arguments to configure
-        """
-        return self._activate_or_not(name, "with", "without", activation_value, variant)
-
-    def enable_or_disable(
-        self,
-        name: str,
-        activation_value: Optional[Union[Callable, str]] = None,
-        variant: Optional[str] = None,
-    ) -> List[str]:
-        """Same as
-        :meth:`~spack.build_systems.autotools.AutotoolsBuilder.with_or_without`
-        but substitute ``with`` with ``enable`` and ``without`` with ``disable``.
-
-        Args:
-            name: name of a valid multi-valued variant
-            activation_value: if present accepts a single value and returns the parameter to be
-                used leading to an entry of the type ``--enable-{name}={parameter}``
-
-                The special value "prefix" can also be assigned and will return
-                ``spec[name].prefix`` as activation parameter.
-
-        Returns:
-            list of arguments to configure
-        """
-        return self._activate_or_not(name, "enable", "disable", activation_value, variant)
-
-    spack.phase_callbacks.run_after("install")(execute_install_time_tests)
-
-    def installcheck(self) -> None:
-        """Run "make" on the ``installcheck`` target, if found."""
-        with fs.working_dir(self.build_directory):
-            self.pkg._if_make_target_execute("installcheck")
-
-    @spack.phase_callbacks.run_after("install")
-    def _remove_libtool_archives(self) -> None:
-        """Remove all .la files in prefix sub-folders if the package sets
-        ``install_libtool_archives`` to be False.
-        """
-        # If .la files are to be installed there's nothing to do
-        if self.install_libtool_archives:
-            return
-
-        # Remove the files and create a log of what was removed
-        libtool_files = fs.find(str(self.pkg.prefix), "*.la", recursive=True)
-        with fs.safe_remove(*libtool_files):
-            fs.mkdirp(os.path.dirname(self._removed_la_files_log))
-            with open(self._removed_la_files_log, mode="w", encoding="utf-8") as f:
-                f.write("\n".join(libtool_files))
-
-    def setup_build_environment(self, env):
-        if self.spec.platform == "darwin" and macos_version() >= Version("11"):
-            # Many configure files rely on matching '10.*' for macOS version
-            # detection and fail to add flags if it shows as version 11.
-            env.set("MACOSX_DEPLOYMENT_TARGET", "10.16")
-
-    # On macOS, force rpaths for shared library IDs and remove duplicate rpaths
-    spack.phase_callbacks.run_after("install", when="platform=darwin")(apply_macos_rpath_fixups)
-
 
 def _autoreconf_search_path_args(spec: spack.spec.Spec) -> List[str]:
     dirs_seen: Set[Tuple[int, int]] = set()

lib/spack/spack/build_systems/cmake.py

@@ -458,18 +458,11 @@ def cmake(
     ) -> None:
         """Runs ``cmake`` in the build directory"""
 
-        if spec.is_develop:
-            # skip cmake phase if it is an incremental develop build
-
-            # Determine the files that will re-run CMake that are generated from a successful
-            # configure step based on state
-            primary_generator = _extract_primary_generator(self.generator)
-            configure_artifact = "Makefile"
-            if primary_generator == "Ninja":
-                configure_artifact = "ninja.build"
-
-            if os.path.isfile(os.path.join(self.build_directory, configure_artifact)):
-                return
+        # skip cmake phase if it is an incremental develop build
+        if spec.is_develop and os.path.isfile(
+            os.path.join(self.build_directory, "CMakeCache.txt")
+        ):
+            return
 
         options = self.std_cmake_args
         options += self.cmake_args()

lib/spack/spack/build_systems/cuda.py

@@ -15,7 +15,7 @@ class CudaPackage(PackageBase):
     """Auxiliary class which contains CUDA variant, dependencies and conflicts
     and is meant to unify and facilitate its usage.
 
-    Maintainers: ax3l, Rombur, davidbeckingsale, pauleonix
+    Maintainers: ax3l, Rombur, davidbeckingsale
     """
 
     # https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#gpu-feature-list
@@ -47,12 +47,6 @@ class CudaPackage(PackageBase):
         "89",
         "90",
         "90a",
-        "100",
-        "100a",
-        "101",
-        "101a",
-        "120",
-        "120a",
     )
 
     # FIXME: keep cuda and cuda_arch separate to make usage easier until
@@ -105,56 +99,39 @@ def compute_capabilities(arch_list: Iterable[str]) -> List[str]:
     # CUDA version vs Architecture
     # https://en.wikipedia.org/wiki/CUDA#GPUs_supported
     # https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/index.html#deprecated-features
-    # Tesla support:
     depends_on("cuda@:6.0", when="cuda_arch=10")
     depends_on("cuda@:6.5", when="cuda_arch=11")
     depends_on("cuda@2.1:6.5", when="cuda_arch=12")
     depends_on("cuda@2.1:6.5", when="cuda_arch=13")
 
-    # Fermi support:
     depends_on("cuda@3.0:8.0", when="cuda_arch=20")
     depends_on("cuda@3.2:8.0", when="cuda_arch=21")
 
-    # Kepler support:
     depends_on("cuda@5.0:10.2", when="cuda_arch=30")
     depends_on("cuda@5.0:10.2", when="cuda_arch=32")
     depends_on("cuda@5.0:11.8", when="cuda_arch=35")
     depends_on("cuda@6.5:11.8", when="cuda_arch=37")
 
-    # Maxwell support:
     depends_on("cuda@6.0:", when="cuda_arch=50")
     depends_on("cuda@6.5:", when="cuda_arch=52")
     depends_on("cuda@6.5:", when="cuda_arch=53")
 
-    # Pascal support:
     depends_on("cuda@8.0:", when="cuda_arch=60")
     depends_on("cuda@8.0:", when="cuda_arch=61")
     depends_on("cuda@8.0:", when="cuda_arch=62")
 
-    # Volta support:
     depends_on("cuda@9.0:", when="cuda_arch=70")
-    # Turing support:
     depends_on("cuda@9.0:", when="cuda_arch=72")
     depends_on("cuda@10.0:", when="cuda_arch=75")
 
-    # Ampere support:
     depends_on("cuda@11.0:", when="cuda_arch=80")
     depends_on("cuda@11.1:", when="cuda_arch=86")
     depends_on("cuda@11.4:", when="cuda_arch=87")
-    # Ada support:
     depends_on("cuda@11.8:", when="cuda_arch=89")
 
-    # Hopper support:
     depends_on("cuda@12.0:", when="cuda_arch=90")
     depends_on("cuda@12.0:", when="cuda_arch=90a")
 
-    # Blackwell support:
-    depends_on("cuda@12.8:", when="cuda_arch=100")
-    depends_on("cuda@12.8:", when="cuda_arch=100a")
-    depends_on("cuda@12.8:", when="cuda_arch=101")
-    depends_on("cuda@12.8:", when="cuda_arch=101a")
-    depends_on("cuda@12.8:", when="cuda_arch=120")
-    depends_on("cuda@12.8:", when="cuda_arch=120a")
     # From the NVIDIA install guide we know of conflicts for particular
     # platforms (linux, darwin), architectures (x86, powerpc) and compilers
     # (gcc, clang). We don't restrict %gcc and %clang conflicts to
@@ -186,7 +163,6 @@ def compute_capabilities(arch_list: Iterable[str]) -> List[str]:
         conflicts("%gcc@12:", when="+cuda ^cuda@:11.8")
         conflicts("%gcc@13:", when="+cuda ^cuda@:12.3")
         conflicts("%gcc@14:", when="+cuda ^cuda@:12.6")
-        conflicts("%gcc@15:", when="+cuda ^cuda@:12.8")
         conflicts("%clang@12:", when="+cuda ^cuda@:11.4.0")
         conflicts("%clang@13:", when="+cuda ^cuda@:11.5")
         conflicts("%clang@14:", when="+cuda ^cuda@:11.7")
@@ -195,7 +171,6 @@ def compute_capabilities(arch_list: Iterable[str]) -> List[str]:
         conflicts("%clang@17:", when="+cuda ^cuda@:12.3")
         conflicts("%clang@18:", when="+cuda ^cuda@:12.5")
         conflicts("%clang@19:", when="+cuda ^cuda@:12.6")
-        conflicts("%clang@20:", when="+cuda ^cuda@:12.8")
 
         # https://gist.github.com/ax3l/9489132#gistcomment-3860114
         conflicts("%gcc@10", when="+cuda ^cuda@:11.4.0")

lib/spack/spack/build_systems/go.py

@@ -9,7 +9,7 @@
 import spack.phase_callbacks
 import spack.spec
 import spack.util.prefix
-from spack.directives import build_system, depends_on
+from spack.directives import build_system, extends
 from spack.multimethod import when
 
 from ._checks import BuilderWithDefaults, execute_install_time_tests
@@ -28,7 +28,9 @@ class GoPackage(spack.package_base.PackageBase):
     build_system("go")
 
     with when("build_system=go"):
-        depends_on("go", type="build")
+        # TODO: this seems like it should be depends_on, see
+        # setup_dependent_build_environment in go for why I kept it like this
+        extends("go@1.14:", type="build")
 
 
 @spack.builder.builder("go")
@@ -71,7 +73,6 @@ class GoBuilder(BuilderWithDefaults):
     def setup_build_environment(self, env):
         env.set("GO111MODULE", "on")
         env.set("GOTOOLCHAIN", "local")
-        env.set("GOPATH", fs.join_path(self.pkg.stage.path, "go"))
 
     @property
     def build_directory(self):
@@ -82,15 +83,7 @@ def build_directory(self):
     def build_args(self):
         """Arguments for ``go build``."""
         # Pass ldflags -s = --strip-all and -w = --no-warnings by default
-        return [
-            "-p",
-            str(self.pkg.module.make_jobs),
-            "-modcacherw",
-            "-ldflags",
-            "-s -w",
-            "-o",
-            f"{self.pkg.name}",
-        ]
+        return ["-modcacherw", "-ldflags", "-s -w", "-o", f"{self.pkg.name}"]
 
     @property
     def check_args(self):

lib/spack/spack/build_systems/python.py

@@ -264,17 +264,16 @@ def update_external_dependencies(self, extendee_spec=None):
                     # Ensure architecture information is present
                     if not python.architecture:
                         host_platform = spack.platforms.host()
-                        host_os = host_platform.default_operating_system()
-                        host_target = host_platform.default_target()
+                        host_os = host_platform.operating_system("default_os")
+                        host_target = host_platform.target("default_target")
                         python.architecture = spack.spec.ArchSpec(
                             (str(host_platform), str(host_os), str(host_target))
                         )
                     else:
                         if not python.architecture.platform:
                             python.architecture.platform = spack.platforms.host()
-                        platform = spack.platforms.by_name(python.architecture.platform)
                         if not python.architecture.os:
-                            python.architecture.os = platform.default_operating_system()
+                            python.architecture.os = "default_os"
                         if not python.architecture.target:
                             python.architecture.target = archspec.cpu.host().family.name
 

lib/spack/spack/build_systems/sip.py

@@ -43,7 +43,6 @@ class SIPPackage(spack.package_base.PackageBase):
     with when("build_system=sip"):
         extends("python", type=("build", "link", "run"))
         depends_on("py-sip", type="build")
-        depends_on("gmake", type="build")
 
     @property
     def import_modules(self):

lib/spack/spack/ci/__init__.py

@@ -14,7 +14,8 @@
 import zipfile
 from collections import namedtuple
 from typing import Callable, Dict, List, Set
-from urllib.request import Request
+from urllib.error import HTTPError, URLError
+from urllib.request import HTTPHandler, Request, build_opener
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
@@ -62,8 +63,6 @@
 
 PushResult = namedtuple("PushResult", "success url")
 
-urlopen = web_util.urlopen  # alias for mocking in tests
-
 
 def get_change_revisions():
     """If this is a git repo get the revisions to use when checking
@@ -473,9 +472,12 @@ def generate_pipeline(env: ev.Environment, args) -> None:
     # Use all unpruned specs to populate the build group for this set
     cdash_config = cfg.get("cdash")
     if options.cdash_handler and options.cdash_handler.auth_token:
-        options.cdash_handler.populate_buildgroup(
-            [options.cdash_handler.build_name(s) for s in pipeline_specs]
-        )
+        try:
+            options.cdash_handler.populate_buildgroup(
+                [options.cdash_handler.build_name(s) for s in pipeline_specs]
+            )
+        except (SpackError, HTTPError, URLError, TimeoutError) as err:
+            tty.warn(f"Problem populating buildgroup: {err}")
     elif cdash_config:
         # warn only if there was actually a CDash configuration.
         tty.warn("Unable to populate buildgroup without CDash credentials")
@@ -629,19 +631,29 @@ def download_and_extract_artifacts(url, work_dir):
     if token:
         headers["PRIVATE-TOKEN"] = token
 
-    request = Request(url, headers=headers, method="GET")
+    opener = build_opener(HTTPHandler)
+
+    request = Request(url, headers=headers)
+    request.get_method = lambda: "GET"
+
+    response = opener.open(request, timeout=SPACK_CDASH_TIMEOUT)
+    response_code = response.getcode()
+
+    if response_code != 200:
+        msg = f"Error response code ({response_code}) in reproduce_ci_job"
+        raise SpackError(msg)
+
     artifacts_zip_path = os.path.join(work_dir, "artifacts.zip")
-    os.makedirs(work_dir, exist_ok=True)
 
-    try:
-        response = urlopen(request, timeout=SPACK_CDASH_TIMEOUT)
-        with open(artifacts_zip_path, "wb") as out_file:
-            shutil.copyfileobj(response, out_file)
-    except OSError as e:
-        raise SpackError(f"Error fetching artifacts: {e}")
+    if not os.path.exists(work_dir):
+        os.makedirs(work_dir)
 
-    with zipfile.ZipFile(artifacts_zip_path) as zip_file:
-        zip_file.extractall(work_dir)
+    with open(artifacts_zip_path, "wb") as out_file:
+        shutil.copyfileobj(response, out_file)
+
+    zip_file = zipfile.ZipFile(artifacts_zip_path)
+    zip_file.extractall(work_dir)
+    zip_file.close()
 
     os.remove(artifacts_zip_path)
 

lib/spack/spack/ci/common.py

@@ -1,21 +1,23 @@
 # Copyright Spack Project Developers. See COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import codecs
 import copy
 import json
 import os
 import re
+import ssl
 import sys
 import time
 from collections import deque
 from enum import Enum
 from typing import Dict, Generator, List, Optional, Set, Tuple
 from urllib.parse import quote, urlencode, urlparse
-from urllib.request import Request
+from urllib.request import HTTPHandler, HTTPSHandler, Request, build_opener
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
-from llnl.util.lang import memoized
+from llnl.util.lang import Singleton, memoized
 
 import spack.binary_distribution as bindist
 import spack.config as cfg
@@ -33,11 +35,32 @@
 from spack.reporters.cdash import SPACK_CDASH_TIMEOUT
 from spack.reporters.cdash import build_stamp as cdash_build_stamp
 
+
+def _urlopen():
+    error_handler = web_util.SpackHTTPDefaultErrorHandler()
+
+    # One opener with HTTPS ssl enabled
+    with_ssl = build_opener(
+        HTTPHandler(), HTTPSHandler(context=web_util.ssl_create_default_context()), error_handler
+    )
+
+    # One opener with HTTPS ssl disabled
+    without_ssl = build_opener(
+        HTTPHandler(), HTTPSHandler(context=ssl._create_unverified_context()), error_handler
+    )
+
+    # And dynamically dispatch based on the config:verify_ssl.
+    def dispatch_open(fullurl, data=None, timeout=None, verify_ssl=True):
+        opener = with_ssl if verify_ssl else without_ssl
+        timeout = timeout or cfg.get("config:connect_timeout", 1)
+        return opener.open(fullurl, data, timeout)
+
+    return dispatch_open
+
+
 IS_WINDOWS = sys.platform == "win32"
 SPACK_RESERVED_TAGS = ["public", "protected", "notary"]
-
-# this exists purely for testing purposes
-_urlopen = web_util.urlopen
+_dyn_mapping_urlopener = Singleton(_urlopen)
 
 
 def copy_files_to_artifacts(src, artifacts_dir):
@@ -256,25 +279,26 @@ def copy_test_results(self, source, dest):
         reports = fs.join_path(source, "*_Test*.xml")
         copy_files_to_artifacts(reports, dest)
 
-    def create_buildgroup(self, headers, url, group_name, group_type):
+    def create_buildgroup(self, opener, headers, url, group_name, group_type):
         data = {"newbuildgroup": group_name, "project": self.project, "type": group_type}
 
         enc_data = json.dumps(data).encode("utf-8")
 
         request = Request(url, data=enc_data, headers=headers)
 
-        try:
-            response_text = _urlopen(request, timeout=SPACK_CDASH_TIMEOUT).read()
-        except OSError as e:
-            tty.warn(f"Failed to create CDash buildgroup: {e}")
+        response = opener.open(request, timeout=SPACK_CDASH_TIMEOUT)
+        response_code = response.getcode()
+
+        if response_code not in [200, 201]:
+            msg = f"Creating buildgroup failed (response code = {response_code})"
+            tty.warn(msg)
             return None
 
-        try:
-            response_json = json.loads(response_text)
-            return response_json["id"]
-        except (json.JSONDecodeError, KeyError) as e:
-            tty.warn(f"Failed to parse CDash response: {e}")
-            return None
+        response_text = response.read()
+        response_json = json.loads(response_text)
+        build_group_id = response_json["id"]
+
+        return build_group_id
 
     def populate_buildgroup(self, job_names):
         url = f"{self.url}/api/v1/buildgroup.php"
@@ -284,11 +308,16 @@ def populate_buildgroup(self, job_names):
             "Content-Type": "application/json",
         }
 
-        parent_group_id = self.create_buildgroup(headers, url, self.build_group, "Daily")
-        group_id = self.create_buildgroup(headers, url, f"Latest {self.build_group}", "Latest")
+        opener = build_opener(HTTPHandler)
+
+        parent_group_id = self.create_buildgroup(opener, headers, url, self.build_group, "Daily")
+        group_id = self.create_buildgroup(
+            opener, headers, url, f"Latest {self.build_group}", "Latest"
+        )
 
         if not parent_group_id or not group_id:
-            tty.warn(f"Failed to create or retrieve buildgroups for {self.build_group}")
+            msg = f"Failed to create or retrieve buildgroups for {self.build_group}"
+            tty.warn(msg)
             return
 
         data = {
@@ -300,12 +329,15 @@ def populate_buildgroup(self, job_names):
 
         enc_data = json.dumps(data).encode("utf-8")
 
-        request = Request(url, data=enc_data, headers=headers, method="PUT")
+        request = Request(url, data=enc_data, headers=headers)
+        request.get_method = lambda: "PUT"
 
-        try:
-            _urlopen(request, timeout=SPACK_CDASH_TIMEOUT)
-        except OSError as e:
-            tty.warn(f"Failed to populate CDash buildgroup: {e}")
+        response = opener.open(request, timeout=SPACK_CDASH_TIMEOUT)
+        response_code = response.getcode()
+
+        if response_code != 200:
+            msg = f"Error response code ({response_code}) in populate_buildgroup"
+            tty.warn(msg)
 
     def report_skipped(self, spec: spack.spec.Spec, report_dir: str, reason: Optional[str]):
         """Explicitly report skipping testing of a spec (e.g., it's CI
@@ -703,6 +735,9 @@ def _apply_section(dest, src):
                 for value in header.values():
                     value = os.path.expandvars(value)
 
+                verify_ssl = mapping.get("verify_ssl", spack.config.get("config:verify_ssl", True))
+                timeout = mapping.get("timeout", spack.config.get("config:connect_timeout", 1))
+
                 required = mapping.get("require", [])
                 allowed = mapping.get("allow", [])
                 ignored = mapping.get("ignore", [])
@@ -736,15 +771,19 @@ def job_query(job):
                         endpoint_url._replace(query=query).geturl(), headers=header, method="GET"
                     )
                     try:
-                        response = _urlopen(request)
-                        config = json.load(response)
+                        response = _dyn_mapping_urlopener(
+                            request, verify_ssl=verify_ssl, timeout=timeout
+                        )
                     except Exception as e:
                         # For now just ignore any errors from dynamic mapping and continue
                         # This is still experimental, and failures should not stop CI
                         # from running normally
-                        tty.warn(f"Failed to fetch dynamic mapping for query:\n\t{query}: {e}")
+                        tty.warn(f"Failed to fetch dynamic mapping for query:\n\t{query}")
+                        tty.warn(f"{e}")
                         continue
 
+                    config = json.load(codecs.getreader("utf-8")(response))
+
                     # Strip ignore keys
                     if ignored:
                         for key in ignored:

lib/spack/spack/cmd/arch.py

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import collections
-import warnings
 
 import archspec.cpu
 
@@ -52,10 +51,10 @@ def setup_parser(subparser):
         "-t", "--target", action="store_true", default=False, help="print only the target"
     )
     parts2.add_argument(
-        "-f", "--frontend", action="store_true", default=False, help="print frontend (DEPRECATED)"
+        "-f", "--frontend", action="store_true", default=False, help="print frontend"
     )
     parts2.add_argument(
-        "-b", "--backend", action="store_true", default=False, help="print backend (DEPRECATED)"
+        "-b", "--backend", action="store_true", default=False, help="print backend"
     )
 
 
@@ -99,14 +98,15 @@ def arch(parser, args):
         display_targets(archspec.cpu.TARGETS)
         return
 
+    os_args, target_args = "default_os", "default_target"
     if args.frontend:
-        warnings.warn("the argument --frontend is deprecated, and will be removed in Spack v1.0")
+        os_args, target_args = "frontend", "frontend"
     elif args.backend:
-        warnings.warn("the argument --backend is deprecated, and will be removed in Spack v1.0")
+        os_args, target_args = "backend", "backend"
 
     host_platform = spack.platforms.host()
-    host_os = host_platform.default_operating_system()
-    host_target = host_platform.default_target()
+    host_os = host_platform.operating_system(os_args)
+    host_target = host_platform.target(target_args)
     if args.family:
         host_target = host_target.family
     elif args.generic:

lib/spack/spack/cmd/bootstrap.py

@@ -1,7 +1,7 @@
 # Copyright Spack Project Developers. See COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import os
+import os.path
 import shutil
 import sys
 import tempfile
@@ -436,7 +436,6 @@ def write_metadata(subdir, metadata):
         shutil.copy(spack.util.path.canonicalize_path(GNUPG_JSON), abs_directory)
         shutil.copy(spack.util.path.canonicalize_path(PATCHELF_JSON), abs_directory)
         instructions += cmd.format("local-binaries", rel_directory)
-        instructions += "  % spack buildcache update-index <final-path>/bootstrap_cache\n"
     print(instructions)
 
 

lib/spack/spack/cmd/common/arguments.py

@@ -4,7 +4,7 @@
 
 
 import argparse
-import os
+import os.path
 import textwrap
 
 from llnl.util.lang import stable_partition

lib/spack/spack/cmd/containerize.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
+import os.path
 
 import llnl.util.tty
 

lib/spack/spack/cmd/debug.py

@@ -86,8 +86,8 @@ def create_db_tarball(args):
 
 def report(args):
     host_platform = spack.platforms.host()
-    host_os = host_platform.default_operating_system()
-    host_target = host_platform.default_target()
+    host_os = host_platform.operating_system("frontend")
+    host_target = host_platform.target("frontend")
     architecture = spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))
     print("* **Spack:**", spack.get_version())
     print("* **Python:**", platform.python_version())

lib/spack/spack/cmd/external.py

@@ -110,7 +110,10 @@ def external_find(args):
             # Note that KeyboardInterrupt does not subclass Exception
             # (so CTRL-C will terminate the program as expected).
             skip_msg = "Skipping manifest and continuing with other external checks"
-            if isinstance(e, OSError) and e.errno in (errno.EPERM, errno.EACCES):
+            if (isinstance(e, IOError) or isinstance(e, OSError)) and e.errno in [
+                errno.EPERM,
+                errno.EACCES,
+            ]:
                 # The manifest file does not have sufficient permissions enabled:
                 # print a warning and keep going
                 tty.warn("Unable to read manifest due to insufficient permissions.", skip_msg)

lib/spack/spack/cmd/help.py

@@ -54,6 +54,10 @@
       @m{target=target}                 specific <target> processor
       @m{arch=platform-os-target}       shortcut for all three above
 
+    cross-compiling:
+      @m{os=backend} or @m{os=be}           build for compute node (backend)
+      @m{os=frontend} or @m{os=fe}          build for login node (frontend)
+
     dependencies:
       ^dependency [constraints]     specify constraints on dependencies
       ^@K{/hash}                        build with a specific installed

lib/spack/spack/cmd/modules/__init__.py

@@ -5,7 +5,7 @@
 """Implementation details of the ``spack module`` command."""
 
 import collections
-import os
+import os.path
 import shutil
 import sys
 

lib/spack/spack/cmd/test.py

@@ -177,15 +177,16 @@ def test_run(args):
         matching = spack.store.STORE.db.query_local(spec, hashes=hashes, explicit=explicit)
         if spec and not matching:
             tty.warn("No {0}installed packages match spec {1}".format(explicit_str, spec))
+            """
+            TODO: Need to write out a log message and/or CDASH Testing
+              output that package not installed IF continue to process
+              these issues here.
 
-            # TODO: Need to write out a log message and/or CDASH Testing
-            #   output that package not installed IF continue to process
-            #   these issues here.
-
-            # if args.log_format:
-            #     # Proceed with the spec assuming the test process
-            #     # to ensure report package as skipped (e.g., for CI)
-            #     specs_to_test.append(spec)
+            if args.log_format:
+                # Proceed with the spec assuming the test process
+                # to ensure report package as skipped (e.g., for CI)
+                specs_to_test.append(spec)
+            """
 
         specs_to_test.extend(matching)
 

lib/spack/spack/cmd/tutorial.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import os
+import os.path
 import shutil
 
 import llnl.util.tty as tty

lib/spack/spack/cmd/unit_test.py

@@ -5,7 +5,7 @@
 import argparse
 import collections
 import io
-import os
+import os.path
 import re
 import sys
 

lib/spack/spack/compilers/__init__.py

@@ -801,17 +801,17 @@ def _extract_compiler_paths(spec: "spack.spec.Spec") -> Optional[Dict[str, str]]
     def _extract_os_and_target(spec: "spack.spec.Spec"):
         if not spec.architecture:
             host_platform = spack.platforms.host()
-            operating_system = host_platform.default_operating_system()
-            target = host_platform.default_target()
+            operating_system = host_platform.operating_system("default_os")
+            target = host_platform.target("default_target")
         else:
             target = spec.architecture.target
             if not target:
-                target = spack.platforms.host().default_target()
+                target = spack.platforms.host().target("default_target")
 
             operating_system = spec.os
             if not operating_system:
                 host_platform = spack.platforms.host()
-                operating_system = host_platform.default_operating_system()
+                operating_system = host_platform.operating_system("default_os")
         return operating_system, target
 
 

lib/spack/spack/config.py

@@ -53,7 +53,6 @@
 import spack.schema.definitions
 import spack.schema.develop
 import spack.schema.env
-import spack.schema.env_vars
 import spack.schema.mirrors
 import spack.schema.modules
 import spack.schema.packages
@@ -71,7 +70,6 @@
     "compilers": spack.schema.compilers.schema,
     "concretizer": spack.schema.concretizer.schema,
     "definitions": spack.schema.definitions.schema,
-    "env_vars": spack.schema.env_vars.schema,
     "view": spack.schema.view.schema,
     "develop": spack.schema.develop.schema,
     "mirrors": spack.schema.mirrors.schema,

lib/spack/spack/container/__init__.py

@@ -57,7 +57,7 @@ def validate(configuration_file):
     # Set the default value of the concretization strategy to unify and
     # warn if the user explicitly set another value
     env_dict.setdefault("concretizer", {"unify": True})
-    if env_dict["concretizer"]["unify"] is not True:
+    if not env_dict["concretizer"]["unify"] is True:
         warnings.warn(
             '"concretizer:unify" is not set to "true", which means the '
             "generated image may contain different variants of the same "

lib/spack/spack/container/images.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Manages the details on the images used in the various stages."""
 import json
-import os
+import os.path
 import shlex
 import sys
 

lib/spack/spack/database.py

@@ -263,6 +263,18 @@ def from_dict(cls, spec, dictionary):
         return InstallRecord(spec, **d)
 
 
+class ForbiddenLockError(SpackError):
+    """Raised when an upstream DB attempts to acquire a lock"""
+
+
+class ForbiddenLock:
+    def __getattr__(self, name):
+        raise ForbiddenLockError(f"Cannot access attribute '{name}' of lock")
+
+    def __reduce__(self):
+        return ForbiddenLock, tuple()
+
+
 class LockConfiguration(NamedTuple):
     """Data class to configure locks in Database objects
 
@@ -605,12 +617,16 @@ def __init__(
         self.db_lock_timeout = lock_cfg.database_timeout
         tty.debug(f"DATABASE LOCK TIMEOUT: {str(self.db_lock_timeout)}s")
 
-        self.lock = lk.Lock(
-            str(self._lock_path),
-            default_timeout=self.db_lock_timeout,
-            desc="database",
-            enable=not self.is_upstream and lock_cfg.enable,
-        )
+        self.lock: Union[ForbiddenLock, lk.Lock]
+        if self.is_upstream:
+            self.lock = ForbiddenLock()
+        else:
+            self.lock = lk.Lock(
+                str(self._lock_path),
+                default_timeout=self.db_lock_timeout,
+                desc="database",
+                enable=lock_cfg.enable,
+            )
         self._data: Dict[str, InstallRecord] = {}
 
         # For every installed spec we keep track of its install prefix, so that
@@ -1034,9 +1050,6 @@ def _write(self, type=None, value=None, traceback=None):
 
         This routine does no locking.
         """
-        if self.is_upstream:
-            raise UpstreamDatabaseLockingError("Cannot write to an upstream database")
-
         self._ensure_parent_directories()
 
         # Do not write if exceptions were raised
@@ -1657,24 +1670,38 @@ def query(
         """
         valid_trees = ["all", "upstream", "local", self.root] + [u.root for u in self.upstream_dbs]
         if install_tree not in valid_trees:
-            raise ValueError(
-                f"Invalid install_tree argument to Database.query(). Try one of {valid_trees}"
+            msg = "Invalid install_tree argument to Database.query()\n"
+            msg += f"Try one of {', '.join(valid_trees)}"
+            tty.error(msg)
+            return []
+
+        upstream_results = []
+        upstreams = self.upstream_dbs
+        if install_tree not in ("all", "upstream"):
+            upstreams = [u for u in self.upstream_dbs if u.root == install_tree]
+        for upstream_db in upstreams:
+            # queries for upstream DBs need to *not* lock - we may not
+            # have permissions to do this and the upstream DBs won't know about
+            # us anyway (so e.g. they should never uninstall specs)
+            upstream_results.extend(
+                upstream_db._query(
+                    query_spec,
+                    predicate_fn=predicate_fn,
+                    installed=installed,
+                    explicit=explicit,
+                    start_date=start_date,
+                    end_date=end_date,
+                    hashes=hashes,
+                    in_buildcache=in_buildcache,
+                    origin=origin,
+                )
+                or []
             )
 
-        if install_tree == "all":
-            databases = [self, *self.upstream_dbs]
-        elif install_tree == "upstream":
-            databases = self.upstream_dbs
-        elif install_tree == "local" or self.root == install_tree:
-            databases = [self]
-        else:
-            databases = [u for u in self.upstream_dbs if u.root == install_tree]
-
-        results: List[spack.spec.Spec] = []
-
-        for db in databases:
-            results.extend(
-                db.query_local(
+        local_results: Set["spack.spec.Spec"] = set()
+        if install_tree in ("all", "local") or self.root == install_tree:
+            local_results = set(
+                self.query_local(
                     query_spec,
                     predicate_fn=predicate_fn,
                     installed=installed,
@@ -1687,13 +1714,8 @@ def query(
                 )
             )
 
-        # Stable deduplication on dag hash picks local specs over upstreams.
-        if len(databases) > 1:
-            results = list(llnl.util.lang.dedupe(results, key=lambda x: x.dag_hash()))
-
-        # reduce number of comparisons with slow default __lt__
-        results.sort(key=lambda s: s.name)
-        results.sort()
+        results = list(local_results) + list(x for x in upstream_results if x not in local_results)
+        results.sort()  # type: ignore[call-overload]
         return results
 
     def query_one(

lib/spack/spack/detection/common.py

@@ -15,6 +15,7 @@
 import glob
 import itertools
 import os
+import os.path
 import pathlib
 import re
 import sys

lib/spack/spack/detection/path.py

@@ -7,6 +7,7 @@
 import collections
 import concurrent.futures
 import os
+import os.path
 import re
 import sys
 import traceback

lib/spack/spack/directives.py

@@ -32,7 +32,7 @@ class OpenMpi(Package):
 """
 import collections
 import collections.abc
-import os
+import os.path
 import re
 from typing import Any, Callable, List, Optional, Tuple, Type, Union
 

lib/spack/spack/environment/environment.py

@@ -581,7 +581,7 @@ def _error_on_nonempty_view_dir(new_root):
     # Check if the target path lexists
     try:
         st = os.lstat(new_root)
-    except OSError:
+    except (IOError, OSError):
         return
 
     # Empty directories are fine
@@ -861,7 +861,7 @@ def regenerate(self, concrete_roots: List[Spec]) -> None:
         ):
             try:
                 shutil.rmtree(old_root)
-            except OSError as e:
+            except (IOError, OSError) as e:
                 msg = "Failed to remove old view at %s\n" % old_root
                 msg += str(e)
                 tty.warn(msg)
@@ -2554,7 +2554,7 @@ def is_latest_format(manifest):
     try:
         with open(manifest, encoding="utf-8") as f:
             data = syaml.load(f)
-    except OSError:
+    except (OSError, IOError):
         return True
     top_level_key = _top_level_key(data)
     changed = spack.schema.env.update(data[top_level_key])
@@ -2634,32 +2634,6 @@ def _ensure_env_dir():
 
     shutil.copy(envfile, target_manifest)
 
-    # Copy relative path includes that live inside the environment dir
-    try:
-        manifest = EnvironmentManifestFile(environment_dir)
-    except Exception:
-        # error handling for bad manifests is handled on other code paths
-        return
-
-    includes = manifest[TOP_LEVEL_KEY].get("include", [])
-    for include in includes:
-        if os.path.isabs(include):
-            continue
-
-        abspath = pathlib.Path(os.path.normpath(environment_dir / include))
-        common_path = pathlib.Path(os.path.commonpath([environment_dir, abspath]))
-        if common_path != environment_dir:
-            tty.debug(f"Will not copy relative include from outside environment: {include}")
-            continue
-
-        orig_abspath = os.path.normpath(envfile.parent / include)
-        if not os.path.exists(orig_abspath):
-            tty.warn(f"Included file does not exist; will not copy: '{include}'")
-            continue
-
-        fs.touchp(abspath)
-        shutil.copy(orig_abspath, abspath)
-
 
 class EnvironmentManifestFile(collections.abc.Mapping):
     """Manages the in-memory representation of a manifest file, and its synchronization

lib/spack/spack/environment/shell.py

@@ -10,7 +10,6 @@
 
 import spack.environment as ev
 import spack.repo
-import spack.schema.environment
 import spack.store
 from spack.util.environment import EnvironmentModifications
 
@@ -157,11 +156,6 @@ def activate(
     # MANPATH, PYTHONPATH, etc. All variables that end in PATH (case-sensitive)
     # become PATH variables.
     #
-
-    env_vars_yaml = env.manifest.configuration.get("env_vars", None)
-    if env_vars_yaml:
-        env_mods.extend(spack.schema.environment.parse(env_vars_yaml))
-
     try:
         if view and env.has_view(view):
             with spack.store.STORE.db.read_transaction():
@@ -195,10 +189,6 @@ def deactivate() -> EnvironmentModifications:
     if active is None:
         return env_mods
 
-    env_vars_yaml = active.manifest.configuration.get("env_vars", None)
-    if env_vars_yaml:
-        env_mods.extend(spack.schema.environment.parse(env_vars_yaml).reversed())
-
     active_view = os.getenv(ev.spack_env_view_var)
 
     if active_view and active.has_view(active_view):

lib/spack/spack/extensions.py

@@ -187,7 +187,7 @@ def path_for_extension(target_name: str, *, paths: List[str]) -> str:
         if name == target_name:
             return path
     else:
-        raise OSError('extension "{0}" not found'.format(target_name))
+        raise IOError('extension "{0}" not found'.format(target_name))
 
 
 def get_module(cmd_name):

lib/spack/spack/fetch_strategy.py

@@ -25,6 +25,7 @@
 import functools
 import http.client
 import os
+import os.path
 import re
 import shutil
 import urllib.error
@@ -320,15 +321,9 @@ def _fetch_urllib(self, url):
 
         request = urllib.request.Request(url, headers={"User-Agent": web_util.SPACK_USER_AGENT})
 
-        if os.path.lexists(save_file):
-            os.remove(save_file)
-
         try:
             response = web_util.urlopen(request)
-            tty.msg(f"Fetching {url}")
-            with open(save_file, "wb") as f:
-                shutil.copyfileobj(response, f)
-        except OSError as e:
+        except (TimeoutError, urllib.error.URLError) as e:
             # clean up archive on failure.
             if self.archive_file:
                 os.remove(self.archive_file)
@@ -336,6 +331,14 @@ def _fetch_urllib(self, url):
                 os.remove(save_file)
             raise FailedDownloadError(e) from e
 
+        tty.msg(f"Fetching {url}")
+
+        if os.path.lexists(save_file):
+            os.remove(save_file)
+
+        with open(save_file, "wb") as f:
+            shutil.copyfileobj(response, f)
+
         # Save the redirected URL for error messages. Sometimes we're redirected to an arbitrary
         # mirror that is broken, leading to spurious download failures. In that case it's helpful
         # for users to know which URL was actually fetched.
@@ -532,16 +535,11 @@ def __init__(self, *, url: str, checksum: Optional[str] = None, **kwargs):
     @_needs_stage
     def fetch(self):
         file = self.stage.save_filename
-
-        if os.path.lexists(file):
-            os.remove(file)
+        tty.msg(f"Fetching {self.url}")
 
         try:
             response = self._urlopen(self.url)
-            tty.msg(f"Fetching {self.url}")
-            with open(file, "wb") as f:
-                shutil.copyfileobj(response, f)
-        except OSError as e:
+        except (TimeoutError, urllib.error.URLError) as e:
             # clean up archive on failure.
             if self.archive_file:
                 os.remove(self.archive_file)
@@ -549,6 +547,12 @@ def fetch(self):
                 os.remove(file)
             raise FailedDownloadError(e) from e
 
+        if os.path.lexists(file):
+            os.remove(file)
+
+        with open(file, "wb") as f:
+            shutil.copyfileobj(response, f)
+
 
 class VCSFetchStrategy(FetchStrategy):
     """Superclass for version control system fetch strategies.

lib/spack/spack/filesystem_view.py

@@ -427,7 +427,7 @@ def needs_file(spec, file):
             try:
                 with open(manifest_file, "r", encoding="utf-8") as f:
                     manifest = s_json.load(f)
-            except OSError:
+            except (OSError, IOError):
                 # if we can't load it, assume it doesn't know about the file.
                 manifest = {}
             return test_path in manifest
@@ -831,7 +831,7 @@ def get_spec_from_file(filename):
     try:
         with open(filename, "r", encoding="utf-8") as f:
             return spack.spec.Spec.from_yaml(f)
-    except OSError:
+    except IOError:
         return None
 
 

lib/spack/spack/hooks/absolutify_elf_sonames.py

@@ -26,7 +26,7 @@ def is_shared_library_elf(filepath):
         with open(filepath, "rb") as f:
             elf = parse_elf(f, interpreter=True, dynamic_section=True)
             return elf.has_pt_dynamic and (elf.has_soname or not elf.has_pt_interp)
-    except (OSError, ElfParsingError):
+    except (IOError, OSError, ElfParsingError):
         return False
 
 

lib/spack/spack/hooks/sbang.py

@@ -166,7 +166,7 @@ def filter_shebangs_in_directory(directory, filenames=None):
         # Only look at executable, non-symlink files.
         try:
             st = os.lstat(path)
-        except OSError:
+        except (IOError, OSError):
             continue
 
         if stat.S_ISLNK(st.st_mode) or stat.S_ISDIR(st.st_mode) or not st.st_mode & is_exe:

lib/spack/spack/installer.py

@@ -814,7 +814,7 @@ def get_depflags(self, pkg: "spack.package_base.PackageBase") -> int:
         # Include build dependencies if pkg is going to be built from sources, or
         # if build deps are explicitly requested.
         if include_build_deps or not (
-            cache_only or pkg.spec.installed and pkg.spec.dag_hash() not in self.overwrite
+            cache_only or pkg.spec.installed and not pkg.spec.dag_hash() in self.overwrite
         ):
             depflag |= dt.BUILD
         if self.run_tests(pkg):

lib/spack/spack/main.py

@@ -14,6 +14,7 @@
 import io
 import operator
 import os
+import os.path
 import pstats
 import re
 import shlex
@@ -163,7 +164,7 @@ def format_help_sections(self, level):
         # lazily add all commands to the parser when needed.
         add_all_commands(self)
 
-        # Print help on subcommands in neatly formatted sections.
+        """Print help on subcommands in neatly formatted sections."""
         formatter = self._get_formatter()
 
         # Create a list of subcommand actions. Argparse internals are nasty!
@@ -728,7 +729,7 @@ def _compatible_sys_types():
     with the current host.
     """
     host_platform = spack.platforms.host()
-    host_os = str(host_platform.default_operating_system())
+    host_os = str(host_platform.operating_system("default_os"))
     host_target = archspec.cpu.host()
     compatible_targets = [host_target] + host_target.ancestors
 

lib/spack/spack/mirrors/layout.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
+import os.path
 from typing import Optional
 
 import llnl.url

lib/spack/spack/mirrors/mirror.py

@@ -64,7 +64,7 @@ def from_local_path(path: str):
     @staticmethod
     def from_url(url: str):
         """Create an anonymous mirror by URL. This method validates the URL."""
-        if urllib.parse.urlparse(url).scheme not in supported_url_schemes:
+        if not urllib.parse.urlparse(url).scheme in supported_url_schemes:
             raise ValueError(
                 f'"{url}" is not a valid mirror URL. '
                 f"Scheme must be one of {supported_url_schemes}."

lib/spack/spack/mirrors/utils.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
+import os.path
 import traceback
 
 import llnl.util.tty as tty

lib/spack/spack/modules/common.py

@@ -31,7 +31,7 @@
 import copy
 import datetime
 import inspect
-import os
+import os.path
 import re
 import string
 from typing import List, Optional

lib/spack/spack/modules/lmod.py

@@ -4,7 +4,7 @@
 
 import collections
 import itertools
-import os
+import os.path
 from typing import Dict, List, Optional, Tuple
 
 import llnl.util.filesystem as fs

lib/spack/spack/modules/tcl.py

@@ -5,7 +5,7 @@
 """This module implements the classes necessary to generate Tcl
 non-hierarchical modules.
 """
-import os
+import os.path
 from typing import Dict, Optional, Tuple
 
 import spack.config

lib/spack/spack/oci/opener.py

@@ -7,7 +7,6 @@
 import base64
 import json
 import re
-import socket
 import time
 import urllib.error
 import urllib.parse
@@ -383,7 +382,6 @@ def create_opener():
     """Create an opener that can handle OCI authentication."""
     opener = urllib.request.OpenerDirector()
     for handler in [
-        urllib.request.ProxyHandler(),
         urllib.request.UnknownHandler(),
         urllib.request.HTTPSHandler(context=spack.util.web.ssl_create_default_context()),
         spack.util.web.SpackHTTPDefaultErrorHandler(),
@@ -412,7 +410,7 @@ def wrapper(*args, **kwargs):
         for i in range(retries):
             try:
                 return f(*args, **kwargs)
-            except OSError as e:
+            except (urllib.error.URLError, TimeoutError) as e:
                 # Retry on internal server errors, and rate limit errors
                 # Potentially this could take into account the Retry-After header
                 # if registries support it
@@ -422,10 +420,9 @@ def wrapper(*args, **kwargs):
                         and (500 <= e.code < 600 or e.code == 429)
                     )
                     or (
-                        isinstance(e, urllib.error.URLError)
-                        and isinstance(e.reason, socket.timeout)
+                        isinstance(e, urllib.error.URLError) and isinstance(e.reason, TimeoutError)
                     )
-                    or isinstance(e, socket.timeout)
+                    or isinstance(e, TimeoutError)
                 ):
                     # Exponential backoff
                     sleep(2**i)

lib/spack/spack/package.py

@@ -2,64 +2,31 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-# flake8: noqa: F401, E402
-"""spack.package defines the public API for Spack packages, by re-exporting useful symbols from
-other modules. Packages should import this module, instead of importing from spack.* directly
-to ensure forward compatibility with future versions of Spack."""
+# flake8: noqa: F401
+"""spack.util.package is a set of useful build tools and directives for packages.
 
+Everything in this module is automatically imported into Spack package files.
+"""
 from os import chdir, environ, getcwd, makedirs, mkdir, remove, removedirs
 from shutil import move, rmtree
 
+from spack.error import InstallError, NoHeadersError, NoLibrariesError
+
+# Emulate some shell commands for convenience
+env = environ
+cd = chdir
+pwd = getcwd
+
 # import most common types used in packages
 from typing import Dict, List, Optional
 
-
-class tty:
-    import llnl.util.tty as _tty
-
-    debug = _tty.debug
-    error = _tty.error
-    info = _tty.info
-    msg = _tty.msg
-    warn = _tty.warn
-
-
-from llnl.util.filesystem import (
-    FileFilter,
-    FileList,
-    HeaderList,
-    LibraryList,
-    ancestor,
-    can_access,
-    change_sed_delimiter,
-    copy,
-    copy_tree,
-    filter_file,
-    find,
-    find_all_headers,
-    find_first,
-    find_headers,
-    find_libraries,
-    find_system_libraries,
-    force_remove,
-    force_symlink,
-    install,
-    install_tree,
-    is_exe,
-    join_path,
-    keep_modification_time,
-    library_extensions,
-    mkdirp,
-    remove_directory_contents,
-    remove_linked_tree,
-    rename,
-    set_executable,
-    set_install_permissions,
-    touch,
-    working_dir,
-)
+import llnl.util.filesystem
+from llnl.util.filesystem import *
 from llnl.util.symlink import symlink
 
+import spack.util.executable
+
+# These props will be overridden when the build env is set up.
 from spack.build_environment import MakeExecutable
 from spack.build_systems.aspell_dict import AspellDictPackage
 from spack.build_systems.autotools import AutotoolsPackage
@@ -109,24 +76,7 @@ class tty:
 from spack.builder import BaseBuilder
 from spack.config import determine_number_of_jobs
 from spack.deptypes import ALL_TYPES as all_deptypes
-from spack.directives import (
-    build_system,
-    can_splice,
-    conditional,
-    conflicts,
-    depends_on,
-    extends,
-    license,
-    maintainers,
-    patch,
-    provides,
-    redistribute,
-    requires,
-    resource,
-    variant,
-    version,
-)
-from spack.error import InstallError, NoHeadersError, NoLibrariesError
+from spack.directives import *
 from spack.install_test import (
     SkipTest,
     cache_extra_test_sources,
@@ -136,28 +86,26 @@ class tty:
     install_test_root,
     test_part,
 )
+from spack.installer import ExternalPackageError, InstallLockError, UpstreamPackageError
 from spack.mixins import filter_compiler_wrappers
 from spack.multimethod import default_args, when
-from spack.package_base import build_system_flags, env_flags, inject_flags, on_package_attributes
-from spack.package_completions import (
-    bash_completion_path,
-    fish_completion_path,
-    zsh_completion_path,
+from spack.package_base import (
+    DependencyConflictError,
+    build_system_flags,
+    env_flags,
+    flatten_dependencies,
+    inject_flags,
+    install_dependency_symlinks,
+    on_package_attributes,
 )
+from spack.package_completions import *
 from spack.phase_callbacks import run_after, run_before
-from spack.spec import Spec
-from spack.util.environment import EnvironmentModifications
-from spack.util.executable import Executable, ProcessError, which, which_string
+from spack.spec import InvalidSpecDetected, Spec
+from spack.util.executable import *
 from spack.util.filesystem import fix_darwin_install_name
-from spack.util.prefix import Prefix
 from spack.variant import any_combination_of, auto_or_any_combination_of, disjoint_sets
 from spack.version import Version, ver
 
-# Emulate some shell commands for convenience
-env = environ
-cd = chdir
-pwd = getcwd
-
 # These are just here for editor support; they may be set when the build env is set up.
 configure: Executable
 make_jobs: int

lib/spack/spack/package_base.py

@@ -30,6 +30,7 @@
 import llnl.util.filesystem as fsys
 import llnl.util.tty as tty
 from llnl.util.lang import classproperty, memoized
+from llnl.util.link_tree import LinkTree
 
 import spack.compilers
 import spack.config
@@ -66,6 +67,10 @@
 ]
 FLAG_HANDLER_TYPE = Callable[[str, Iterable[str]], FLAG_HANDLER_RETURN_TYPE]
 
+"""Allowed URL schemes for spack packages."""
+_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
+
+
 #: Filename for the Spack build/install log.
 _spack_build_logfile = "spack-build-out.txt"
 
@@ -697,6 +702,9 @@ class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta):
     #: Verbosity level, preserved across installs.
     _verbose = None
 
+    #: index of patches by sha256 sum, built lazily
+    _patches_by_hash = None
+
     #: Package homepage where users can find more information about the package
     homepage: Optional[str] = None
 
@@ -2284,6 +2292,19 @@ def rpath_args(self):
 build_system_flags = PackageBase.build_system_flags
 
 
+def install_dependency_symlinks(pkg, spec, prefix):
+    """
+    Execute a dummy install and flatten dependencies.
+
+    This routine can be used in a ``package.py`` definition by setting
+    ``install = install_dependency_symlinks``.
+
+    This feature comes in handy for creating a common location for the
+    the installation of third-party libraries.
+    """
+    flatten_dependencies(spec, prefix)
+
+
 def use_cray_compiler_names():
     """Compiler names for builds that rely on cray compiler names."""
     os.environ["CC"] = "cc"
@@ -2292,6 +2313,23 @@ def use_cray_compiler_names():
     os.environ["F77"] = "ftn"
 
 
+def flatten_dependencies(spec, flat_dir):
+    """Make each dependency of spec present in dir via symlink."""
+    for dep in spec.traverse(root=False):
+        name = dep.name
+
+        dep_path = spack.store.STORE.layout.path_for_spec(dep)
+        dep_files = LinkTree(dep_path)
+
+        os.mkdir(flat_dir + "/" + name)
+
+        conflict = dep_files.find_conflict(flat_dir + "/" + name)
+        if conflict:
+            raise DependencyConflictError(conflict)
+
+        dep_files.merge(flat_dir + "/" + name)
+
+
 def possible_dependencies(
     *pkg_or_spec: Union[str, spack.spec.Spec, typing.Type[PackageBase]],
     transitive: bool = True,

lib/spack/spack/patch.py

@@ -4,6 +4,7 @@
 
 import hashlib
 import os
+import os.path
 import pathlib
 import sys
 from typing import Any, Dict, Optional, Tuple, Type, Union

lib/spack/spack/platforms/__init__.py

@@ -52,7 +52,8 @@ def use_platform(new_platform):
 
     import spack.config
 
-    assert isinstance(new_platform, Platform), f'"{new_platform}" must be an instance of Platform'
+    msg = '"{0}" must be an instance of Platform'
+    assert isinstance(new_platform, Platform), msg.format(new_platform)
 
     original_host_fn = host
 

lib/spack/spack/platforms/_platform.py

@@ -1,22 +1,42 @@
 # Copyright Spack Project Developers. See COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import warnings
 from typing import Optional
 
 import archspec.cpu
 
 import llnl.util.lang
 
+import spack.error
+
+
+class NoPlatformError(spack.error.SpackError):
+    def __init__(self):
+        msg = "Could not determine a platform for this machine"
+        super().__init__(msg)
+
 
 @llnl.util.lang.lazy_lexicographic_ordering
 class Platform:
     """Platform is an abstract class extended by subclasses.
 
+    To add a new type of platform (such as cray_xe), create a subclass and set all the
+    class attributes such as priority, front_target, back_target, front_os, back_os.
+
     Platform also contain a priority class attribute. A lower number signifies higher
     priority. These numbers are arbitrarily set and can be changed though often there
     isn't much need unless a new platform is added and the user wants that to be
     detected first.
+
+    Targets are created inside the platform subclasses. Most architecture (like linux,
+    and darwin) will have only one target family (x86_64) but in the case of Cray
+    machines, there is both a frontend and backend processor. The user can specify
+    which targets are present on front-end and back-end architecture.
+
+    Depending on the platform, operating systems are either autodetected or are
+    set. The user can set the frontend and backend operating setting by the class
+    attributes front_os and back_os. The operating system will be responsible for
+    compiler detection.
     """
 
     # Subclass sets number. Controls detection order
@@ -25,72 +45,82 @@ class Platform:
     #: binary formats used on this platform; used by relocation logic
     binary_formats = ["elf"]
 
-    default: str
-    default_os: str
+    front_end: Optional[str] = None
+    back_end: Optional[str] = None
+    default: Optional[str] = None  # The default back end target.
+
+    front_os: Optional[str] = None
+    back_os: Optional[str] = None
+    default_os: Optional[str] = None
 
     reserved_targets = ["default_target", "frontend", "fe", "backend", "be"]
     reserved_oss = ["default_os", "frontend", "fe", "backend", "be"]
-    deprecated_names = ["frontend", "fe", "backend", "be"]
 
     def __init__(self, name):
         self.targets = {}
         self.operating_sys = {}
         self.name = name
-        self._init_targets()
 
     def add_target(self, name: str, target: archspec.cpu.Microarchitecture) -> None:
+        """Used by the platform specific subclass to list available targets.
+        Raises an error if the platform specifies a name
+        that is reserved by spack as an alias.
+        """
         if name in Platform.reserved_targets:
-            msg = f"{name} is a spack reserved alias and cannot be the name of a target"
-            raise ValueError(msg)
+            msg = "{0} is a spack reserved alias and cannot be the name of a target"
+            raise ValueError(msg.format(name))
         self.targets[name] = target
 
-    def _init_targets(self):
-        self.default = archspec.cpu.host().name
+    def _add_archspec_targets(self):
         for name, microarchitecture in archspec.cpu.TARGETS.items():
             self.add_target(name, microarchitecture)
 
     def target(self, name):
+        """This is a getter method for the target dictionary
+        that handles defaulting based on the values provided by default,
+        front-end, and back-end. This can be overwritten
+        by a subclass for which we want to provide further aliasing options.
+        """
+        # TODO: Check if we can avoid using strings here
         name = str(name)
-        if name in Platform.deprecated_names:
-            warnings.warn(f"target={name} is deprecated, use target={self.default} instead")
-
-        if name in Platform.reserved_targets:
+        if name == "default_target":
             name = self.default
+        elif name == "frontend" or name == "fe":
+            name = self.front_end
+        elif name == "backend" or name == "be":
+            name = self.back_end
 
         return self.targets.get(name, None)
 
     def add_operating_system(self, name, os_class):
-        if name in Platform.reserved_oss + Platform.deprecated_names:
-            msg = f"{name} is a spack reserved alias and cannot be the name of an OS"
-            raise ValueError(msg)
+        """Add the operating_system class object into the
+        platform.operating_sys dictionary.
+        """
+        if name in Platform.reserved_oss:
+            msg = "{0} is a spack reserved alias and cannot be the name of an OS"
+            raise ValueError(msg.format(name))
         self.operating_sys[name] = os_class
 
-    def default_target(self):
-        return self.target(self.default)
-
-    def default_operating_system(self):
-        return self.operating_system(self.default_os)
-
     def operating_system(self, name):
-        if name in Platform.deprecated_names:
-            warnings.warn(f"os={name} is deprecated, use os={self.default_os} instead")
-
-        if name in Platform.reserved_oss:
+        if name == "default_os":
             name = self.default_os
+        if name == "frontend" or name == "fe":
+            name = self.front_os
+        if name == "backend" or name == "be":
+            name = self.back_os
 
         return self.operating_sys.get(name, None)
 
     def setup_platform_environment(self, pkg, env):
-        """Platform-specific build environment modifications.
-
-        This method is meant toi be overridden by subclasses, when needed.
+        """Subclass can override this method if it requires any
+        platform-specific build environment modifications.
         """
         pass
 
     @classmethod
     def detect(cls):
-        """Returns True if the host platform is detected to be the current Platform class,
-        False otherwise.
+        """Return True if the the host platform is detected to be the current
+        Platform class, False otherwise.
 
         Derived classes are responsible for implementing this method.
         """
@@ -105,7 +135,11 @@ def __str__(self):
     def _cmp_iter(self):
         yield self.name
         yield self.default
+        yield self.front_end
+        yield self.back_end
         yield self.default_os
+        yield self.front_os
+        yield self.back_os
 
         def targets():
             for t in sorted(self.targets.values()):

lib/spack/spack/platforms/cray.py

@@ -1,7 +1,7 @@
 # Copyright Spack Project Developers. See COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import os
+import os.path
 
 
 def slingshot_network():

lib/spack/spack/platforms/darwin.py

@@ -4,6 +4,8 @@
 
 import platform as py_platform
 
+import archspec.cpu
+
 from spack.operating_systems.mac_os import MacOs
 from spack.version import Version
 
@@ -17,8 +19,18 @@ class Darwin(Platform):
 
     def __init__(self):
         super().__init__("darwin")
+        self._add_archspec_targets()
+
+        self.default = archspec.cpu.host().name
+        self.front_end = self.default
+        self.back_end = self.default
+
         mac_os = MacOs()
+
         self.default_os = str(mac_os)
+        self.front_os = str(mac_os)
+        self.back_os = str(mac_os)
+
         self.add_operating_system(str(mac_os), mac_os)
 
     @classmethod

lib/spack/spack/platforms/freebsd.py

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import platform
 
+import archspec.cpu
+
 from spack.operating_systems.freebsd import FreeBSDOs
 
 from ._platform import Platform
@@ -13,8 +15,18 @@ class FreeBSD(Platform):
 
     def __init__(self):
         super().__init__("freebsd")
+
+        self._add_archspec_targets()
+
+        # Get specific default
+        self.default = archspec.cpu.host().name
+        self.front_end = self.default
+        self.back_end = self.default
+
         os = FreeBSDOs()
         self.default_os = str(os)
+        self.front_os = self.default_os
+        self.back_os = self.default_os
         self.add_operating_system(str(os), os)
 
     @classmethod

lib/spack/spack/platforms/linux.py

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import platform
 
+import archspec.cpu
+
 from spack.operating_systems.linux_distro import LinuxDistro
 
 from ._platform import Platform
@@ -13,8 +15,18 @@ class Linux(Platform):
 
     def __init__(self):
         super().__init__("linux")
+
+        self._add_archspec_targets()
+
+        # Get specific default
+        self.default = archspec.cpu.host().name
+        self.front_end = self.default
+        self.back_end = self.default
+
         linux_dist = LinuxDistro()
         self.default_os = str(linux_dist)
+        self.front_os = self.default_os
+        self.back_os = self.default_os
         self.add_operating_system(str(linux_dist), linux_dist)
 
     @classmethod

lib/spack/spack/platforms/test.py

@@ -16,19 +16,31 @@ class Test(Platform):
     if platform.system().lower() == "darwin":
         binary_formats = ["macho"]
 
+    if platform.machine() == "arm64":
+        front_end = "aarch64"
+        back_end = "m1"
+        default = "m1"
+    else:
+        front_end = "x86_64"
+        back_end = "core2"
+        default = "core2"
+
+    front_os = "redhat6"
+    back_os = "debian6"
     default_os = "debian6"
-    default = "m1" if platform.machine() == "arm64" else "core2"
 
     def __init__(self, name=None):
         name = name or "test"
         super().__init__(name)
-        self.add_operating_system("debian6", spack.operating_systems.OperatingSystem("debian", 6))
-        self.add_operating_system("redhat6", spack.operating_systems.OperatingSystem("redhat", 6))
+        self.add_target(self.default, archspec.cpu.TARGETS[self.default])
+        self.add_target(self.front_end, archspec.cpu.TARGETS[self.front_end])
 
-    def _init_targets(self):
-        targets = ("aarch64", "m1") if platform.machine() == "arm64" else ("x86_64", "core2")
-        for t in targets:
-            self.add_target(t, archspec.cpu.TARGETS[t])
+        self.add_operating_system(
+            self.default_os, spack.operating_systems.OperatingSystem("debian", 6)
+        )
+        self.add_operating_system(
+            self.front_os, spack.operating_systems.OperatingSystem("redhat", 6)
+        )
 
     @classmethod
     def detect(cls):

lib/spack/spack/platforms/windows.py

@@ -4,6 +4,8 @@
 
 import platform
 
+import archspec.cpu
+
 from spack.operating_systems.windows_os import WindowsOs
 
 from ._platform import Platform
@@ -14,8 +16,18 @@ class Windows(Platform):
 
     def __init__(self):
         super().__init__("windows")
+        self._add_archspec_targets()
+
+        self.default = archspec.cpu.host().name
+        self.front_end = self.default
+        self.back_end = self.default
+
         windows_os = WindowsOs()
+
         self.default_os = str(windows_os)
+        self.front_os = str(windows_os)
+        self.back_os = str(windows_os)
+
         self.add_operating_system(str(windows_os), windows_os)
 
     @classmethod

lib/spack/spack/repo.py

@@ -14,6 +14,7 @@
 import inspect
 import itertools
 import os
+import os.path
 import random
 import re
 import shutil
@@ -1041,7 +1042,7 @@ def _read_config(self) -> Dict[str, str]:
 
                 return yaml_data["repo"]
 
-        except OSError:
+        except IOError:
             tty.die(f"Error reading {self.config_file} when opening {self.root}")
 
     def get(self, spec: "spack.spec.Spec") -> "spack.package_base.PackageBase":
@@ -1369,7 +1370,7 @@ def create_repo(root, namespace=None, subdir=packages_dir_name):
             if subdir != packages_dir_name:
                 config.write(f"  subdirectory: '{subdir}'\n")
 
-    except OSError as e:
+    except (IOError, OSError) as e:
         # try to clean up.
         if existed:
             shutil.rmtree(config_path, ignore_errors=True)

lib/spack/spack/reporters/cdash.py

@@ -1,10 +1,9 @@
 # Copyright Spack Project Developers. See COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import codecs
 import collections
 import hashlib
-import os
+import os.path
 import platform
 import posixpath
 import re
@@ -14,7 +13,7 @@
 import xml.sax.saxutils
 from typing import Dict, Optional
 from urllib.parse import urlencode
-from urllib.request import Request
+from urllib.request import HTTPSHandler, Request, build_opener
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import working_dir
@@ -25,10 +24,10 @@
 import spack.spec
 import spack.tengine
 import spack.util.git
-import spack.util.web as web_util
 from spack.error import SpackError
 from spack.util.crypto import checksum
 from spack.util.log_parse import parse_log_events
+from spack.util.web import ssl_create_default_context
 
 from .base import Reporter
 from .extract import extract_test_parts
@@ -107,7 +106,7 @@ def __init__(self, configuration: CDashConfiguration):
         self.site = configuration.site or socket.gethostname()
         self.osname = platform.system()
         self.osrelease = platform.release()
-        self.target = spack.platforms.host().default_target()
+        self.target = spack.platforms.host().target("default_target")
         self.starttime = int(time.time())
         self.endtime = self.starttime
         self.buildstamp = (
@@ -434,6 +433,7 @@ def upload(self, filename):
         # Compute md5 checksum for the contents of this file.
         md5sum = checksum(hashlib.md5, filename, block_size=8192)
 
+        opener = build_opener(HTTPSHandler(context=ssl_create_default_context()))
         with open(filename, "rb") as f:
             params_dict = {
                 "build": self.buildname,
@@ -443,21 +443,26 @@ def upload(self, filename):
             }
             encoded_params = urlencode(params_dict)
             url = "{0}&{1}".format(self.cdash_upload_url, encoded_params)
-            request = Request(url, data=f, method="PUT")
+            request = Request(url, data=f)
             request.add_header("Content-Type", "text/xml")
             request.add_header("Content-Length", os.path.getsize(filename))
             if self.authtoken:
                 request.add_header("Authorization", "Bearer {0}".format(self.authtoken))
             try:
-                response = web_util.urlopen(request, timeout=SPACK_CDASH_TIMEOUT)
+                # By default, urllib2 only support GET and POST.
+                # CDash expects this file to be uploaded via PUT.
+                request.get_method = lambda: "PUT"
+                response = opener.open(request, timeout=SPACK_CDASH_TIMEOUT)
                 if self.current_package_name not in self.buildIds:
-                    resp_value = codecs.getreader("utf-8")(response).read()
+                    resp_value = response.read()
+                    if isinstance(resp_value, bytes):
+                        resp_value = resp_value.decode("utf-8")
                     match = self.buildid_regexp.search(resp_value)
                     if match:
                         buildid = match.group(1)
                         self.buildIds[self.current_package_name] = buildid
             except Exception as e:
-                print(f"Upload to CDash failed: {e}")
+                print("Upload to CDash failed: {0}".format(e))
 
     def finalize_report(self):
         if self.buildIds:

lib/spack/spack/reporters/junit.py

@@ -1,7 +1,7 @@
 # Copyright Spack Project Developers. See COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import os
+import os.path
 
 import spack.tengine
 

lib/spack/spack/rewiring.py

@@ -3,11 +3,15 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import os
+import shutil
 import tempfile
 
 import spack.binary_distribution as bindist
+import spack.deptypes as dt
 import spack.error
 import spack.hooks
+import spack.platforms
+import spack.relocate as relocate
 import spack.store
 
 
@@ -38,12 +42,64 @@ def rewire_node(spec, explicit):
 
     spack.hooks.pre_install(spec)
     bindist.extract_buildcache_tarball(tarball, destination=spec.prefix)
-    bindist.relocate_package(spec)
+    buildinfo = bindist.read_buildinfo_file(spec.prefix)
 
-    # run post install hooks and add to db
-    spack.hooks.post_install(spec, explicit)
+    # compute prefix-to-prefix for every node from the build spec to the spliced
+    # spec
+    prefix_to_prefix = {spec.build_spec.prefix: spec.prefix}
+    build_spec_ids = set(id(s) for s in spec.build_spec.traverse(deptype=dt.ALL & ~dt.BUILD))
+    for s in bindist.specs_to_relocate(spec):
+        analog = s
+        if id(s) not in build_spec_ids:
+            analogs = [
+                d
+                for d in spec.build_spec.traverse(deptype=dt.ALL & ~dt.BUILD)
+                if s._splice_match(d, self_root=spec, other_root=spec.build_spec)
+            ]
+            if analogs:
+                # Prefer same-name analogs and prefer higher versions
+                # This matches the preferences in Spec.splice, so we will find same node
+                analog = max(analogs, key=lambda a: (a.name == s.name, a.version))
+
+        prefix_to_prefix[analog.prefix] = s.prefix
+
+    platform = spack.platforms.by_name(spec.platform)
+
+    text_to_relocate = [
+        os.path.join(spec.prefix, rel_path) for rel_path in buildinfo["relocate_textfiles"]
+    ]
+    if text_to_relocate:
+        relocate.relocate_text(files=text_to_relocate, prefix_to_prefix=prefix_to_prefix)
+    links = [os.path.join(spec.prefix, f) for f in buildinfo["relocate_links"]]
+    relocate.relocate_links(links, prefix_to_prefix)
+    bins_to_relocate = [
+        os.path.join(spec.prefix, rel_path) for rel_path in buildinfo["relocate_binaries"]
+    ]
+    if bins_to_relocate:
+        if "macho" in platform.binary_formats:
+            relocate.relocate_macho_binaries(bins_to_relocate, prefix_to_prefix)
+        if "elf" in platform.binary_formats:
+            relocate.relocate_elf_binaries(bins_to_relocate, prefix_to_prefix)
+        relocate.relocate_text_bin(binaries=bins_to_relocate, prefix_to_prefix=prefix_to_prefix)
+    shutil.rmtree(tempdir)
+    install_manifest = os.path.join(
+        spec.prefix,
+        spack.store.STORE.layout.metadata_dir,
+        spack.store.STORE.layout.manifest_file_name,
+    )
+    try:
+        os.unlink(install_manifest)
+    except FileNotFoundError:
+        pass
+    # Write the spliced spec into spec.json. Without this, Database.add would fail because it
+    # checks the spec.json in the prefix against the spec being added to look for mismatches
+    spack.store.STORE.layout.write_spec(spec, spack.store.STORE.layout.spec_file_path(spec))
+    # add to database, not sure about explicit
     spack.store.STORE.db.add(spec, explicit=explicit)
 
+    # run post install hooks
+    spack.hooks.post_install(spec, explicit)
+
 
 class RewireError(spack.error.SpackError):
     """Raised when something goes wrong with rewiring."""

lib/spack/spack/schema/env_vars.py

@@ -1,22 +0,0 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-"""Schema for env_vars.yaml configuration file.
-
-.. literalinclude:: _spack_root/lib/spack/spack/schema/env_vars.py
-   :lines: 15-
-"""
-from typing import Any, Dict
-
-import spack.schema.environment
-
-properties: Dict[str, Any] = {"env_vars": spack.schema.environment.definition}
-
-#: Full schema with metadata
-schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "Spack env_vars configuration file schema",
-    "type": "object",
-    "additionalProperties": False,
-    "properties": properties,
-}

lib/spack/spack/schema/merged.py

@@ -20,7 +20,6 @@
 import spack.schema.container
 import spack.schema.definitions
 import spack.schema.develop
-import spack.schema.env_vars
 import spack.schema.mirrors
 import spack.schema.modules
 import spack.schema.packages
@@ -39,7 +38,6 @@
     spack.schema.ci.properties,
     spack.schema.definitions.properties,
     spack.schema.develop.properties,
-    spack.schema.env_vars.properties,
     spack.schema.mirrors.properties,
     spack.schema.modules.properties,
     spack.schema.packages.properties,

lib/spack/spack/spec.py

@@ -238,14 +238,23 @@ class ArchSpec:
     """Aggregate the target platform, the operating system and the target microarchitecture."""
 
     @staticmethod
-    def default_arch():
-        """Return the default architecture"""
+    def _return_arch(os_tag, target_tag):
         platform = spack.platforms.host()
-        default_os = platform.default_operating_system()
-        default_target = platform.default_target()
+        default_os = platform.operating_system(os_tag)
+        default_target = platform.target(target_tag)
         arch_tuple = str(platform), str(default_os), str(default_target)
         return ArchSpec(arch_tuple)
 
+    @staticmethod
+    def default_arch():
+        """Return the default architecture"""
+        return ArchSpec._return_arch("default_os", "default_target")
+
+    @staticmethod
+    def frontend_arch():
+        """Return the frontend architecture"""
+        return ArchSpec._return_arch("frontend", "frontend")
+
     __slots__ = "_platform", "_os", "_target"
 
     def __init__(self, spec_or_platform_tuple=(None, None, None)):
@@ -452,9 +461,6 @@ def _target_satisfies(self, other: "ArchSpec", strict: bool) -> bool:
         return bool(self._target_intersection(other))
 
     def _target_constrain(self, other: "ArchSpec") -> bool:
-        if self.target is None and other.target is None:
-            return False
-
         if not other._target_satisfies(self, strict=False):
             raise UnsatisfiableArchitectureSpecError(self, other)
 
@@ -503,56 +509,21 @@ def _target_intersection(self, other):
                     if (not s_min or o_comp >= s_min) and (not s_max or o_comp <= s_max):
                         results.append(o_min)
                 else:
-                    # Take the "min" of the two max, if there is a partial ordering.
-                    n_max = ""
-                    if s_max and o_max:
-                        _s_max = _make_microarchitecture(s_max)
-                        _o_max = _make_microarchitecture(o_max)
-                        if _s_max.family != _o_max.family:
-                            continue
-                        if _s_max <= _o_max:
-                            n_max = s_max
-                        elif _o_max < _s_max:
-                            n_max = o_max
-                        else:
-                            continue
-                    elif s_max:
-                        n_max = s_max
-                    elif o_max:
-                        n_max = o_max
-
-                    # Take the "max" of the two min.
-                    n_min = ""
-                    if s_min and o_min:
-                        _s_min = _make_microarchitecture(s_min)
-                        _o_min = _make_microarchitecture(o_min)
-                        if _s_min.family != _o_min.family:
-                            continue
-                        if _s_min >= _o_min:
-                            n_min = s_min
-                        elif _o_min > _s_min:
-                            n_min = o_min
-                        else:
-                            continue
-                    elif s_min:
-                        n_min = s_min
-                    elif o_min:
-                        n_min = o_min
-
-                    if n_min and n_max:
-                        _n_min = _make_microarchitecture(n_min)
-                        _n_max = _make_microarchitecture(n_max)
-                        if _n_min.family != _n_max.family or not _n_min <= _n_max:
-                            continue
-                        if n_min == n_max:
-                            results.append(n_min)
-                        else:
-                            results.append(f"{n_min}:{n_max}")
-                    elif n_min:
-                        results.append(f"{n_min}:")
-                    elif n_max:
-                        results.append(f":{n_max}")
+                    # Take intersection of two ranges
+                    # Lots of comparisons needed
+                    _s_min = _make_microarchitecture(s_min)
+                    _s_max = _make_microarchitecture(s_max)
+                    _o_min = _make_microarchitecture(o_min)
+                    _o_max = _make_microarchitecture(o_max)
 
+                    n_min = s_min if _s_min >= _o_min else o_min
+                    n_max = s_max if _s_max <= _o_max else o_max
+                    _n_min = _make_microarchitecture(n_min)
+                    _n_max = _make_microarchitecture(n_max)
+                    if _n_min == _n_max:
+                        results.append(n_min)
+                    elif not n_min or not n_max or _n_min < _n_max:
+                        results.append("%s:%s" % (n_min, n_max))
         return results
 
     def constrain(self, other: "ArchSpec") -> bool:
@@ -3180,13 +3151,18 @@ def constrain(self, other, deps=True):
             if not self.variants[v].compatible(other.variants[v]):
                 raise vt.UnsatisfiableVariantSpecError(self.variants[v], other.variants[v])
 
+        # TODO: Check out the logic here
         sarch, oarch = self.architecture, other.architecture
-        if (
-            sarch is not None
-            and oarch is not None
-            and not self.architecture.intersects(other.architecture)
-        ):
-            raise UnsatisfiableArchitectureSpecError(sarch, oarch)
+        if sarch is not None and oarch is not None:
+            if sarch.platform is not None and oarch.platform is not None:
+                if sarch.platform != oarch.platform:
+                    raise UnsatisfiableArchitectureSpecError(sarch, oarch)
+            if sarch.os is not None and oarch.os is not None:
+                if sarch.os != oarch.os:
+                    raise UnsatisfiableArchitectureSpecError(sarch, oarch)
+            if sarch.target is not None and oarch.target is not None:
+                if sarch.target != oarch.target:
+                    raise UnsatisfiableArchitectureSpecError(sarch, oarch)
 
         changed = False
 
@@ -3209,12 +3185,18 @@ def constrain(self, other, deps=True):
 
         changed |= self.compiler_flags.constrain(other.compiler_flags)
 
+        old = str(self.architecture)
         sarch, oarch = self.architecture, other.architecture
-        if sarch is not None and oarch is not None:
-            changed |= self.architecture.constrain(other.architecture)
-        elif oarch is not None:
-            self.architecture = oarch
-            changed = True
+        if sarch is None or other.architecture is None:
+            self.architecture = sarch or oarch
+        else:
+            if sarch.platform is None or oarch.platform is None:
+                self.architecture.platform = sarch.platform or oarch.platform
+            if sarch.os is None or oarch.os is None:
+                sarch.os = sarch.os or oarch.os
+            if sarch.target is None or oarch.target is None:
+                sarch.target = sarch.target or oarch.target
+        changed |= str(self.architecture) != old
 
         if deps:
             changed |= self._constrain_dependencies(other)
@@ -4726,10 +4708,7 @@ def __str__(self):
         bool_keys = []
         kv_keys = []
         for key in sorted_keys:
-            if isinstance(self[key].value, bool):
-                bool_keys.append(key)
-            else:
-                kv_keys.append(key)
+            bool_keys.append(key) if isinstance(self[key].value, bool) else kv_keys.append(key)
 
         # add spaces before and after key/value variants.
         string = io.StringIO()
@@ -5185,10 +5164,12 @@ def get_host_environment_metadata() -> Dict[str, str]:
 
 
 def get_host_environment() -> Dict[str, Any]:
-    """Returns a dictionary with host information (not including the os.environ)."""
+    """Return a dictionary (lookup) with host information (not including the
+    os.environ).
+    """
     host_platform = spack.platforms.host()
-    host_target = host_platform.default_target()
-    host_os = host_platform.default_operating_system()
+    host_target = host_platform.target("default_target")
+    host_os = host_platform.operating_system("default_os")
     arch_fmt = "platform={0} os={1} target={2}"
     arch_spec = Spec(arch_fmt.format(host_platform, host_os, host_target))
     return {

lib/spack/spack/test/architecture.py

@@ -60,7 +60,8 @@ def test_user_input_combination(config, target_str, os_str):
     """Test for all the valid user input combinations that both the target and
     the operating system match.
     """
-    spec = Spec(f"libelf os={os_str} target={target_str}")
+    spec_str = "libelf os={} target={}".format(os_str, target_str)
+    spec = Spec(spec_str)
     assert spec.architecture.os == str(TEST_PLATFORM.operating_system(os_str))
     assert spec.architecture.target == TEST_PLATFORM.target(target_str)
 
@@ -70,8 +71,8 @@ def test_default_os_and_target(default_mock_concretization):
     after concretization.
     """
     spec = default_mock_concretization("libelf")
-    assert spec.architecture.os == str(TEST_PLATFORM.default_operating_system())
-    assert spec.architecture.target == TEST_PLATFORM.default_target()
+    assert spec.architecture.os == str(TEST_PLATFORM.operating_system("default_os"))
+    assert spec.architecture.target == TEST_PLATFORM.target("default_target")
 
 
 def test_operating_system_conversion_to_dict():

lib/spack/spack/test/bindist.py

@@ -36,7 +36,6 @@
 import spack.mirrors.mirror
 import spack.oci.image
 import spack.paths
-import spack.repo
 import spack.spec
 import spack.store
 import spack.util.gpg
@@ -95,7 +94,7 @@ def config_directory(tmp_path_factory):
 
 
 @pytest.fixture(scope="function")
-def default_config(tmp_path, config_directory, mock_repo_path, install_mockery):
+def default_config(tmp_path, config_directory, monkeypatch, install_mockery):
     # This fixture depends on install_mockery to ensure
     # there is a clear order of initialization. The substitution of the
     # config scopes here is done on top of the substitution that comes with
@@ -110,6 +109,7 @@ def default_config(tmp_path, config_directory, mock_repo_path, install_mockery):
     ]
 
     with spack.config.use_configuration(*scopes):
+        spack.config.CONFIG.set("repos", [spack.paths.mock_packages_path])
         njobs = spack.config.get("config:build_jobs")
         if not njobs:
             spack.config.set("config:build_jobs", 4, scope="user")
@@ -130,8 +130,8 @@ def default_config(tmp_path, config_directory, mock_repo_path, install_mockery):
         timeout = spack.config.get("config:connect_timeout")
         if not timeout:
             spack.config.set("config:connect_timeout", 10, scope="user")
-        with spack.repo.use_repositories(mock_repo_path):
-            yield spack.config.CONFIG
+
+        yield spack.config.CONFIG
 
 
 @pytest.fixture(scope="function")

lib/spack/spack/test/build_distribution.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import os
+import os.path
 
 import pytest
 

lib/spack/spack/test/builder.py

@@ -1,7 +1,7 @@
 # Copyright Spack Project Developers. See COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import os
+import os.path
 
 import pytest
 
@@ -14,7 +14,7 @@
 
 
 @pytest.fixture()
-def builder_test_repository(config):
+def builder_test_repository():
     builder_test_path = os.path.join(spack.paths.repos_path, "builder.test")
     with spack.repo.use_repositories(builder_test_path) as mock_repo:
         yield mock_repo

lib/spack/spack/test/ci.py

@@ -1,10 +1,8 @@
 # Copyright Spack Project Developers. See COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import io
 import os
 import subprocess
-from urllib.error import HTTPError
 
 import pytest
 
@@ -17,7 +15,6 @@
 import spack.paths as spack_paths
 import spack.repo as repo
 import spack.util.git
-from spack.test.conftest import MockHTTPResponse
 
 pytestmark = [pytest.mark.usefixtures("mock_packages")]
 
@@ -165,8 +162,38 @@ def test_import_signing_key(mock_gnupghome):
     ci.import_signing_key(signing_key)
 
 
-def test_download_and_extract_artifacts(tmpdir, monkeypatch):
-    monkeypatch.setenv("GITLAB_PRIVATE_TOKEN", "faketoken")
+class FakeWebResponder:
+    def __init__(self, response_code=200, content_to_read=[]):
+        self._resp_code = response_code
+        self._content = content_to_read
+        self._read = [False for c in content_to_read]
+
+    def open(self, request, data=None, timeout=object()):
+        return self
+
+    def getcode(self):
+        return self._resp_code
+
+    def read(self, length=None):
+        if len(self._content) <= 0:
+            return None
+
+        if not self._read[-1]:
+            return_content = self._content[-1]
+            if length:
+                self._read[-1] = True
+            else:
+                self._read.pop()
+                self._content.pop()
+            return return_content
+
+        self._read.pop()
+        self._content.pop()
+        return None
+
+
+def test_download_and_extract_artifacts(tmpdir, monkeypatch, working_env):
+    os.environ.update({"GITLAB_PRIVATE_TOKEN": "faketoken"})
 
     url = "https://www.nosuchurlexists.itsfake/artifacts.zip"
     working_dir = os.path.join(tmpdir.strpath, "repro")
@@ -174,13 +201,10 @@ def test_download_and_extract_artifacts(tmpdir, monkeypatch):
         spack_paths.test_path, "data", "ci", "gitlab", "artifacts.zip"
     )
 
-    def _urlopen_OK(*args, **kwargs):
-        with open(test_artifacts_path, "rb") as f:
-            return MockHTTPResponse(
-                "200", "OK", {"Content-Type": "application/zip"}, io.BytesIO(f.read())
-            )
+    with open(test_artifacts_path, "rb") as fd:
+        fake_responder = FakeWebResponder(content_to_read=[fd.read()])
 
-    monkeypatch.setattr(ci, "urlopen", _urlopen_OK)
+    monkeypatch.setattr(ci, "build_opener", lambda handler: fake_responder)
 
     ci.download_and_extract_artifacts(url, working_dir)
 
@@ -190,11 +214,7 @@ def _urlopen_OK(*args, **kwargs):
     found_install = fs.find(working_dir, "install.sh")
     assert len(found_install) == 1
 
-    def _urlopen_500(*args, **kwargs):
-        raise HTTPError(url, 500, "Internal Server Error", {}, None)
-
-    monkeypatch.setattr(ci, "urlopen", _urlopen_500)
-
+    fake_responder._resp_code = 400
     with pytest.raises(spack.error.SpackError):
         ci.download_and_extract_artifacts(url, working_dir)
 
@@ -308,14 +328,16 @@ def test_get_spec_filter_list(mutable_mock_env_path, mutable_mock_repo):
     e1.add("hypre")
     e1.concretize()
 
-    # Concretizing the above environment results in the following graphs:
+    """
+    Concretizing the above environment results in the following graphs:
 
-    # mpileaks -> mpich (provides mpi virtual dep of mpileaks)
-    #          -> callpath -> dyninst -> libelf
-    #                                 -> libdwarf -> libelf
-    #                      -> mpich (provides mpi dep of callpath)
+    mpileaks -> mpich (provides mpi virtual dep of mpileaks)
+             -> callpath -> dyninst -> libelf
+                                    -> libdwarf -> libelf
+                         -> mpich (provides mpi dep of callpath)
 
-    # hypre -> openblas-with-lapack (provides lapack and blas virtual deps of hypre)
+    hypre -> openblas-with-lapack (provides lapack and blas virtual deps of hypre)
+    """
 
     touched = ["libdwarf"]
 

lib/spack/spack/test/cmd/bootstrap.py

@@ -1,7 +1,7 @@
 # Copyright Spack Project Developers. See COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import os
+import os.path
 import sys
 
 import pytest

lib/spack/spack/test/cmd/ci.py

@@ -5,6 +5,7 @@
 import os
 import pathlib
 import shutil
+from io import BytesIO
 from typing import NamedTuple
 
 import jsonschema
@@ -31,7 +32,6 @@
 from spack.schema.buildcache_spec import schema as specfile_schema
 from spack.schema.database_index import schema as db_idx_schema
 from spack.spec import Spec
-from spack.test.conftest import MockHTTPResponse
 
 config_cmd = spack.main.SpackCommand("config")
 ci_cmd = spack.main.SpackCommand("ci")
@@ -239,7 +239,7 @@ def test_ci_generate_with_cdash_token(ci_generate_test, tmp_path, mock_binary_in
     # That fake token should have resulted in being unable to
     # register build group with cdash, but the workload should
     # still have been generated.
-    assert "Failed to create or retrieve buildgroups" in output
+    assert "Problem populating buildgroup" in output
     expected_keys = ["rebuild-index", "stages", "variables", "workflow"]
     assert all([key in yaml_contents.keys() for key in expected_keys])
 
@@ -329,14 +329,14 @@ def test_ci_generate_pkg_with_deps(ci_generate_test, tmp_path, ci_base_environme
         f"""\
 spack:
   specs:
-    - dependent-install
+    - flatten-deps
   mirrors:
     buildcache-destination: {tmp_path / 'ci-mirror'}
   ci:
     pipeline-gen:
     - submapping:
       - match:
-          - dependent-install
+          - flatten-deps
         build-job:
           tags:
             - donotcare
@@ -355,12 +355,12 @@ def test_ci_generate_pkg_with_deps(ci_generate_test, tmp_path, ci_base_environme
             assert "stage" in ci_obj
             assert ci_obj["stage"] == "stage-0"
             found.append("dependency-install")
-        if "dependent-install" in ci_key:
+        if "flatten-deps" in ci_key:
             assert "stage" in ci_obj
             assert ci_obj["stage"] == "stage-1"
-            found.append("dependent-install")
+            found.append("flatten-deps")
 
-    assert "dependent-install" in found
+    assert "flatten-deps" in found
     assert "dependency-install" in found
 
 
@@ -372,14 +372,14 @@ def test_ci_generate_for_pr_pipeline(ci_generate_test, tmp_path, monkeypatch):
         f"""\
 spack:
   specs:
-    - dependent-install
+    - flatten-deps
   mirrors:
     buildcache-destination: {tmp_path / 'ci-mirror'}
   ci:
     pipeline-gen:
     - submapping:
       - match:
-          - dependent-install
+          - flatten-deps
         build-job:
           tags:
             - donotcare
@@ -899,7 +899,7 @@ def test_ci_generate_override_runner_attrs(
         f"""\
 spack:
   specs:
-    - dependent-install
+    - flatten-deps
     - pkg-a
   mirrors:
     buildcache-destination: {tmp_path / "ci-mirror"}
@@ -908,7 +908,7 @@ def test_ci_generate_override_runner_attrs(
     - match_behavior: {match_behavior}
       submapping:
         - match:
-            - dependent-install
+            - flatten-deps
           build-job:
             tags:
               - specific-one
@@ -1006,8 +1006,8 @@ def test_ci_generate_override_runner_attrs(
             assert the_elt["script"][0] == "main step"
             assert len(the_elt["after_script"]) == 1
             assert the_elt["after_script"][0] == "post step one"
-        if "dependent-install" in ci_key:
-            # The dependent-install match specifies that we keep the two
+        if "flatten-deps" in ci_key:
+            # The flatten-deps match specifies that we keep the two
             # top level variables, but add a third specifc one.  It
             # also adds a custom tag which should be combined with
             # the top-level tag.
@@ -1182,12 +1182,12 @@ def test_ci_generate_read_broken_specs_url(
     spec_a = spack.concretize.concretize_one("pkg-a")
     a_dag_hash = spec_a.dag_hash()
 
-    spec_flattendeps = spack.concretize.concretize_one("dependent-install")
+    spec_flattendeps = spack.concretize.concretize_one("flatten-deps")
     flattendeps_dag_hash = spec_flattendeps.dag_hash()
 
     broken_specs_url = tmp_path.as_uri()
 
-    # Mark 'a' as broken (but not 'dependent-install')
+    # Mark 'a' as broken (but not 'flatten-deps')
     broken_spec_a_url = "{0}/{1}".format(broken_specs_url, a_dag_hash)
     job_stack = "job_stack"
     a_job_url = "a_job_url"
@@ -1201,7 +1201,7 @@ def test_ci_generate_read_broken_specs_url(
             f"""\
 spack:
   specs:
-    - dependent-install
+    - flatten-deps
     - pkg-a
   mirrors:
     buildcache-destination: {(tmp_path / "ci-mirror").as_uri()}
@@ -1211,7 +1211,7 @@ def test_ci_generate_read_broken_specs_url(
     - submapping:
       - match:
           - pkg-a
-          - dependent-install
+          - flatten-deps
           - pkg-b
           - dependency-install
         build-job:
@@ -1234,7 +1234,7 @@ def test_ci_generate_read_broken_specs_url(
             )
             assert expected in output
 
-            not_expected = f"dependent-install/{flattendeps_dag_hash[:7]} (in stack"
+            not_expected = f"flatten-deps/{flattendeps_dag_hash[:7]} (in stack"
             assert not_expected not in output
 
 
@@ -1447,7 +1447,7 @@ def test_gitlab_config_scopes(ci_generate_test, tmp_path):
   include: [{configs_path}]
   view: false
   specs:
-    - dependent-install
+    - flatten-deps
   mirrors:
     buildcache-destination: {tmp_path / "ci-mirror"}
   ci:
@@ -1548,10 +1548,10 @@ def test_ci_dynamic_mapping_empty(
     ci_base_environment,
 ):
     # The test will always return an empty dictionary
-    def _urlopen(*args, **kwargs):
-        return MockHTTPResponse.with_json(200, "OK", headers={}, body={})
+    def fake_dyn_mapping_urlopener(*args, **kwargs):
+        return BytesIO("{}".encode())
 
-    monkeypatch.setattr(ci.common, "_urlopen", _urlopen)
+    monkeypatch.setattr(ci.common, "_dyn_mapping_urlopener", fake_dyn_mapping_urlopener)
 
     _ = dynamic_mapping_setup(tmpdir)
     with tmpdir.as_cwd():
@@ -1572,15 +1572,15 @@ def test_ci_dynamic_mapping_full(
     monkeypatch,
     ci_base_environment,
 ):
-    def _urlopen(*args, **kwargs):
-        return MockHTTPResponse.with_json(
-            200,
-            "OK",
-            headers={},
-            body={"variables": {"MY_VAR": "hello"}, "ignored_field": 0, "unallowed_field": 0},
+    # The test will always return an empty dictionary
+    def fake_dyn_mapping_urlopener(*args, **kwargs):
+        return BytesIO(
+            json.dumps(
+                {"variables": {"MY_VAR": "hello"}, "ignored_field": 0, "unallowed_field": 0}
+            ).encode()
         )
 
-    monkeypatch.setattr(ci.common, "_urlopen", _urlopen)
+    monkeypatch.setattr(ci.common, "_dyn_mapping_urlopener", fake_dyn_mapping_urlopener)
 
     label = dynamic_mapping_setup(tmpdir)
     with tmpdir.as_cwd():

lib/spack/spack/test/cmd/debug.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import os
+import os.path
 import platform
 
 import pytest
@@ -51,8 +52,8 @@ def test_create_db_tarball(tmpdir, database):
 def test_report():
     out = debug("report")
     host_platform = spack.platforms.host()
-    host_os = host_platform.default_operating_system()
-    host_target = host_platform.default_target()
+    host_os = host_platform.operating_system("frontend")
+    host_target = host_platform.target("frontend")
     architecture = spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))
 
     assert spack.get_version() in out

lib/spack/spack/test/cmd/env.py

@@ -1038,58 +1038,6 @@ def test_init_from_yaml(environment_from_manifest):
     assert not e2.specs_by_hash
 
 
-def test_init_from_yaml_relative_includes(tmp_path):
-    files = [
-        "relative_copied/packages.yaml",
-        "./relative_copied/compilers.yaml",
-        "repos.yaml",
-        "./config.yaml",
-    ]
-
-    manifest = f"""
-spack:
-  specs: []
-  include: {files}
-"""
-
-    e1_path = tmp_path / "e1"
-    e1_manifest = e1_path / "spack.yaml"
-    fs.mkdirp(e1_path)
-    with open(e1_manifest, "w", encoding="utf-8") as f:
-        f.write(manifest)
-
-    for f in files:
-        fs.touchp(e1_path / f)
-
-    e2 = _env_create("test2", init_file=e1_manifest)
-
-    for f in files:
-        assert os.path.exists(os.path.join(e2.path, f))
-
-
-def test_init_from_yaml_relative_includes_outside_env(tmp_path):
-    files = ["../outside_env_not_copied/repos.yaml"]
-
-    manifest = f"""
-spack:
-  specs: []
-  include: {files}
-"""
-
-    # subdir to ensure parent of environment dir is not shared
-    e1_path = tmp_path / "e1_subdir" / "e1"
-    e1_manifest = e1_path / "spack.yaml"
-    fs.mkdirp(e1_path)
-    with open(e1_manifest, "w", encoding="utf-8") as f:
-        f.write(manifest)
-
-    for f in files:
-        fs.touchp(e1_path / f)
-
-    with pytest.raises(spack.config.ConfigFileError, match="Detected 1 missing include"):
-        _ = _env_create("test2", init_file=e1_manifest)
-
-
 def test_env_view_external_prefix(tmp_path, mutable_database, mock_packages):
     fake_prefix = tmp_path / "a-prefix"
     fake_bin = fake_prefix / "bin"
@@ -2666,7 +2614,7 @@ def test_stack_yaml_remove_from_matrix_no_effect(tmpdir):
     - packages:
         - matrix:
             - [mpileaks, callpath]
-            - [target=default_target]
+            - [target=be]
   specs:
     - $packages
 """
@@ -2691,7 +2639,7 @@ def test_stack_yaml_force_remove_from_matrix(tmpdir):
     - packages:
         - matrix:
             - [mpileaks, callpath]
-            - [target=default_target]
+            - [target=be]
   specs:
     - $packages
 """
@@ -2711,7 +2659,7 @@ def test_stack_yaml_force_remove_from_matrix(tmpdir):
 
             assert before_user == after_user
 
-            mpileaks_spec = Spec("mpileaks target=default_target")
+            mpileaks_spec = Spec("mpileaks target=be")
             assert mpileaks_spec in before_conc
             assert mpileaks_spec not in after_conc
 
@@ -3075,35 +3023,6 @@ def test_stack_view_activate_from_default(
         assert "FOOBAR=mpileaks" in shell
 
 
-def test_envvar_set_in_activate(tmpdir, mock_fetch, mock_packages, mock_archive, install_mockery):
-    filename = str(tmpdir.join("spack.yaml"))
-    with open(filename, "w", encoding="utf-8") as f:
-        f.write(
-            """\
-spack:
-  specs:
-    - cmake%gcc
-  env_vars:
-    set:
-      ENVAR_SET_IN_ENV_LOAD: "True"
-"""
-        )
-    with tmpdir.as_cwd():
-        env("create", "test", "./spack.yaml")
-        with ev.read("test"):
-            install()
-
-        test_env = ev.read("test")
-        output = env("activate", "--sh", "test")
-
-        assert "ENVAR_SET_IN_ENV_LOAD=True" in output
-
-        with test_env:
-            with spack.util.environment.set_env(ENVAR_SET_IN_ENV_LOAD="True"):
-                output = env("deactivate", "--sh")
-                assert "unset ENVAR_SET_IN_ENV_LOAD" in output
-
-
 def test_stack_view_no_activate_without_default(
     installed_environment, template_combinatorial_env, tmp_path
 ):

lib/spack/spack/test/cmd/external.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
+import os.path
 import sys
 
 import pytest

lib/spack/spack/test/cmd/gc.py

@@ -139,7 +139,7 @@ def test_gc_except_specific_environments(mutable_database, mutable_mock_env_path
 def test_gc_except_nonexisting_dir_env(mutable_database, mutable_mock_env_path, tmpdir):
     output = gc("-ye", tmpdir.strpath, fail_on_error=False)
     assert "No such environment" in output
-    assert gc.returncode == 1
+    gc.returncode == 1
 
 
 @pytest.mark.db

lib/spack/spack/test/cmd/license.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import os
+import os.path
 import re
 
 import pytest

lib/spack/spack/test/cmd/list.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import os
+import os.path
 import sys
 from textwrap import dedent
 

lib/spack/spack/test/cmd/load.py

@@ -26,9 +26,9 @@ def test_manpath_trailing_colon(
         else ("--sh", "export %s=%s", ";")
     )
 
-    # Test that the commands generated by load add the MANPATH prefix
-    # inspections. Also test that Spack correctly preserves the default/existing
-    # manpath search path via a trailing colon
+    """Test that the commands generated by load add the MANPATH prefix
+    inspections. Also test that Spack correctly preserves the default/existing
+    manpath search path via a trailing colon"""
     install("mpileaks")
 
     sh_out = load(shell, "mpileaks")
@@ -81,9 +81,7 @@ def extract_value(output, variable):
 
         # Finally, do we list them in topo order?
         for i, pkg in enumerate(pkgs):
-            assert {s.name for s in mpileaks_spec[pkg].traverse(direction="parents")}.issubset(
-                pkgs[: i + 1]
-            )
+            set(s.name for s in mpileaks_spec[pkg].traverse(direction="parents")) in set(pkgs[:i])
 
         # Lastly, do we keep track that mpileaks was loaded?
         assert (

lib/spack/spack/test/cmd/module.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import os
+import os.path
 import re
 
 import pytest

lib/spack/spack/test/cmd/repo.py

@@ -1,7 +1,7 @@
 # Copyright Spack Project Developers. See COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import os
+import os.path
 
 import pytest
 

lib/spack/spack/test/cmd/view.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-import os
+import os.path
 import sys
 
 import pytest