foo

.github/workflows/audit.yaml

@@ -23,7 +23,7 @@ jobs:
         operating_system: ["ubuntu-latest", "macos-latest"]
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
+    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
       with:
         python-version: ${{inputs.python_version}}
     - name: Install Python packages

.github/workflows/bootstrap.yml

@@ -159,7 +159,7 @@ jobs:
           brew install cmake bison@2.7 tree
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
-      - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
+      - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
         with:
           python-version: "3.12"
       - name: Bootstrap clingo

.github/workflows/build-containers.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
 
-      - uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4
+      - uses: docker/metadata-action@e6428a5c4e294a61438ed7f43155db912025b6b3
         id: docker_meta
         with:
           images: |

.github/workflows/nightly-win-builds.yml

@@ -17,7 +17,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
+    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
       with:
         python-version: 3.9
     - name: Install Python packages

.github/workflows/unit_tests.yaml

@@ -54,7 +54,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
+    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install System packages
@@ -101,7 +101,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
+    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -159,7 +159,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
+    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
       with:
         python-version: '3.11'
     - name: Install System packages
@@ -194,7 +194,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # @v2
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # @v2
+    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # @v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Python packages

.github/workflows/valid-style.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
+    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
       with:
         python-version: '3.11'
         cache: 'pip'
@@ -38,7 +38,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
+    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
       with:
         python-version: '3.11'
         cache: 'pip'

.github/workflows/windows_python.yml

@@ -18,7 +18,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
+    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
       with:
         python-version: 3.9
     - name: Install Python packages
@@ -42,7 +42,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
+    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
       with:
         python-version: 3.9
     - name: Install Python packages
@@ -66,7 +66,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
+    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236
       with:
         python-version: 3.9
     - name: Install Python packages

README.md

@@ -66,11 +66,10 @@ Resources:
 * **Matrix space**: [#spack-space:matrix.org](https://matrix.to/#/#spack-space:matrix.org):
   [bridged](https://github.com/matrix-org/matrix-appservice-slack#matrix-appservice-slack) to Slack.
 * [**Github Discussions**](https://github.com/spack/spack/discussions):
-  for Q&A and discussions. Note the pinned discussions for announcements.
+  not just for discussions, but also Q&A.
+* **Mailing list**: [groups.google.com/d/forum/spack](https://groups.google.com/d/forum/spack)
 * **Twitter**: [@spackpm](https://twitter.com/spackpm). Be sure to
   `@mention` us!
-* **Mailing list**: [groups.google.com/d/forum/spack](https://groups.google.com/d/forum/spack):
-  only for announcements. Please use other venues for discussions.
 
 Contributing
 ------------------------

lib/spack/docs/extensions.rst

@@ -9,42 +9,46 @@
 Custom Extensions
 =================
 
-*Spack extensions* allow you to extend Spack capabilities by deploying your
+*Spack extensions* permit you to extend Spack capabilities by deploying your
 own custom commands or logic in an arbitrary location on your filesystem.
 This might be extremely useful e.g. to develop and maintain a command whose purpose is
 too specific to be considered for reintegration into the mainline or to
 evolve a command through its early stages before starting a discussion to merge
 it upstream.
-
 From Spack's point of view an extension is any path in your filesystem which
-respects the following naming and layout for files:
+respects a prescribed naming and layout for files:
 
 .. code-block:: console
 
   spack-scripting/ # The top level directory must match the format 'spack-{extension_name}'
   ├── pytest.ini # Optional file if the extension ships its own tests
   ├── scripting # Folder that may contain modules that are needed for the extension commands
-  │   ├── cmd # Folder containing extension commands
-  │   │   └── filter.py # A new command that will be available
-  │   └── functions.py # Module with internal details
-  └── tests # Tests for this extension
+  │   └── cmd # Folder containing extension commands
+  │       └── filter.py # A new command that will be available
+  ├── tests # Tests for this extension
   │   ├── conftest.py
   │   └── test_filter.py
   └── templates # Templates that may be needed by the extension
 
-In the example above, the extension is named *scripting*. It adds an additional command
-(``spack filter``) and unit tests to verify its behavior.
+In the example above the extension named *scripting* adds an additional command (``filter``)
+and unit tests to verify its behavior. The code for this example can be
+obtained by cloning the corresponding git repository:
 
-The extension can import any core Spack module in its implementation. When loaded by
-the ``spack`` command, the extension itself is imported as a Python package in the
-``spack.extensions`` namespace. In the example above, since the extension is named
-"scripting", the corresponding Python module is ``spack.extensions.scripting``.
-
-The code for this example extension can be obtained by cloning the corresponding git repository:
+.. TODO: write an ad-hoc "hello world" extension and make it part of the spack organization
 
 .. code-block:: console
 
-   $ git -C /tmp clone https://github.com/spack/spack-scripting.git
+   $ cd ~/
+   $ mkdir tmp && cd tmp
+   $ git clone https://github.com/alalazo/spack-scripting.git
+   Cloning into 'spack-scripting'...
+   remote: Counting objects: 11, done.
+   remote: Compressing objects: 100% (7/7), done.
+   remote: Total 11 (delta 0), reused 11 (delta 0), pack-reused 0
+   Receiving objects: 100% (11/11), done.
+
+As you can see by inspecting the sources, Python modules that are part of the extension
+can import any core Spack module.
 
 ---------------------------------
 Configure Spack to Use Extensions
@@ -57,7 +61,7 @@ paths to ``config.yaml``. In the case of our example this means ensuring that:
 
    config:
      extensions:
-     - /tmp/spack-scripting
+     - ~/tmp/spack-scripting
 
 is part of your configuration file. Once this is setup any command that the extension provides
 will be available from the command line:
@@ -82,32 +86,37 @@ will be available from the command line:
      --implicit       select specs that are not installed or were installed implicitly
      --output OUTPUT  where to dump the result
 
-The corresponding unit tests can be run giving the appropriate options to ``spack unit-test``:
+The corresponding unit tests can be run giving the appropriate options
+to ``spack unit-test``:
 
 .. code-block:: console
 
    $ spack unit-test --extension=scripting
-   ========================================== test session starts ===========================================
-   platform linux -- Python 3.11.5, pytest-7.4.3, pluggy-1.3.0
-   rootdir: /home/culpo/github/spack-scripting
-   configfile: pytest.ini
-   testpaths: tests
-   plugins: xdist-3.5.0
+
+   ============================================================== test session starts ===============================================================
+   platform linux2 -- Python 2.7.15rc1, pytest-3.2.5, py-1.4.34, pluggy-0.4.0
+   rootdir: /home/mculpo/tmp/spack-scripting, inifile: pytest.ini
    collected 5 items
 
-   tests/test_filter.py .....                                                                         [100%]
+   tests/test_filter.py ...XX
+   ============================================================ short test summary info =============================================================
+   XPASS tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
+   XPASS tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
 
-   ========================================== slowest 30 durations ==========================================
-   2.31s setup    tests/test_filter.py::test_filtering_specs[kwargs0-specs0-expected0]
-   0.57s call     tests/test_filter.py::test_filtering_specs[kwargs2-specs2-expected2]
-   0.56s call     tests/test_filter.py::test_filtering_specs[kwargs4-specs4-expected4]
-   0.54s call     tests/test_filter.py::test_filtering_specs[kwargs3-specs3-expected3]
-   0.54s call     tests/test_filter.py::test_filtering_specs[kwargs1-specs1-expected1]
-   0.48s call     tests/test_filter.py::test_filtering_specs[kwargs0-specs0-expected0]
-   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs4-specs4-expected4]
-   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs2-specs2-expected2]
-   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs1-specs1-expected1]
-   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs3-specs3-expected3]
-
-   (5 durations < 0.005s hidden.  Use -vv to show these durations.)
-   =========================================== 5 passed in 5.06s ============================================
+   =========================================================== slowest 20 test durations ============================================================
+   3.74s setup    tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
+   0.17s call     tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
+   0.16s call     tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
+   0.15s call     tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
+   0.13s call     tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
+   0.08s call     tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
+   0.04s teardown tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
+   0.00s setup    tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
+   0.00s setup    tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
+   0.00s setup    tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
+   0.00s setup    tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
+   0.00s teardown tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
+   0.00s teardown tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
+   0.00s teardown tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
+   0.00s teardown tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
+   ====================================================== 3 passed, 2 xpassed in 4.51 seconds =======================================================

lib/spack/docs/gpu_configuration.rst

@@ -111,28 +111,3 @@ CUDA is split into fewer components and is simpler to specify:
          prefix: /opt/cuda/cuda-11.0.2/
 
 where ``/opt/cuda/cuda-11.0.2/lib/`` contains ``libcudart.so``.
-
-
-
------------------------------------
-Using an External OpenGL API
------------------------------------
-Depending on whether we have a graphics card or not, we may choose to use OSMesa or GLX to implement the OpenGL API.
-
-If a graphics card is unavailable, OSMesa is recommended and can typically be built with Spack.
-However, if we prefer to utilize the system GLX tailored to our graphics card, we need to declare it as an external. Here's how to do it:
-
-
-.. code-block:: yaml
-
-  packages:
-    libglx:
-      require: [opengl]
-    opengl:
-      buildable: false
-      externals:
-      - prefix: /usr/
-        spec: opengl@4.6
-
-Note that prefix has to be the root of both the libraries and the headers, using is /usr not the path the the lib.
-To know which spec for opengl is available use ``cd /usr/include/GL && grep -Ri gl_version``.

lib/spack/llnl/util/lang.py

@@ -1047,9 +1047,9 @@ def __bool__(self):
         """Whether any exceptions were handled."""
         return bool(self.exceptions)
 
-    def forward(self, context: str, base: type = BaseException) -> "GroupedExceptionForwarder":
+    def forward(self, context: str) -> "GroupedExceptionForwarder":
         """Return a contextmanager which extracts tracebacks and prefixes a message."""
-        return GroupedExceptionForwarder(context, self, base)
+        return GroupedExceptionForwarder(context, self)
 
     def _receive_forwarded(self, context: str, exc: Exception, tb: List[str]):
         self.exceptions.append((context, exc, tb))
@@ -1072,18 +1072,15 @@ class GroupedExceptionForwarder:
     """A contextmanager to capture exceptions and forward them to a
     GroupedExceptionHandler."""
 
-    def __init__(self, context: str, handler: GroupedExceptionHandler, base: type):
+    def __init__(self, context: str, handler: GroupedExceptionHandler):
         self._context = context
         self._handler = handler
-        self._base = base
 
     def __enter__(self):
         return None
 
     def __exit__(self, exc_type, exc_value, tb):
         if exc_value is not None:
-            if not issubclass(exc_type, self._base):
-                return False
             self._handler._receive_forwarded(self._context, exc_value, traceback.format_tb(tb))
 
         # Suppress any exception from being re-raised:

lib/spack/spack/audit.py

@@ -499,7 +499,7 @@ def _ensure_packages_are_pickeleable(pkgs, error_cls):
 @package_properties
 def _ensure_packages_are_unparseable(pkgs, error_cls):
     """Ensure that all packages can unparse and that unparsed code is valid Python"""
-    import spack.util.package_hash as ph
+    import spack.package_hash as ph
 
     errors = []
     for pkg_name in pkgs:
@@ -646,7 +646,11 @@ def _linting_package_file(pkgs, error_cls):
         if pkg_cls.homepage.startswith("http://"):
             https = re.sub("http", "https", pkg_cls.homepage, 1)
             try:
-                response = urlopen(https)
+                response = urlopen(
+                    https,
+                    verify_ssl=spack.config.get("config:verify_ssl", True),
+                    timeout=spack.config.get("config:connect_timeout", 10),
+                )
             except Exception as e:
                 msg = 'Error with attempting https for "{0}": '
                 errors.append(error_cls(msg.format(pkg_cls.name), [str(e)]))
@@ -726,37 +730,13 @@ def _unknown_variants_in_directives(pkgs, error_cls):
 
 
 @package_directives
-def _issues_in_depends_on_directive(pkgs, error_cls):
-    """Reports issues with 'depends_on' directives.
-
-    Issues might be unknown dependencies, unknown variants or variant values, or declaration
-    of nested dependencies.
-    """
+def _unknown_variants_in_dependencies(pkgs, error_cls):
+    """Report unknown dependencies and wrong variants for dependencies"""
     errors = []
     for pkg_name in pkgs:
         pkg_cls = spack.repo.PATH.get_pkg_class(pkg_name)
         filename = spack.repo.PATH.filename_for_package_name(pkg_name)
         for dependency_name, dependency_data in pkg_cls.dependencies.items():
-            # Check if there are nested dependencies declared. We don't want directives like:
-            #
-            #     depends_on('foo+bar ^fee+baz')
-            #
-            # but we'd like to have two dependencies listed instead.
-            for when, dependency_edge in dependency_data.items():
-                dependency_spec = dependency_edge.spec
-                nested_dependencies = dependency_spec.dependencies()
-                if nested_dependencies:
-                    summary = (
-                        f"{pkg_name}: invalid nested dependency "
-                        f"declaration '{str(dependency_spec)}'"
-                    )
-                    details = [
-                        f"split depends_on('{str(dependency_spec)}', when='{str(when)}') "
-                        f"into {len(nested_dependencies) + 1} directives",
-                        f"in {filename}",
-                    ]
-                    errors.append(error_cls(summary=summary, details=details))
-
             # No need to analyze virtual packages
             if spack.repo.PATH.is_virtual(dependency_name):
                 continue

lib/spack/spack/binary_distribution.py

@@ -38,12 +38,14 @@
 import spack.config as config
 import spack.database as spack_db
 import spack.error
+import spack.gpg
 import spack.hooks
 import spack.hooks.sbang
 import spack.mirror
 import spack.oci.image
 import spack.oci.oci
 import spack.oci.opener
+import spack.paths
 import spack.platforms
 import spack.relocate as relocate
 import spack.repo
@@ -52,7 +54,6 @@
 import spack.traverse as traverse
 import spack.util.crypto
 import spack.util.file_cache as file_cache
-import spack.util.gpg
 import spack.util.path
 import spack.util.spack_json as sjson
 import spack.util.spack_yaml as syaml
@@ -486,7 +487,10 @@ def _fetch_and_cache_index(self, mirror_url, cache_entry={}):
         scheme = urllib.parse.urlparse(mirror_url).scheme
 
         if scheme != "oci" and not web_util.url_exists(
-            url_util.join(mirror_url, BUILD_CACHE_RELATIVE_PATH, "index.json")
+            url_util.join(mirror_url, BUILD_CACHE_RELATIVE_PATH, "index.json"),
+            fetch_method=spack.config.get("config:url_fetch_method", "urllib"),
+            verify_ssl=spack.config.get("config:verify_ssl"),
+            timeout=spack.config.get("config:connect_timeout", 10),
         ):
             return False
 
@@ -532,7 +536,9 @@ def _fetch_and_cache_index(self, mirror_url, cache_entry={}):
 def binary_index_location():
     """Set up a BinaryCacheIndex for remote buildcache dbs in the user's homedir."""
     cache_root = os.path.join(misc_cache_location(), "indices")
-    return spack.util.path.canonicalize_path(cache_root)
+    return spack.util.path.canonicalize_path(
+        cache_root, replacements=spack.paths.path_replacements()
+    )
 
 
 #: Default binary cache index instance
@@ -827,7 +833,7 @@ def tarball_path_name(spec, ext):
 
 def select_signing_key(key=None):
     if key is None:
-        keys = spack.util.gpg.signing_keys()
+        keys = spack.gpg.signing_keys()
         if len(keys) == 1:
             key = keys[0]
 
@@ -852,7 +858,7 @@ def sign_specfile(key, force, specfile_path):
             raise NoOverwriteException(signed_specfile_path)
 
     key = select_signing_key(key)
-    spack.util.gpg.sign(key, specfile_path, signed_specfile_path, clearsign=True)
+    spack.gpg.sign(key, specfile_path, signed_specfile_path, clearsign=True)
 
 
 def _read_specs_and_push_index(file_list, read_method, cache_prefix, db, temp_dir, concurrency):
@@ -904,6 +910,7 @@ def _read_specs_and_push_index(file_list, read_method, cache_prefix, db, temp_di
         url_util.join(cache_prefix, "index.json"),
         keep_original=False,
         extra_args={"ContentType": "application/json", "CacheControl": "no-cache"},
+        verify_ssl=spack.config.get("config:verify_ssl", True),
     )
 
     # Push the hash
@@ -912,6 +919,7 @@ def _read_specs_and_push_index(file_list, read_method, cache_prefix, db, temp_di
         url_util.join(cache_prefix, "index.json.hash"),
         keep_original=False,
         extra_args={"ContentType": "text/plain", "CacheControl": "no-cache"},
+        verify_ssl=spack.config.get("config:verify_ssl", True),
     )
 
 
@@ -976,9 +984,13 @@ def _specs_from_cache_fallback(cache_prefix):
     def url_read_method(url):
         contents = None
         try:
-            _, _, spec_file = web_util.read_from_url(url)
+            _, _, spec_file = web_util.read_from_url(
+                url,
+                verify_ssl=spack.config.get("config:verify_ssl", True),
+                timeout=spack.config.get("config:connect_timeout", 10),
+            )
             contents = codecs.getreader("utf-8")(spec_file).read()
-        except (URLError, web_util.SpackWebError) as url_err:
+        except (URLError, web_util.WebError) as url_err:
             tty.error("Error reading specfile: {0}".format(url))
             tty.error(url_err)
         return contents
@@ -986,7 +998,9 @@ def url_read_method(url):
     try:
         file_list = [
             url_util.join(cache_prefix, entry)
-            for entry in web_util.list_url(cache_prefix)
+            for entry in web_util.list_url(
+                cache_prefix, verify_ssl=spack.config.get("config:verify_ssl", True)
+            )
             if entry.endswith("spec.json") or entry.endswith("spec.json.sig")
         ]
         read_fn = url_read_method
@@ -1084,7 +1098,9 @@ def generate_key_index(key_prefix, tmpdir=None):
     try:
         fingerprints = (
             entry[:-4]
-            for entry in web_util.list_url(key_prefix, recursive=False)
+            for entry in web_util.list_url(
+                key_prefix, recursive=False, verify_ssl=spack.config.get("config:verify_ssl", True)
+            )
             if entry.endswith(".pub")
         )
     except KeyError as inst:
@@ -1121,6 +1137,7 @@ def generate_key_index(key_prefix, tmpdir=None):
                 url_util.join(key_prefix, "index.json"),
                 keep_original=False,
                 extra_args={"ContentType": "application/json"},
+                verify_ssl=spack.config.get("config:verify_ssl", True),
             )
         except Exception as err:
             msg = "Encountered problem pushing key index to {0}: {1}".format(key_prefix, err)
@@ -1364,10 +1381,18 @@ def _build_tarball_in_stage_dir(spec: Spec, out_url: str, stage_dir: str, option
     spackfile_path = os.path.join(cache_prefix, tarball_path_name(spec, ".spack"))
     remote_spackfile_path = url_util.join(out_url, os.path.relpath(spackfile_path, stage_dir))
 
+    fetch_method = (spack.config.get("config:url_fetch_method", "urllib"),)
+    verify_ssl = (spack.config.get("config:verify_ssl"),)
+    timeout = spack.config.get("config:connect_timeout", 10)
+
+    url_args = {"fetch_method": fetch_method, "verify_ssl": verify_ssl, "timeout": timeout}
+
     mkdirp(tarfile_dir)
-    if web_util.url_exists(remote_spackfile_path):
+    if web_util.url_exists(remote_spackfile_path, **url_args):
         if options.force:
-            web_util.remove_url(remote_spackfile_path)
+            web_util.remove_url(
+                remote_spackfile_path, verify_ssl=spack.config.get("config:verify_ssl", True)
+            )
         else:
             raise NoOverwriteException(url_util.format(remote_spackfile_path))
 
@@ -1387,12 +1412,13 @@ def _build_tarball_in_stage_dir(spec: Spec, out_url: str, stage_dir: str, option
 
     # If force and exists, overwrite. Otherwise raise exception on collision.
     if options.force:
-        if web_util.url_exists(remote_specfile_path):
-            web_util.remove_url(remote_specfile_path)
-        if web_util.url_exists(remote_signed_specfile_path):
-            web_util.remove_url(remote_signed_specfile_path)
-    elif web_util.url_exists(remote_specfile_path) or web_util.url_exists(
-        remote_signed_specfile_path
+        verify_ssl = spack.config.get("config:verify_ssl", True)
+        if web_util.url_exists(remote_specfile_path, **url_args):
+            web_util.remove_url(remote_specfile_path, verify_ssl=verify_ssl)
+        if web_util.url_exists(remote_signed_specfile_path, **url_args):
+            web_util.remove_url(remote_signed_specfile_path, verify_ssl=verify_ssl)
+    elif web_util.url_exists(remote_specfile_path, **url_args) or web_util.url_exists(
+        remote_signed_specfile_path, **url_args
     ):
         raise NoOverwriteException(url_util.format(remote_specfile_path))
 
@@ -1426,11 +1452,17 @@ def _build_tarball_in_stage_dir(spec: Spec, out_url: str, stage_dir: str, option
         sign_specfile(key, options.force, specfile_path)
 
     # push tarball and signed spec json to remote mirror
-    web_util.push_to_url(spackfile_path, remote_spackfile_path, keep_original=False)
+    web_util.push_to_url(
+        spackfile_path,
+        remote_spackfile_path,
+        keep_original=False,
+        verify_ssl=spack.config.get("config:verify_ssl", True),
+    )
     web_util.push_to_url(
         signed_specfile_path if not options.unsigned else specfile_path,
         remote_signed_specfile_path if not options.unsigned else remote_specfile_path,
         keep_original=False,
+        verify_ssl=spack.config.get("config:verify_ssl", True),
     )
 
     # push the key to the build cache's _pgp directory so it can be
@@ -1534,7 +1566,7 @@ def try_verify(specfile_path):
     suppress = config.get("config:suppress_gpg_warnings", False)
 
     try:
-        spack.util.gpg.verify(specfile_path, suppress_warnings=suppress)
+        spack.gpg.verify(specfile_path, suppress_warnings=suppress)
     except Exception:
         return False
 
@@ -2005,7 +2037,7 @@ def _extract_inner_tarball(spec, filename, extract_to, unsigned, remote_checksum
         if os.path.exists("%s.asc" % specfile_path):
             suppress = config.get("config:suppress_gpg_warnings", False)
             try:
-                spack.util.gpg.verify("%s.asc" % specfile_path, specfile_path, suppress)
+                spack.gpg.verify("%s.asc" % specfile_path, specfile_path, suppress)
             except Exception:
                 raise NoVerifyException(
                     "Spack was unable to verify package "
@@ -2211,7 +2243,8 @@ def install_root_node(spec, unsigned=False, force=False, sha256=None):
         tty.debug("Verified SHA256 checksum of the build cache")
 
     # don't print long padded paths while extracting/relocating binaries
-    with spack.util.path.filter_padding():
+    padding = spack.config.get("config:install_tree:padded_length", None)
+    with spack.util.path.filter_padding(padding=padding):
         tty.msg('Installing "{0}" from a buildcache'.format(spec.format()))
         extract_tarball(spec, download_result, unsigned, force)
         spack.hooks.post_install(spec, False)
@@ -2250,12 +2283,20 @@ def try_direct_fetch(spec, mirrors=None):
             mirror.fetch_url, BUILD_CACHE_RELATIVE_PATH, signed_specfile_name
         )
         try:
-            _, _, fs = web_util.read_from_url(buildcache_fetch_url_signed_json)
+            _, _, fs = web_util.read_from_url(
+                buildcache_fetch_url_signed_json,
+                verify_ssl=spack.config.get("config:verify_ssl", True),
+                timeout=spack.config.get("config:connect_timeout", 10),
+            )
             specfile_is_signed = True
-        except (URLError, web_util.SpackWebError, HTTPError) as url_err:
+        except (URLError, web_util.WebError, HTTPError) as url_err:
             try:
-                _, _, fs = web_util.read_from_url(buildcache_fetch_url_json)
-            except (URLError, web_util.SpackWebError, HTTPError) as url_err_x:
+                _, _, fs = web_util.read_from_url(
+                    buildcache_fetch_url_json,
+                    verify_ssl=spack.config.get("config:verify_ssl", True),
+                    timeout=spack.config.get("config:connect_timeout", 10),
+                )
+            except (URLError, web_util.WebError, HTTPError) as url_err_x:
                 tty.debug(
                     "Did not find {0} on {1}".format(
                         specfile_name, buildcache_fetch_url_signed_json
@@ -2359,10 +2400,19 @@ def get_keys(install=False, trust=False, force=False, mirrors=None):
         tty.debug("Finding public keys in {0}".format(url_util.format(fetch_url)))
 
         try:
-            _, _, json_file = web_util.read_from_url(keys_index)
+            _, _, json_file = web_util.read_from_url(
+                keys_index,
+                verify_ssl=spack.config.get("config:verify_ssl", True),
+                timeout=spack.config.get("config:connect_timeout", 10),
+            )
             json_index = sjson.load(codecs.getreader("utf-8")(json_file))
-        except (URLError, web_util.SpackWebError) as url_err:
-            if web_util.url_exists(keys_index):
+        except (URLError, web_util.WebError) as url_err:
+            if web_util.url_exists(
+                keys_index,
+                fetch_method=spack.config.get("config:url_fetch_method", "urllib"),
+                verify_ssl=spack.config.get("config:verify_ssl"),
+                timeout=spack.config.get("config:connect_timeout", 10),
+            ):
                 err_msg = [
                     "Unable to find public keys in {0},",
                     " caught exception attempting to read from {1}.",
@@ -2393,7 +2443,7 @@ def get_keys(install=False, trust=False, force=False, mirrors=None):
             tty.debug("Found key {0}".format(fingerprint))
             if install:
                 if trust:
-                    spack.util.gpg.trust(stage.save_filename)
+                    spack.gpg.trust(stage.save_filename)
                     tty.debug("Added this key to trusted keys.")
                 else:
                     tty.debug(
@@ -2411,7 +2461,7 @@ def push_keys(*mirrors, **kwargs):
     tmpdir = kwargs.get("tmpdir")
     remove_tmpdir = False
 
-    keys = spack.util.gpg.public_keys(*(keys or []))
+    keys = spack.gpg.public_keys(*(keys or []))
 
     try:
         for mirror in mirrors:
@@ -2443,7 +2493,7 @@ def push_keys(*mirrors, **kwargs):
                 export_target = os.path.join(prefix, filename)
 
                 # Export public keys (private is set to False)
-                spack.util.gpg.export_keys(export_target, [fingerprint])
+                spack.gpg.export_keys(export_target, [fingerprint])
 
                 # If mirror is local, the above export writes directly to the
                 # mirror (export_target points directly to the mirror).
@@ -2452,7 +2502,10 @@ def push_keys(*mirrors, **kwargs):
                 # uploaded to the mirror.
                 if not keys_local:
                     spack.util.web.push_to_url(
-                        export_target, url_util.join(keys_url, filename), keep_original=False
+                        export_target,
+                        url_util.join(keys_url, filename),
+                        keep_original=False,
+                        verify_ssl=spack.config.get("config:verify_ssl", True),
                     )
 
             if regenerate_index:
@@ -2486,7 +2539,12 @@ def needs_rebuild(spec, mirror_url):
     # Only check for the presence of the json version of the spec.  If the
     # mirror only has the json version, or doesn't have the spec at all, we
     # need to rebuild.
-    return not web_util.url_exists(specfile_path)
+    return not web_util.url_exists(
+        specfile_path,
+        fetch_method=spack.config.get("config:url_fetch_method", "urllib"),
+        verify_ssl=spack.config.get("config:verify_ssl"),
+        timeout=spack.config.get("config:connect_timeout", 10),
+    )
 
 
 def check_specs_against_mirrors(mirrors, specs, output_file=None):
@@ -2652,7 +2710,11 @@ def get_remote_hash(self):
         # Failure to fetch index.json.hash is not fatal
         url_index_hash = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json.hash")
         try:
-            response = self.urlopen(urllib.request.Request(url_index_hash, headers=self.headers))
+            response = self.urlopen(
+                urllib.request.Request(url_index_hash, headers=self.headers),
+                verify_ssl=spack.config.get("config:verify_ssl", True),
+                timeout=spack.config.get("config:connect_timeout", 10),
+            )
         except urllib.error.URLError:
             return None
 
@@ -2674,7 +2736,11 @@ def conditional_fetch(self) -> FetchIndexResult:
         url_index = url_util.join(self.url, BUILD_CACHE_RELATIVE_PATH, "index.json")
 
         try:
-            response = self.urlopen(urllib.request.Request(url_index, headers=self.headers))
+            response = self.urlopen(
+                urllib.request.Request(url_index, headers=self.headers),
+                verify_ssl=spack.config.get("config:verify_ssl", True),
+                timeout=spack.config.get("config:connect_timeout", 10),
+            )
         except urllib.error.URLError as e:
             raise FetchIndexError("Could not fetch index from {}".format(url_index), e) from e
 
@@ -2722,7 +2788,11 @@ def conditional_fetch(self) -> FetchIndexResult:
         }
 
         try:
-            response = self.urlopen(urllib.request.Request(url, headers=headers))
+            response = self.urlopen(
+                urllib.request.Request(url, headers=headers),
+                verify_ssl=spack.config.get("config:verify_ssl", True),
+                timeout=spack.config.get("config:connect_timeout", 10),
+            )
         except urllib.error.HTTPError as e:
             if e.getcode() == 304:
                 # Not modified; that means fresh.

lib/spack/spack/bootstrap/_common.py

@@ -16,7 +16,6 @@
 import llnl.util.filesystem as fs
 from llnl.util import tty
 
-import spack.platforms
 import spack.store
 import spack.util.environment
 import spack.util.executable
@@ -207,19 +206,17 @@ def _root_spec(spec_str: str) -> str:
     """Add a proper compiler and target to a spec used during bootstrapping.
 
     Args:
-        spec_str: spec to be bootstrapped. Must be without compiler and target.
+        spec_str (str): spec to be bootstrapped. Must be without compiler and target.
     """
-    # Add a compiler requirement to the root spec.
-    platform = str(spack.platforms.host())
-    if platform == "darwin":
+    # Add a proper compiler hint to the root spec. We use GCC for
+    # everything but MacOS and Windows.
+    if str(spack.platforms.host()) == "darwin":
         spec_str += " %apple-clang"
-    elif platform == "windows":
+    elif str(spack.platforms.host()) == "windows":
         # TODO (johnwparent): Remove version constraint when clingo patch is up
         spec_str += " %msvc@:19.37"
-    elif platform == "linux":
+    else:
         spec_str += " %gcc"
-    elif platform == "freebsd":
-        spec_str += " %clang"
 
     target = archspec.cpu.host().family
     spec_str += f" target={target}"

lib/spack/spack/bootstrap/config.py

@@ -45,7 +45,8 @@ def spec_for_current_python() -> str:
 def root_path() -> str:
     """Root of all the bootstrap related folders"""
     return spack.util.path.canonicalize_path(
-        spack.config.get("bootstrap:root", spack.paths.default_user_bootstrap_path)
+        spack.config.get("bootstrap:root", spack.paths.default_user_bootstrap_path),
+        replacements=spack.paths.path_replacements(),
     )
 
 
@@ -79,12 +80,16 @@ def spack_python_interpreter() -> Generator:
 
 def _store_path() -> str:
     bootstrap_root_path = root_path()
-    return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "store"))
+    return spack.util.path.canonicalize_path(
+        os.path.join(bootstrap_root_path, "store"), replacements=spack.paths.path_replacements()
+    )
 
 
 def _config_path() -> str:
     bootstrap_root_path = root_path()
-    return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "config"))
+    return spack.util.path.canonicalize_path(
+        os.path.join(bootstrap_root_path, "config"), replacements=spack.paths.path_replacements()
+    )
 
 
 @contextlib.contextmanager

lib/spack/spack/bootstrap/core.py

@@ -92,7 +92,9 @@ class Bootstrapper:
     def __init__(self, conf: ConfigDictionary) -> None:
         self.conf = conf
         self.name = conf["name"]
-        self.metadata_dir = spack.util.path.canonicalize_path(conf["metadata"])
+        self.metadata_dir = spack.util.path.canonicalize_path(
+            conf["metadata"], replacements=spack.paths.path_replacements()
+        )
 
         # Promote (relative) paths to file urls
         url = conf["info"]["url"]
@@ -386,7 +388,7 @@ def ensure_module_importable_or_raise(module: str, abstract_spec: Optional[str]
     exception_handler = GroupedExceptionHandler()
 
     for current_config in bootstrapping_sources():
-        with exception_handler.forward(current_config["name"], Exception):
+        with exception_handler.forward(current_config["name"]):
             source_is_enabled_or_raise(current_config)
             current_bootstrapper = create_bootstrapper(current_config)
             if current_bootstrapper.try_import(module, abstract_spec):
@@ -441,7 +443,7 @@ def ensure_executables_in_path_or_raise(
     exception_handler = GroupedExceptionHandler()
 
     for current_config in bootstrapping_sources():
-        with exception_handler.forward(current_config["name"], Exception):
+        with exception_handler.forward(current_config["name"]):
             source_is_enabled_or_raise(current_config)
             current_bootstrapper = create_bootstrapper(current_config)
             if current_bootstrapper.try_search_path(executables, abstract_spec):
@@ -585,7 +587,9 @@ def bootstrapping_sources(scope: Optional[str] = None):
     list_of_sources = []
     for entry in source_configs:
         current = copy.copy(entry)
-        metadata_dir = spack.util.path.canonicalize_path(entry["metadata"])
+        metadata_dir = spack.util.path.canonicalize_path(
+            entry["metadata"], replacements=spack.paths.path_replacements()
+        )
         metadata_yaml = os.path.join(metadata_dir, METADATA_YAML_FILENAME)
         with open(metadata_yaml, encoding="utf-8") as stream:
             current.update(spack.util.spack_yaml.load(stream))

lib/spack/spack/bootstrap/environment.py

@@ -16,9 +16,11 @@
 from llnl.util import tty
 
 import spack.environment
+import spack.paths
 import spack.tengine
 import spack.util.cpus
 import spack.util.executable
+from spack.environment import depfile
 
 from ._common import _root_spec
 from .config import root_path, spec_for_current_python, store_path
@@ -49,7 +51,8 @@ def environment_root(cls) -> pathlib.Path:
         environment_dir = f"{python_part}-{arch_part}-{interpreter_part}"
         return pathlib.Path(
             spack.util.path.canonicalize_path(
-                os.path.join(bootstrap_root_path, "environments", environment_dir)
+                os.path.join(bootstrap_root_path, "environments", environment_dir),
+                replacements=spack.paths.path_replacements(),
             )
         )
 
@@ -85,9 +88,12 @@ def __init__(self) -> None:
         super().__init__(self.environment_root())
 
     def update_installations(self) -> None:
-        """Update the installations of this environment."""
-        log_enabled = tty.is_debug() or tty.is_verbose()
-        with tty.SuppressOutput(msg_enabled=log_enabled, warn_enabled=log_enabled):
+        """Update the installations of this environment.
+
+        The update is done using a depfile on Linux and macOS, and using the ``install_all``
+        method of environments on Windows.
+        """
+        with tty.SuppressOutput(msg_enabled=False, warn_enabled=False):
             specs = self.concretize()
         if specs:
             colorized_specs = [
@@ -96,9 +102,11 @@ def update_installations(self) -> None:
             ]
             tty.msg(f"[BOOTSTRAPPING] Installing dependencies ({', '.join(colorized_specs)})")
             self.write(regenerate=False)
-            with tty.SuppressOutput(msg_enabled=log_enabled, warn_enabled=log_enabled):
+            if sys.platform == "win32":
                 self.install_all()
-                self.write(regenerate=True)
+            else:
+                self._install_with_depfile()
+            self.write(regenerate=True)
 
     def update_syspath_and_environ(self) -> None:
         """Update ``sys.path`` and the PATH, PYTHONPATH environment variables to point to
@@ -116,6 +124,27 @@ def update_syspath_and_environ(self) -> None:
             + [str(x) for x in self.pythonpaths()]
         )
 
+    def _install_with_depfile(self) -> None:
+        model = depfile.MakefileModel.from_env(self)
+        template = spack.tengine.make_environment().get_template(
+            os.path.join("depfile", "Makefile")
+        )
+        makefile = self.environment_root() / "Makefile"
+        makefile.write_text(template.render(model.to_dict()))
+        make = spack.util.executable.which("make")
+        kwargs = {}
+        if not tty.is_debug():
+            kwargs = {"output": os.devnull, "error": os.devnull}
+        make(
+            "-C",
+            str(self.environment_root()),
+            "-j",
+            str(
+                spack.util.cpus.determine_number_of_jobs(parallel=True, config=spack.config.CONFIG)
+            ),
+            **kwargs,
+        )
+
     def _write_spack_yaml_file(self) -> None:
         tty.msg(
             "[BOOTSTRAPPING] Spack has missing dependencies, creating a bootstrapping environment"

lib/spack/spack/bootstrap/status.py

@@ -66,6 +66,7 @@ def _core_requirements() -> List[RequiredResponseType]:
     _core_system_exes = {
         "make": _missing("make", "required to build software from sources"),
         "patch": _missing("patch", "required to patch source code before building"),
+        "bash": _missing("bash", "required for Spack compiler wrapper"),
         "tar": _missing("tar", "required to manage code archives"),
         "gzip": _missing("gzip", "required to compress/decompress code archives"),
         "unzip": _missing("unzip", "required to compress/decompress code archives"),

lib/spack/spack/build_systems/cached_cmake.py

@@ -9,7 +9,6 @@
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 
-import spack.build_environment
 import spack.builder
 
 from .cmake import CMakeBuilder, CMakePackage
@@ -286,19 +285,6 @@ def initconfig_hardware_entries(self):
     def std_initconfig_entries(self):
         cmake_prefix_path_env = os.environ["CMAKE_PREFIX_PATH"]
         cmake_prefix_path = cmake_prefix_path_env.replace(os.pathsep, ";")
-        cmake_rpaths_env = spack.build_environment.get_rpaths(self.pkg)
-        cmake_rpaths_path = ";".join(cmake_rpaths_env)
-        complete_rpath_list = cmake_rpaths_path
-        if "SPACK_COMPILER_EXTRA_RPATHS" in os.environ:
-            spack_extra_rpaths_env = os.environ["SPACK_COMPILER_EXTRA_RPATHS"]
-            spack_extra_rpaths_path = spack_extra_rpaths_env.replace(os.pathsep, ";")
-            complete_rpath_list = "{0};{1}".format(complete_rpath_list, spack_extra_rpaths_path)
-
-        if "SPACK_COMPILER_IMPLICIT_RPATHS" in os.environ:
-            spack_implicit_rpaths_env = os.environ["SPACK_COMPILER_IMPLICIT_RPATHS"]
-            spack_implicit_rpaths_path = spack_implicit_rpaths_env.replace(os.pathsep, ";")
-            complete_rpath_list = "{0};{1}".format(complete_rpath_list, spack_implicit_rpaths_path)
-
         return [
             "#------------------{0}".format("-" * 60),
             "# !!!! This is a generated file, edit at own risk !!!!",
@@ -306,9 +292,6 @@ def std_initconfig_entries(self):
             "# CMake executable path: {0}".format(self.pkg.spec["cmake"].command.path),
             "#------------------{0}\n".format("-" * 60),
             cmake_cache_string("CMAKE_PREFIX_PATH", cmake_prefix_path),
-            cmake_cache_string("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON"),
-            cmake_cache_string("CMAKE_BUILD_RPATH", complete_rpath_list),
-            cmake_cache_string("CMAKE_INSTALL_RPATH", complete_rpath_list),
             self.define_cmake_cache_from_variant("CMAKE_BUILD_TYPE", "build_type"),
         ]
 

lib/spack/spack/build_systems/python.py

@@ -6,14 +6,13 @@
 import os
 import re
 import shutil
-from typing import Iterable, List, Mapping, Optional
+from typing import Optional
 
 import archspec
 
 import llnl.util.filesystem as fs
 import llnl.util.lang as lang
 import llnl.util.tty as tty
-from llnl.util.filesystem import HeaderList, LibraryList
 
 import spack.builder
 import spack.config
@@ -26,18 +25,14 @@
 from spack.directives import build_system, depends_on, extends, maintainers
 from spack.error import NoHeadersError, NoLibrariesError
 from spack.install_test import test_part
-from spack.spec import Spec
-from spack.util.prefix import Prefix
 
 from ._checks import BaseBuilder, execute_install_time_tests
 
 
-def _flatten_dict(dictionary: Mapping[str, object]) -> Iterable[str]:
+def _flatten_dict(dictionary):
     """Iterable that yields KEY=VALUE paths through a dictionary.
-
     Args:
         dictionary: Possibly nested dictionary of arbitrary keys and values.
-
     Yields:
         A single path through the dictionary.
     """
@@ -55,7 +50,7 @@ class PythonExtension(spack.package_base.PackageBase):
     maintainers("adamjstewart")
 
     @property
-    def import_modules(self) -> Iterable[str]:
+    def import_modules(self):
         """Names of modules that the Python package provides.
 
         These are used to test whether or not the installation succeeded.
@@ -70,7 +65,7 @@ def import_modules(self) -> Iterable[str]:
         detected, this property can be overridden by the package.
 
         Returns:
-            List of strings of module names.
+            list: list of strings of module names
         """
         modules = []
         pkg = self.spec["python"].package
@@ -107,14 +102,14 @@ def import_modules(self) -> Iterable[str]:
         return modules
 
     @property
-    def skip_modules(self) -> Iterable[str]:
+    def skip_modules(self):
         """Names of modules that should be skipped when running tests.
 
         These are a subset of import_modules. If a module has submodules,
         they are skipped as well (meaning a.b is skipped if a is contained).
 
         Returns:
-            List of strings of module names.
+            list: list of strings of module names
         """
         return []
 
@@ -190,12 +185,12 @@ def remove_files_from_view(self, view, merge_map):
 
         view.remove_files(to_remove)
 
-    def test_imports(self) -> None:
+    def test_imports(self):
         """Attempts to import modules of the installed package."""
 
         # Make sure we are importing the installed modules,
         # not the ones in the source directory
-        python = inspect.getmodule(self).python  # type: ignore[union-attr]
+        python = inspect.getmodule(self).python
         for module in self.import_modules:
             with test_part(
                 self,
@@ -320,27 +315,24 @@ class PythonPackage(PythonExtension):
     py_namespace: Optional[str] = None
 
     @lang.classproperty
-    def homepage(cls) -> Optional[str]:  # type: ignore[override]
+    def homepage(cls):
         if cls.pypi:
             name = cls.pypi.split("/")[0]
-            return f"https://pypi.org/project/{name}/"
-        return None
+            return "https://pypi.org/project/" + name + "/"
 
     @lang.classproperty
-    def url(cls) -> Optional[str]:
+    def url(cls):
         if cls.pypi:
-            return f"https://files.pythonhosted.org/packages/source/{cls.pypi[0]}/{cls.pypi}"
-        return None
+            return "https://files.pythonhosted.org/packages/source/" + cls.pypi[0] + "/" + cls.pypi
 
     @lang.classproperty
-    def list_url(cls) -> Optional[str]:  # type: ignore[override]
+    def list_url(cls):
         if cls.pypi:
             name = cls.pypi.split("/")[0]
-            return f"https://pypi.org/simple/{name}/"
-        return None
+            return "https://pypi.org/simple/" + name + "/"
 
     @property
-    def headers(self) -> HeaderList:
+    def headers(self):
         """Discover header files in platlib."""
 
         # Remove py- prefix in package name
@@ -358,7 +350,7 @@ def headers(self) -> HeaderList:
         raise NoHeadersError(msg.format(self.spec.name, include, platlib))
 
     @property
-    def libs(self) -> LibraryList:
+    def libs(self):
         """Discover libraries in platlib."""
 
         # Remove py- prefix in package name
@@ -392,7 +384,7 @@ class PythonPipBuilder(BaseBuilder):
     install_time_test_callbacks = ["test"]
 
     @staticmethod
-    def std_args(cls) -> List[str]:
+    def std_args(cls):
         return [
             # Verbose
             "-vvv",
@@ -417,7 +409,7 @@ def std_args(cls) -> List[str]:
         ]
 
     @property
-    def build_directory(self) -> str:
+    def build_directory(self):
         """The root directory of the Python package.
 
         This is usually the directory containing one of the following files:
@@ -428,51 +420,51 @@ def build_directory(self) -> str:
         """
         return self.pkg.stage.source_path
 
-    def config_settings(self, spec: Spec, prefix: Prefix) -> Mapping[str, object]:
+    def config_settings(self, spec, prefix):
         """Configuration settings to be passed to the PEP 517 build backend.
 
         Requires pip 22.1 or newer for keys that appear only a single time,
         or pip 23.1 or newer if the same key appears multiple times.
 
         Args:
-            spec: Build spec.
-            prefix: Installation prefix.
+            spec (spack.spec.Spec): build spec
+            prefix (spack.util.prefix.Prefix): installation prefix
 
         Returns:
-            Possibly nested dictionary of KEY, VALUE settings.
+            dict: Possibly nested dictionary of KEY, VALUE settings
         """
         return {}
 
-    def install_options(self, spec: Spec, prefix: Prefix) -> Iterable[str]:
+    def install_options(self, spec, prefix):
         """Extra arguments to be supplied to the setup.py install command.
 
         Requires pip 23.0 or older.
 
         Args:
-            spec: Build spec.
-            prefix: Installation prefix.
+            spec (spack.spec.Spec): build spec
+            prefix (spack.util.prefix.Prefix): installation prefix
 
         Returns:
-            List of options.
+            list: list of options
         """
         return []
 
-    def global_options(self, spec: Spec, prefix: Prefix) -> Iterable[str]:
+    def global_options(self, spec, prefix):
         """Extra global options to be supplied to the setup.py call before the install
         or bdist_wheel command.
 
         Deprecated in pip 23.1.
 
         Args:
-            spec: Build spec.
-            prefix: Installation prefix.
+            spec (spack.spec.Spec): build spec
+            prefix (spack.util.prefix.Prefix): installation prefix
 
         Returns:
-            List of options.
+            list: list of options
         """
         return []
 
-    def install(self, pkg: PythonPackage, spec: Spec, prefix: Prefix) -> None:
+    def install(self, pkg, spec, prefix):
         """Install everything from build directory."""
 
         args = PythonPipBuilder.std_args(pkg) + [f"--prefix={prefix}"]

lib/spack/spack/build_systems/racket.py

@@ -10,6 +10,7 @@
 import llnl.util.tty as tty
 
 import spack.builder
+import spack.config
 from spack.build_environment import SPACK_NO_PARALLEL_MAKE
 from spack.directives import build_system, extends, maintainers
 from spack.package_base import PackageBase
@@ -93,7 +94,7 @@ def install(self, pkg, spec, prefix):
                 "--copy",
                 "-i",
                 "-j",
-                str(determine_number_of_jobs(parallel=parallel)),
+                str(determine_number_of_jobs(parallel=parallel, config=spack.config.CONFIG)),
                 "--",
                 os.getcwd(),
             ]

lib/spack/spack/caches.py

@@ -26,7 +26,7 @@ def misc_cache_location():
     providers and for which packages provide which tags.
     """
     path = spack.config.get("config:misc_cache", spack.paths.default_misc_cache_path)
-    return spack.util.path.canonicalize_path(path)
+    return spack.util.path.canonicalize_path(path, replacements=spack.paths.path_replacements())
 
 
 def _misc_cache():
@@ -49,7 +49,7 @@ def fetch_cache_location():
     path = spack.config.get("config:source_cache")
     if not path:
         path = spack.paths.default_fetch_cache_path
-    path = spack.util.path.canonicalize_path(path)
+    path = spack.util.path.canonicalize_path(path, replacements=spack.paths.path_replacements())
     return path
 
 

lib/spack/spack/ci.py

@@ -31,13 +31,13 @@
 import spack.binary_distribution as bindist
 import spack.config as cfg
 import spack.environment as ev
+import spack.gpg
 import spack.main
 import spack.mirror
 import spack.paths
 import spack.repo
 import spack.spec
 import spack.util.git
-import spack.util.gpg as gpg_util
 import spack.util.spack_yaml as syaml
 import spack.util.url as url_util
 import spack.util.web as web_util
@@ -1454,13 +1454,13 @@ def can_sign_binaries():
     """Utility method to determine if this spack instance is capable of
     signing binary packages.  This is currently only possible if the
     spack gpg keystore contains exactly one secret key."""
-    return len(gpg_util.signing_keys()) == 1
+    return len(spack.gpg.signing_keys()) == 1
 
 
 def can_verify_binaries():
     """Utility method to determin if this spack instance is capable (at
     least in theory) of verifying signed binaries."""
-    return len(gpg_util.public_keys()) >= 1
+    return len(spack.gpg.public_keys()) >= 1
 
 
 def _push_mirror_contents(input_spec, sign_binaries, mirror_url):
@@ -1756,7 +1756,11 @@ def reproduce_ci_job(url, work_dir, autostart, gpg_url, runtime):
 
     gpg_path = None
     if gpg_url:
-        gpg_path = web_util.fetch_url_text(gpg_url, dest_dir=os.path.join(work_dir, "_pgp"))
+        gpg_path = web_util.fetch_url_text(
+            gpg_url,
+            dest_dir=os.path.join(work_dir, "_pgp"),
+            fetch_method=spack.config.get("config:url_fetch_method"),
+        )
         rel_gpg_path = gpg_path.replace(work_dir, "").lstrip(os.path.sep)
 
     lock_file = fs.find(work_dir, "spack.lock")[0]
@@ -2113,7 +2117,11 @@ def write_broken_spec(url, pkg_name, stack_name, job_url, pipeline_url, spec_dic
         with open(file_path, "w") as fd:
             fd.write(syaml.dump(broken_spec_details))
         web_util.push_to_url(
-            file_path, url, keep_original=False, extra_args={"ContentType": "text/plain"}
+            file_path,
+            url,
+            keep_original=False,
+            extra_args={"ContentType": "text/plain"},
+            verify_ssl=spack.config.get("config:verify_ssl", True),
         )
     except Exception as err:
         # If there is an S3 error (e.g., access denied or connection
@@ -2130,8 +2138,12 @@ def read_broken_spec(broken_spec_url):
     object.
     """
     try:
-        _, _, fs = web_util.read_from_url(broken_spec_url)
-    except (URLError, web_util.SpackWebError, HTTPError):
+        _, _, fs = web_util.read_from_url(
+            broken_spec_url,
+            verify_ssl=cfg.get("config:verify_ssl", True),
+            timeout=cfg.get("config:connect_timeout", 10),
+        )
+    except (URLError, web_util.WebError, HTTPError):
         tty.warn("Unable to read broken spec from {0}".format(broken_spec_url))
         return None
 

lib/spack/spack/cmd/bootstrap.py

@@ -18,6 +18,7 @@
 import spack.config
 import spack.main
 import spack.mirror
+import spack.paths
 import spack.spec
 import spack.stage
 import spack.util.path
@@ -191,7 +192,9 @@ def _root(args):
 
     root = spack.config.get("bootstrap:root", default=None, scope=args.scope)
     if root:
-        root = spack.util.path.canonicalize_path(root)
+        root = spack.util.path.canonicalize_path(
+            root, replacements=spack.paths.path_replacements()
+        )
     print(root)
 
 
@@ -335,7 +338,9 @@ def _add(args):
         raise RuntimeError(msg.format(args.name))
 
     # Check that the metadata file exists
-    metadata_dir = spack.util.path.canonicalize_path(args.metadata_dir)
+    metadata_dir = spack.util.path.canonicalize_path(
+        args.metadata_dir, replacements=spack.paths.path_replacements()
+    )
     if not os.path.exists(metadata_dir) or not os.path.isdir(metadata_dir):
         raise RuntimeError('the directory "{0}" does not exist'.format(args.metadata_dir))
 
@@ -384,7 +389,9 @@ def _remove(args):
 
 
 def _mirror(args):
-    mirror_dir = spack.util.path.canonicalize_path(os.path.join(args.root_dir, LOCAL_MIRROR_DIR))
+    mirror_dir = spack.util.path.canonicalize_path(
+        os.path.join(args.root_dir, LOCAL_MIRROR_DIR), replacements=spack.paths.path_replacements()
+    )
 
     # TODO: Here we are adding gnuconfig manually, but this can be fixed
     # TODO: as soon as we have an option to add to a mirror all the possible
@@ -433,9 +440,24 @@ def write_metadata(subdir, metadata):
     instructions += cmd.format("local-sources", rel_directory)
     if args.binary_packages:
         abs_directory, rel_directory = write_metadata(subdir="binaries", metadata=BINARY_METADATA)
-        shutil.copy(spack.util.path.canonicalize_path(CLINGO_JSON), abs_directory)
-        shutil.copy(spack.util.path.canonicalize_path(GNUPG_JSON), abs_directory)
-        shutil.copy(spack.util.path.canonicalize_path(PATCHELF_JSON), abs_directory)
+        shutil.copy(
+            spack.util.path.canonicalize_path(
+                CLINGO_JSON, replacements=spack.paths.path_replacements()
+            ),
+            abs_directory,
+        )
+        shutil.copy(
+            spack.util.path.canonicalize_path(
+                GNUPG_JSON, replacements=spack.paths.path_replacements()
+            ),
+            abs_directory,
+        )
+        shutil.copy(
+            spack.util.path.canonicalize_path(
+                PATCHELF_JSON, replacements=spack.paths.path_replacements()
+            ),
+            abs_directory,
+        )
         instructions += cmd.format("local-binaries", rel_directory)
     print(instructions)
 

lib/spack/spack/cmd/buildcache.py

@@ -188,16 +188,14 @@ def setup_parser(subparser: argparse.ArgumentParser):
         default=lambda: spack.config.default_modify_scope(),
         help="configuration scope containing mirrors to check",
     )
-    # Unfortunately there are 3 ways to do the same thing here:
-    check_specs = check.add_mutually_exclusive_group()
-    check_specs.add_argument(
+    check_spec_or_specfile = check.add_mutually_exclusive_group(required=True)
+    check_spec_or_specfile.add_argument(
         "-s", "--spec", help="check single spec instead of release specs file"
     )
-    check_specs.add_argument(
+    check_spec_or_specfile.add_argument(
         "--spec-file",
         help="check single spec from json or yaml file instead of release specs file",
     )
-    arguments.add_common_arguments(check, ["specs"])
 
     check.set_defaults(func=check_fn)
 
@@ -815,24 +813,15 @@ def check_fn(args: argparse.Namespace):
     exit code is non-zero, then at least one of the indicated specs needs to be rebuilt
     """
     if args.spec_file:
-        specs_arg = (
-            args.spec_file if os.path.sep in args.spec_file else os.path.join(".", args.spec_file)
-        )
         tty.warn(
             "The flag `--spec-file` is deprecated and will be removed in Spack 0.22. "
-            f"Use `spack buildcache check {specs_arg}` instead."
+            "Use --spec instead."
         )
-    elif args.spec:
-        specs_arg = args.spec
-        tty.warn(
-            "The flag `--spec` is deprecated and will be removed in Spack 0.23. "
-            f"Use `spack buildcache check {specs_arg}` instead."
-        )
-    else:
-        specs_arg = args.specs
 
-    if specs_arg:
-        specs = _matching_specs(spack.cmd.parse_specs(specs_arg))
+    specs = spack.cmd.parse_specs(args.spec or args.spec_file)
+
+    if specs:
+        specs = _matching_specs(specs)
     else:
         specs = spack.cmd.require_active_env("buildcache check").all_specs()
 
@@ -929,7 +918,12 @@ def copy_buildcache_file(src_url, dest_url, local_path=None):
         try:
             temp_stage.create()
             temp_stage.fetch()
-            web_util.push_to_url(local_path, dest_url, keep_original=True)
+            web_util.push_to_url(
+                local_path,
+                dest_url,
+                keep_original=True,
+                verify_ssl=spack.config.get("config:verify_ssl", True),
+            )
         except spack.error.FetchError as e:
             # Expected, since we have to try all the possible extensions
             tty.debug("no such file: {0}".format(src_url))

lib/spack/spack/cmd/checksum.py

@@ -11,6 +11,7 @@
 from llnl.util import tty
 
 import spack.cmd
+import spack.config
 import spack.repo
 import spack.spec
 import spack.stage
@@ -275,4 +276,4 @@ def add_versions_to_package(pkg: PackageBase, version_lines: str):
         tty.msg(f"Open {filename} to review the additions.")
 
     if sys.stdout.isatty():
-        editor(filename)
+        editor(filename, debug=spack.config.get("config:debug"))

lib/spack/spack/cmd/ci.py

@@ -16,10 +16,9 @@
 import spack.cmd.buildcache as buildcache
 import spack.config as cfg
 import spack.environment as ev
-import spack.environment.depfile
+import spack.gpg
 import spack.hash_types as ht
 import spack.mirror
-import spack.util.gpg as gpg_util
 import spack.util.timer as timer
 import spack.util.url as url_util
 import spack.util.web as web_util
@@ -306,7 +305,7 @@ def ci_rebuild(args):
     # Fail early if signing is required but we don't have a signing key
     sign_binaries = require_signing is not None and require_signing.lower() == "true"
     if sign_binaries and not spack_ci.can_sign_binaries():
-        gpg_util.list(False, True)
+        spack.gpg.list(False, True)
         tty.die("SPACK_REQUIRE_SIGNING=True => spack must have exactly one signing key")
 
     # Construct absolute paths relative to current $CI_PROJECT_DIR
@@ -607,9 +606,7 @@ def ci_rebuild(args):
             "SPACK_INSTALL_FLAGS={}".format(args_to_string(deps_install_args)),
             "-j$(nproc)",
             "install-deps/{}".format(
-                spack.environment.depfile.MakefileSpec(job_spec).safe_format(
-                    "{name}-{version}-{hash}"
-                )
+                ev.depfile.MakefileSpec(job_spec).safe_format("{name}-{version}-{hash}")
             ),
         ],
         spack_cmd + ["install"] + root_install_args,
@@ -733,10 +730,17 @@ def ci_rebuild(args):
             broken_specs_url = ci_config["broken-specs-url"]
             just_built_hash = job_spec.dag_hash()
             broken_spec_path = url_util.join(broken_specs_url, just_built_hash)
-            if web_util.url_exists(broken_spec_path):
+            if web_util.url_exists(
+                broken_spec_path,
+                fetch_method=cfg.get("config:url_fetch_method", "urllib"),
+                verify_ssl=cfg.get("config:verify_ssl"),
+                timeout=cfg.get("config:connect_timeout", 10),
+            ):
                 tty.msg("Removing {0} from the list of broken specs".format(broken_spec_path))
                 try:
-                    web_util.remove_url(broken_spec_path)
+                    web_util.remove_url(
+                        broken_spec_path, verify_ssl=cfg.get("config:verify_ssl", True)
+                    )
                 except Exception as err:
                     # If there is an S3 error (e.g., access denied or connection
                     # error), the first non boto-specific class in the exception

lib/spack/spack/cmd/clean.py

@@ -14,6 +14,7 @@
 import spack.caches
 import spack.cmd.test
 import spack.config
+import spack.paths
 import spack.repo
 import spack.stage
 import spack.store
@@ -133,7 +134,9 @@ def clean(parser, args):
         remove_python_cache()
 
     if args.bootstrap:
-        bootstrap_prefix = spack.util.path.canonicalize_path(spack.config.get("bootstrap:root"))
+        bootstrap_prefix = spack.util.path.canonicalize_path(
+            spack.config.get("bootstrap:root"), replacements=spack.paths.path_replacements()
+        )
         msg = 'Removing bootstrapped software and configuration in "{0}"'
         tty.msg(msg.format(bootstrap_prefix))
         llnl.util.filesystem.remove_directory_contents(bootstrap_prefix)

lib/spack/spack/cmd/config.py

@@ -180,7 +180,7 @@ def config_edit(args):
     if args.print_file:
         print(config_file)
     else:
-        editor(config_file)
+        editor(config_file, debug=spack.config.get("config:debug"))
 
 
 def config_list(args):

lib/spack/spack/cmd/create.py

@@ -11,6 +11,7 @@
 import llnl.util.tty as tty
 from llnl.util.filesystem import mkdirp
 
+import spack.config
 import spack.repo
 import spack.stage
 import spack.util.web
@@ -986,4 +987,4 @@ def create(parser, args):
 
     # Optionally open up the new package file in your $EDITOR
     if not args.skip_editor:
-        editor(pkg_path)
+        editor(pkg_path, debug=spack.config.get("config:debug"))

lib/spack/spack/cmd/develop.py

@@ -8,6 +8,7 @@
 import llnl.util.tty as tty
 
 import spack.cmd
+import spack.paths
 import spack.spec
 import spack.util.path
 import spack.version
@@ -55,7 +56,9 @@ def develop(parser, args):
         # download all dev specs
         for name, entry in env.dev_specs.items():
             path = entry.get("path", name)
-            abspath = spack.util.path.canonicalize_path(path, default_wd=env.path)
+            abspath = spack.util.path.canonicalize_path(
+                path, default_wd=env.path, replacements=spack.paths.path_replacements()
+            )
 
             if os.path.exists(abspath):
                 msg = "Skipping developer download of %s" % entry["spec"]
@@ -86,7 +89,9 @@ def develop(parser, args):
 
     # default path is relative path to spec.name
     path = args.path or spec.name
-    abspath = spack.util.path.canonicalize_path(path, default_wd=env.path)
+    abspath = spack.util.path.canonicalize_path(
+        path, default_wd=env.path, replacements=spack.paths.path_replacements()
+    )
 
     # clone default: only if the path doesn't exist
     clone = args.clone

lib/spack/spack/cmd/edit.py

@@ -9,6 +9,7 @@
 import llnl.util.tty as tty
 
 import spack.cmd
+import spack.config
 import spack.paths
 import spack.repo
 from spack.spec import Spec
@@ -45,7 +46,7 @@ def edit_package(name, repo_path, namespace):
     else:
         raise spack.repo.UnknownPackageError(spec.name)
 
-    editor(path)
+    editor(path, debug=spack.config.get("config:debug"))
 
 
 def setup_parser(subparser):

lib/spack/spack/cmd/gpg.py

@@ -7,9 +7,9 @@
 import os
 
 import spack.binary_distribution
+import spack.gpg
 import spack.mirror
 import spack.paths
-import spack.util.gpg
 import spack.util.url
 from spack.cmd.common import arguments
 
@@ -129,40 +129,38 @@ def setup_parser(subparser):
 def gpg_create(args):
     """create a new key"""
     if args.export or args.secret:
-        old_sec_keys = spack.util.gpg.signing_keys()
+        old_sec_keys = spack.gpg.signing_keys()
 
     # Create the new key
-    spack.util.gpg.create(
-        name=args.name, email=args.email, comment=args.comment, expires=args.expires
-    )
+    spack.gpg.create(name=args.name, email=args.email, comment=args.comment, expires=args.expires)
     if args.export or args.secret:
-        new_sec_keys = set(spack.util.gpg.signing_keys())
+        new_sec_keys = set(spack.gpg.signing_keys())
         new_keys = new_sec_keys.difference(old_sec_keys)
 
     if args.export:
-        spack.util.gpg.export_keys(args.export, new_keys)
+        spack.gpg.export_keys(args.export, new_keys)
     if args.secret:
-        spack.util.gpg.export_keys(args.secret, new_keys, secret=True)
+        spack.gpg.export_keys(args.secret, new_keys, secret=True)
 
 
 def gpg_export(args):
     """export a gpg key, optionally including secret key"""
     keys = args.keys
     if not keys:
-        keys = spack.util.gpg.signing_keys()
-    spack.util.gpg.export_keys(args.location, keys, args.secret)
+        keys = spack.gpg.signing_keys()
+    spack.gpg.export_keys(args.location, keys, args.secret)
 
 
 def gpg_list(args):
     """list keys available in the keyring"""
-    spack.util.gpg.list(args.trusted, args.signing)
+    spack.gpg.list(args.trusted, args.signing)
 
 
 def gpg_sign(args):
     """sign a package"""
     key = args.key
     if key is None:
-        keys = spack.util.gpg.signing_keys()
+        keys = spack.gpg.signing_keys()
         if len(keys) == 1:
             key = keys[0]
         elif not keys:
@@ -173,12 +171,12 @@ def gpg_sign(args):
     if not output:
         output = args.spec[0] + ".asc"
     # TODO: Support the package format Spack creates.
-    spack.util.gpg.sign(key, " ".join(args.spec), output, args.clearsign)
+    spack.gpg.sign(key, " ".join(args.spec), output, args.clearsign)
 
 
 def gpg_trust(args):
     """add a key to the keyring"""
-    spack.util.gpg.trust(args.keyfile)
+    spack.gpg.trust(args.keyfile)
 
 
 def gpg_init(args):
@@ -191,12 +189,12 @@ def gpg_init(args):
         for filename in filenames:
             if not filename.endswith(".key"):
                 continue
-            spack.util.gpg.trust(os.path.join(root, filename))
+            spack.gpg.trust(os.path.join(root, filename))
 
 
 def gpg_untrust(args):
     """remove a key from the keyring"""
-    spack.util.gpg.untrust(args.signing, *args.keys)
+    spack.gpg.untrust(args.signing, *args.keys)
 
 
 def gpg_verify(args):
@@ -205,7 +203,7 @@ def gpg_verify(args):
     signature = args.signature
     if signature is None:
         signature = args.spec[0] + ".asc"
-    spack.util.gpg.verify(signature, " ".join(args.spec))
+    spack.gpg.verify(signature, " ".join(args.spec))
 
 
 def gpg_publish(args):

lib/spack/spack/cmd/info.py

@@ -327,7 +327,7 @@ def _variants_by_name_when(pkg):
     """Adaptor to get variants keyed by { name: { when: { [Variant...] } }."""
     # TODO: replace with pkg.variants_by_name(when=True) when unified directive dicts are merged.
     variants = {}
-    for name, (variant, whens) in sorted(pkg.variants.items()):
+    for name, (variant, whens) in pkg.variants.items():
         for when in whens:
             variants.setdefault(name, {}).setdefault(when, []).append(variant)
     return variants

lib/spack/spack/cmd/mirror.py

@@ -495,7 +495,9 @@ def mirror_destroy(args):
     elif args.mirror_url:
         mirror_url = args.mirror_url
 
-    web_util.remove_url(mirror_url, recursive=True)
+    web_util.remove_url(
+        mirror_url, recursive=True, verify_ssl=spack.config.get("config:verify_ssl", True)
+    )
 
 
 def mirror(parser, args):

lib/spack/spack/cmd/pkg.py

@@ -12,10 +12,10 @@
 from llnl.util.tty.colify import colify
 
 import spack.cmd
+import spack.package_hash as ph
 import spack.paths
 import spack.repo
 import spack.util.executable as exe
-import spack.util.package_hash as ph
 from spack.cmd.common import arguments
 
 description = "query packages associated with particular git revisions"

lib/spack/spack/cmd/repo.py

@@ -9,6 +9,7 @@
 import llnl.util.tty as tty
 
 import spack.config
+import spack.paths
 import spack.repo
 import spack.util.path
 from spack.cmd.common import arguments
@@ -83,7 +84,9 @@ def repo_add(args):
     path = args.path
 
     # real_path is absolute and handles substitution.
-    canon_path = spack.util.path.canonicalize_path(path)
+    canon_path = spack.util.path.canonicalize_path(
+        path, replacements=spack.paths.path_replacements()
+    )
 
     # check if the path exists
     if not os.path.exists(canon_path):
@@ -115,9 +118,13 @@ def repo_remove(args):
     namespace_or_path = args.namespace_or_path
 
     # If the argument is a path, remove that repository from config.
-    canon_path = spack.util.path.canonicalize_path(namespace_or_path)
+    canon_path = spack.util.path.canonicalize_path(
+        namespace_or_path, replacements=spack.paths.path_replacements()
+    )
     for repo_path in repos:
-        repo_canon_path = spack.util.path.canonicalize_path(repo_path)
+        repo_canon_path = spack.util.path.canonicalize_path(
+            repo_path, replacements=spack.paths.path_replacements()
+        )
         if canon_path == repo_canon_path:
             repos.remove(repo_path)
             spack.config.set("repos", repos, args.scope)

lib/spack/spack/cmd/tutorial.py

@@ -11,9 +11,9 @@
 
 import spack
 import spack.config
+import spack.gpg
 import spack.paths
 import spack.util.git
-import spack.util.gpg
 from spack.cmd.common import arguments
 from spack.util.spack_yaml import syaml_dict
 
@@ -76,7 +76,7 @@ def tutorial(parser, args):
     spack.config.set("mirrors", mirror_config, scope="user")
 
     tty.msg("Ensuring that we trust tutorial binaries", f"spack gpg trust {tutorial_key}")
-    spack.util.gpg.trust(tutorial_key)
+    spack.gpg.trust(tutorial_key)
 
     # Note that checkout MUST be last. It changes Spack under our feet.
     # If you don't put this last, you'll get import errors for the code

lib/spack/spack/cmd/unit_test.py

@@ -227,7 +227,9 @@ def unit_test(parser, args, unknown_args):
     # has been used, then test that extension.
     pytest_root = spack.paths.spack_root
     if args.extension:
-        pytest_root = spack.extensions.load_extension(args.extension)
+        target = args.extension
+        extensions = spack.extensions.get_extension_paths()
+        pytest_root = spack.extensions.path_for_extension(target, *extensions)
 
     # pytest.ini lives in the root of the spack repository.
     with llnl.util.filesystem.working_dir(pytest_root):

lib/spack/spack/compilers/cce.py

@@ -20,16 +20,16 @@ def __init__(self, *args, **kwargs):
             self.version_argument = "-V"
 
     # Subclasses use possible names of C compiler
-    cc_names = ["craycc"]
+    cc_names = ["craycc", "cc"]
 
     # Subclasses use possible names of C++ compiler
-    cxx_names = ["crayCC"]
+    cxx_names = ["crayCC", "CC"]
 
     # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["crayftn"]
+    f77_names = ["crayftn", "ftn"]
 
     # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["crayftn"]
+    fc_names = ["crayftn", "ftn"]
 
     # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
     suffixes = [r"-mp-\d\.\d"]

lib/spack/spack/concretize.py

@@ -31,6 +31,7 @@
 import spack.config
 import spack.environment
 import spack.error
+import spack.paths
 import spack.platforms
 import spack.repo
 import spack.spec
@@ -91,7 +92,9 @@ def concretize_develop(self, spec):
         if not dev_info:
             return False
 
-        path = spack.util.path.canonicalize_path(dev_info["path"], default_wd=env.path)
+        path = spack.util.path.canonicalize_path(
+            dev_info["path"], default_wd=env.path, replacements=spack.paths.path_replacements()
+        )
 
         if "dev_path" in spec.variants:
             assert spec.variants["dev_path"].value == path

lib/spack/spack/config.py

@@ -1451,7 +1451,9 @@ def fetch_remote_configs(url: str, dest_dir: str, skip_existing: bool = True) ->
     def _fetch_file(url):
         raw = raw_github_gitlab_url(url)
         tty.debug("Reading config from url {0}".format(raw))
-        return web_util.fetch_url_text(raw, dest_dir=dest_dir)
+        return web_util.fetch_url_text(
+            raw, dest_dir=dest_dir, fetch_method=CONFIG.get("config:url_fetch_method")
+        )
 
     if not url:
         raise ConfigFileError("Cannot retrieve configuration without a URL")

lib/spack/spack/detection/common.py

@@ -309,14 +309,10 @@ def find_windows_kit_roots() -> List[str]:
         return glob.glob(kit_base)
 
     @staticmethod
-    def find_windows_kit_bin_paths(
-        kit_base: Union[Optional[str], Optional[list]] = None
-    ) -> List[str]:
+    def find_windows_kit_bin_paths(kit_base: Optional[str] = None) -> List[str]:
         """Returns Windows kit bin directory per version"""
         kit_base = WindowsKitExternalPaths.find_windows_kit_roots() if not kit_base else kit_base
         assert kit_base, "Unexpectedly empty value for Windows kit base path"
-        if isinstance(kit_base, str):
-            kit_base = kit_base.split(";")
         kit_paths = []
         for kit in kit_base:
             kit_bin = os.path.join(kit, "bin")
@@ -324,14 +320,10 @@ def find_windows_kit_bin_paths(
         return kit_paths
 
     @staticmethod
-    def find_windows_kit_lib_paths(
-        kit_base: Union[Optional[str], Optional[list]] = None
-    ) -> List[str]:
+    def find_windows_kit_lib_paths(kit_base: Optional[str] = None) -> List[str]:
         """Returns Windows kit lib directory per version"""
         kit_base = WindowsKitExternalPaths.find_windows_kit_roots() if not kit_base else kit_base
         assert kit_base, "Unexpectedly empty value for Windows kit base path"
-        if isinstance(kit_base, str):
-            kit_base = kit_base.split(";")
         kit_paths = []
         for kit in kit_base:
             kit_lib = os.path.join(kit, "Lib")

lib/spack/spack/environment/environment.py

@@ -90,7 +90,8 @@
 def env_root_path():
     """Override default root path if the user specified it"""
     return spack.util.path.canonicalize_path(
-        spack.config.get("config:environments_root", default=default_env_path)
+        spack.config.get("config:environments_root", default=default_env_path),
+        replacements=spack.paths.path_replacements(),
     )
 
 
@@ -478,7 +479,9 @@ def __init__(
     ):
         self.base = base_path
         self.raw_root = root
-        self.root = spack.util.path.canonicalize_path(root, default_wd=base_path)
+        self.root = spack.util.path.canonicalize_path(
+            root, default_wd=base_path, replacements=spack.paths.path_replacements()
+        )
         self.projections = projections
         self.select = select
         self.exclude = exclude
@@ -493,7 +496,9 @@ def exclude_fn(self, spec):
 
     def update_root(self, new_path):
         self.raw_root = new_path
-        self.root = spack.util.path.canonicalize_path(new_path, default_wd=self.base)
+        self.root = spack.util.path.canonicalize_path(
+            new_path, default_wd=self.base, replacements=spack.paths.path_replacements()
+        )
 
     def __eq__(self, other):
         return all(
@@ -985,7 +990,9 @@ def included_config_scopes(self):
         missing = []
         for i, config_path in enumerate(reversed(includes)):
             # allow paths to contain spack config/environment variables, etc.
-            config_path = substitute_path_variables(config_path)
+            config_path = substitute_path_variables(
+                config_path, replacements=spack.paths.path_replacements()
+            )
 
             include_url = urllib.parse.urlparse(config_path)
 
@@ -1296,7 +1303,9 @@ def develop(self, spec: Spec, path: str, clone: bool = False) -> bool:
             # to be created, then copy it afterwards somewhere else. It would be
             # better if we can create the `source_path` directly into its final
             # destination.
-            abspath = spack.util.path.canonicalize_path(path, default_wd=self.path)
+            abspath = spack.util.path.canonicalize_path(
+                path, default_wd=self.path, replacements=spack.paths.path_replacements()
+            )
             pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)
             # We construct a package class ourselves, rather than asking for
             # Spec.package, since Spec only allows this when it is concrete

lib/spack/spack/extensions.py

@@ -6,18 +6,17 @@
 for Spack's command extensions.
 """
 import difflib
-import glob
 import importlib
 import os
 import re
 import sys
 import types
-from typing import List
 
 import llnl.util.lang
 
 import spack.config
 import spack.error
+import spack.paths
 import spack.util.path
 
 _extension_regexp = re.compile(r"spack-(\w[-\w]*)$")
@@ -77,15 +76,6 @@ def load_command_extension(command, path):
     if not os.path.exists(cmd_path):
         return None
 
-    ensure_extension_loaded(extension, path=path)
-
-    module = importlib.import_module(module_name)
-    sys.modules[module_name] = module
-
-    return module
-
-
-def ensure_extension_loaded(extension, *, path):
     def ensure_package_creation(name):
         package_name = "{0}.{1}".format(__name__, name)
         if package_name in sys.modules:
@@ -111,28 +101,17 @@ def ensure_package_creation(name):
     ensure_package_creation(extension)
     ensure_package_creation(extension + ".cmd")
 
+    module = importlib.import_module(module_name)
+    sys.modules[module_name] = module
 
-def load_extension(name: str) -> str:
-    """Loads a single extension into the 'spack.extensions' package.
-
-    Args:
-        name: name of the extension
-    """
-    extension_root = path_for_extension(name, paths=get_extension_paths())
-    ensure_extension_loaded(name, path=extension_root)
-    commands = glob.glob(
-        os.path.join(extension_root, extension_name(extension_root), "cmd", "*.py")
-    )
-    commands = [os.path.basename(x).rstrip(".py") for x in commands]
-    for command in commands:
-        load_command_extension(command, extension_root)
-    return extension_root
+    return module
 
 
 def get_extension_paths():
     """Return the list of canonicalized extension paths from config:extensions."""
     extension_paths = spack.config.get("config:extensions") or []
-    paths = [spack.util.path.canonicalize_path(p) for p in extension_paths]
+    r = spack.paths.path_replacements()
+    paths = [spack.util.path.canonicalize_path(p, replacements=r) for p in extension_paths]
     return paths
 
 
@@ -148,7 +127,7 @@ def get_command_paths():
     return command_paths
 
 
-def path_for_extension(target_name: str, *, paths: List[str]) -> str:
+def path_for_extension(target_name, *paths):
     """Return the test root dir for a given extension.
 
     Args:

lib/spack/spack/fetch_strategy.py

@@ -301,7 +301,12 @@ def fetch(self):
         url = None
         errors = []
         for url in self.candidate_urls:
-            if not web_util.url_exists(url):
+            if not web_util.url_exists(
+                url,
+                fetch_method=spack.config.get("config:url_fetch_method", "urllib"),
+                verify_ssl=spack.config.get("config:verify_ssl"),
+                timeout=spack.config.get("config:connect_timeout", 10),
+            ):
                 tty.debug("URL does not exist: " + url)
                 continue
 
@@ -338,8 +343,12 @@ def _fetch_urllib(self, url):
 
         # Run urllib but grab the mime type from the http headers
         try:
-            url, headers, response = web_util.read_from_url(url)
-        except web_util.SpackWebError as e:
+            url, headers, response = web_util.read_from_url(
+                url,
+                verify_ssl=spack.config.get("config:verify_ssl", True),
+                timeout=spack.config.get("config:connect_timeout", 10),
+            )
+        except web_util.WebError as e:
             # clean up archive on failure.
             if self.archive_file:
                 os.remove(self.archive_file)
@@ -385,7 +394,15 @@ def _fetch_curl(self, url):
 
             timeout = self.extra_options.get("timeout")
 
-        base_args = web_util.base_curl_fetch_args(url, timeout)
+        connect_timeout = spack.config.get("config:connect_timeout", 10)
+        if timeout:
+            timeout = max(int(timeout), int(connect_timeout))
+        else:
+            timeout = int(connect_timeout)
+
+        base_args = web_util.base_curl_fetch_args(
+            url, timeout=timeout, verify_ssl=spack.config.get("config:verify_ssl")
+        )
         curl_args = save_args + base_args + cookie_args
 
         # Run curl but grab the mime type from the http headers
@@ -403,7 +420,7 @@ def _fetch_curl(self, url):
 
             try:
                 web_util.check_curl_code(curl.returncode)
-            except spack.error.FetchError as err:
+            except web_util.WebError as err:
                 raise spack.fetch_strategy.FailedDownloadError(url, str(err))
 
         self._check_headers(headers)
@@ -463,7 +480,10 @@ def archive(self, destination):
             raise NoArchiveFileError("Cannot call archive() before fetching.")
 
         web_util.push_to_url(
-            self.archive_file, url_util.path_to_file_url(destination), keep_original=True
+            self.archive_file,
+            url_util.path_to_file_url(destination),
+            keep_original=True,
+            verify_ssl=spack.config.get("config:verify_ssl", True),
         )
 
     @_needs_stage
@@ -1330,7 +1350,11 @@ def fetch(self):
         basename = os.path.basename(parsed_url.path)
 
         with working_dir(self.stage.path):
-            _, headers, stream = web_util.read_from_url(self.url)
+            _, headers, stream = web_util.read_from_url(
+                self.url,
+                verify_ssl=spack.config.get("config:verify_ssl", True),
+                timeout=spack.config.get("config:connect_timeout", 10),
+            )
 
             with open(basename, "wb") as f:
                 shutil.copyfileobj(stream, f)
@@ -1377,7 +1401,11 @@ def fetch(self):
         basename = os.path.basename(parsed_url.path)
 
         with working_dir(self.stage.path):
-            _, headers, stream = web_util.read_from_url(self.url)
+            _, headers, stream = web_util.read_from_url(
+                self.url,
+                verify_ssl=spack.config.get("config:verify_ssl", True),
+                timeout=spack.config.get("config:connect_timeout", 10),
+            )
 
             with open(basename, "wb") as f:
                 shutil.copyfileobj(stream, f)

lib/spack/spack/gpg.py

@@ -8,10 +8,8 @@
 import os
 import re
 
-import spack.error
-import spack.paths
+import spack.util.error
 import spack.util.executable
-import spack.version
 
 #: Executable instance for "gpg", initialized lazily
 GPG = None
@@ -29,7 +27,7 @@ def clear():
     GPG, GPGCONF, SOCKET_DIR, GNUPGHOME = None, None, None, None
 
 
-def init(gnupghome=None, force=False):
+def init(gnupghome=None, force=False, gpg_path=None):
     """Initialize the global objects in the module, if not set.
 
     When calling any gpg executable, the GNUPGHOME environment
@@ -56,7 +54,7 @@ def init(gnupghome=None, force=False):
         return
 
     # Set the value of GNUPGHOME to be used in this module
-    GNUPGHOME = gnupghome or os.getenv("SPACK_GNUPGHOME") or spack.paths.gpg_path
+    GNUPGHOME = gnupghome or os.getenv("SPACK_GNUPGHOME") or gpg_path
 
     # Set the executable objects for "gpg" and "gpgconf"
     with spack.bootstrap.ensure_bootstrap_configuration():
@@ -165,7 +163,7 @@ def _get_unimported_public_keys(output):
     return keys
 
 
-class SpackGPGError(spack.error.SpackError):
+class SpackGPGError(spack.util.error.UtilityError):
     """Class raised when GPG errors are detected."""
 
 
@@ -334,11 +332,11 @@ def _verify_exe_or_raise(exe):
         raise SpackGPGError(msg)
 
     output = exe("--version", output=str)
-    match = re.search(r"^gpg(conf)? \(GnuPG\) (.*)$", output, re.M)
+    match = re.search(r"^gpg(conf)? \(GnuPG\) (\d+).*$", output, re.M)
     if not match:
         raise SpackGPGError('Could not determine "{0}" version'.format(exe.name))
 
-    if spack.version.Version(match.group(2)) < spack.version.Version("2"):
+    if int(match.group(2)) < 2:
         raise SpackGPGError(msg)
 
 

lib/spack/spack/hooks/licensing.py

@@ -9,6 +9,7 @@
 from llnl.util.filesystem import mkdirp
 from llnl.util.symlink import symlink
 
+import spack.config
 import spack.util.editor as ed
 
 
@@ -39,7 +40,7 @@ def set_up_license(pkg):
             write_license_file(pkg, license_path)
 
             # use spack.util.executable so the editor does not hang on return here
-            ed.editor(license_path, exec_fn=ed.executable)
+            ed.editor(license_path, exec_fn=ed.executable, debug=spack.config.get("config:debug"))
         else:
             # Use already existing license file
             tty.msg("Found already existing license %s" % license_path)

lib/spack/spack/hooks/permissions_setters.py

@@ -5,18 +5,18 @@
 
 import os
 
-import spack.util.file_permissions as fp
+import spack.package_prefs as pp
 
 
 def post_install(spec, explicit=None):
     if not spec.external:
-        fp.set_permissions_by_spec(spec.prefix, spec)
+        pp.set_permissions_by_spec(spec.prefix, spec)
 
         # os.walk explicitly set not to follow links
         for root, dirs, files in os.walk(spec.prefix, followlinks=False):
             for d in dirs:
                 if not os.path.islink(os.path.join(root, d)):
-                    fp.set_permissions_by_spec(os.path.join(root, d), spec)
+                    pp.set_permissions_by_spec(os.path.join(root, d), spec)
             for f in files:
                 if not os.path.islink(os.path.join(root, f)):
-                    fp.set_permissions_by_spec(os.path.join(root, f), spec)
+                    pp.set_permissions_by_spec(os.path.join(root, f), spec)

lib/spack/spack/install_test.py

@@ -91,7 +91,8 @@ def get_test_stage_dir():
             the default test stage path
     """
     return spack.util.path.canonicalize_path(
-        spack.config.get("config:test_stage", spack.paths.default_test_path)
+        spack.config.get("config:test_stage", spack.paths.default_test_path),
+        replacements=spack.paths.path_replacements(),
     )
 
 

lib/spack/spack/installer.py

@@ -357,8 +357,7 @@ def _print_installed_pkg(message: str) -> None:
     Args:
         message (str): message to be output
     """
-    if tty.msg_enabled():
-        print(colorize("@*g{[+]} ") + spack.util.path.debug_padded_filter(message))
+    print(colorize("@*g{[+]} ") + spack.util.path.debug_padded_filter(message))
 
 
 def print_install_test_log(pkg: "spack.package_base.PackageBase") -> None:
@@ -492,7 +491,8 @@ def _process_binary_cache_tarball(
 
     tty.msg(f"Extracting {package_id(pkg)} from binary cache")
 
-    with timer.measure("install"), spack.util.path.filter_padding():
+    padding = spack.config.get("config:install_tree:padded_length", None)
+    with timer.measure("install"), spack.util.path.filter_padding(padding=padding):
         binary_distribution.extract_tarball(
             pkg.spec, download_result, unsigned=unsigned, force=False, timer=timer
         )
@@ -2008,9 +2008,7 @@ def install(self) -> None:
 
         # Only enable the terminal status line when we're in a tty without debug info
         # enabled, so that the output does not get cluttered.
-        term_status = TermStatusLine(
-            enabled=sys.stdout.isatty() and tty.msg_enabled() and not tty.is_debug()
-        )
+        term_status = TermStatusLine(enabled=sys.stdout.isatty() and not tty.is_debug())
 
         while self.build_pq:
             task = self._pop_task()
@@ -2495,7 +2493,8 @@ def build_process(pkg: "spack.package_base.PackageBase", install_args: dict) ->
     installer = BuildProcessInstaller(pkg, install_args)
 
     # don't print long padded paths in executable debug output.
-    with spack.util.path.filter_padding():
+    padding = spack.config.get("config:install_tree:padded_length", None)
+    with spack.util.path.filter_padding(padding=padding):
         return installer.run()
 
 

lib/spack/spack/mirror.py

@@ -30,6 +30,7 @@
 import spack.fetch_strategy
 import spack.mirror
 import spack.oci.image
+import spack.paths
 import spack.spec
 import spack.util.path
 import spack.util.spack_json as sjson
@@ -51,7 +52,11 @@ def _url_or_path_to_url(url_or_path: str) -> str:
         return url_or_path
 
     # Otherwise we interpret it as path, and we should promote it to file:// URL.
-    return url_util.path_to_file_url(spack.util.path.canonicalize_path(url_or_path))
+    return url_util.path_to_file_url(
+        spack.util.path.canonicalize_path(
+            url_or_path, replacements=spack.paths.path_replacements()
+        )
+    )
 
 
 class Mirror:

lib/spack/spack/modules/common.py

@@ -46,6 +46,7 @@
 import spack.environment
 import spack.error
 import spack.modules.common
+import spack.package_prefs as pp
 import spack.paths
 import spack.projections as proj
 import spack.repo
@@ -54,7 +55,6 @@
 import spack.store
 import spack.tengine as tengine
 import spack.util.environment
-import spack.util.file_permissions as fp
 import spack.util.path
 import spack.util.spack_yaml as syaml
 from spack.context import Context
@@ -225,7 +225,7 @@ def root_path(name, module_set_name):
     roots = spack.config.merge_yaml(defaults, roots)
 
     path = roots.get(name, os.path.join(spack.paths.share_path, name))
-    return spack.util.path.canonicalize_path(path)
+    return spack.util.path.canonicalize_path(path, replacements=spack.paths.path_replacements())
 
 
 def generate_module_index(root, modules, overwrite=False):
@@ -968,7 +968,7 @@ def write(self, overwrite=False):
 
         # Set the file permissions of the module to match that of the package
         if os.path.exists(self.layout.filename):
-            fp.set_permissions_by_spec(self.layout.filename, self.spec)
+            pp.set_permissions_by_spec(self.layout.filename, self.spec)
 
         # Symlink defaults if needed
         self.update_module_defaults()

lib/spack/spack/operating_systems/__init__.py

@@ -5,20 +5,11 @@
 from ._operating_system import OperatingSystem
 from .cray_backend import CrayBackend
 from .cray_frontend import CrayFrontend
-from .freebsd import FreeBSDOs
 from .linux_distro import LinuxDistro
 from .mac_os import MacOs
 from .windows_os import WindowsOs
 
-__all__ = [
-    "OperatingSystem",
-    "LinuxDistro",
-    "MacOs",
-    "CrayFrontend",
-    "CrayBackend",
-    "WindowsOs",
-    "FreeBSDOs",
-]
+__all__ = ["OperatingSystem", "LinuxDistro", "MacOs", "CrayFrontend", "CrayBackend", "WindowsOs"]
 
 #: List of all the Operating Systems known to Spack
-operating_systems = [LinuxDistro, MacOs, CrayFrontend, CrayBackend, WindowsOs, FreeBSDOs]
+operating_systems = [LinuxDistro, MacOs, CrayFrontend, CrayBackend, WindowsOs]

lib/spack/spack/operating_systems/freebsd.py

@@ -1,15 +0,0 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import platform as py_platform
-
-from spack.version import Version
-
-from ._operating_system import OperatingSystem
-
-
-class FreeBSDOs(OperatingSystem):
-    def __init__(self):
-        release = py_platform.release().split("-", 1)[0]
-        super().__init__("freebsd", Version(release))

lib/spack/spack/package.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 # flake8: noqa: F401
-"""spack.util.package is a set of useful build tools and directives for packages.
+"""spack.package is a set of useful build tools and directives for packages.
 
 Everything in this module is automatically imported into Spack package files.
 """
@@ -101,7 +101,6 @@
     on_package_attributes,
 )
 from spack.spec import InvalidSpecDetected, Spec
-from spack.util.cpus import determine_number_of_jobs
 from spack.util.executable import *
 from spack.variant import (
     any_combination_of,

lib/spack/spack/package_base.py

@@ -63,9 +63,9 @@
     install_test_root,
 )
 from spack.installer import InstallError, PackageInstaller
+from spack.package_hash import package_hash
 from spack.stage import DIYStage, ResourceStage, Stage, StageComposite, compute_stage_name
 from spack.util.executable import ProcessError, which
-from spack.util.package_hash import package_hash
 from spack.version import GitVersion, StandardVersion, Version
 
 FLAG_HANDLER_RETURN_TYPE = Tuple[
@@ -829,7 +829,9 @@ def name(cls):
     @classproperty
     def global_license_dir(cls):
         """Returns the directory where license files for all packages are stored."""
-        return spack.util.path.canonicalize_path(spack.config.get("config:license_dir"))
+        return spack.util.path.canonicalize_path(
+            spack.config.get("config:license_dir"), replacements=spack.paths.path_replacements()
+        )
 
     @property
     def global_license_file(self):
@@ -985,7 +987,12 @@ def find_valid_url_for_version(self, version):
         urls = self.all_urls_for_version(version)
 
         for u in urls:
-            if spack.util.web.url_exists(u):
+            if spack.util.web.url_exists(
+                u,
+                fetch_method=spack.config.get("config:url_fetch_method", "urllib"),
+                verify_ssl=spack.config.get("config:verify_ssl"),
+                timeout=spack.config.get("config:connect_timeout", 10),
+            ):
                 return u
 
         return None

lib/spack/spack/package_prefs.py

@@ -2,11 +2,14 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os
 import stat
 import warnings
 
 import spack.error
+import spack.paths
 import spack.repo
+import spack.util.file_permissions as fp
 from spack.config import ConfigError
 from spack.util.path import canonicalize_path
 from spack.version import Version
@@ -177,7 +180,9 @@ def _package(maybe_abstract_spec):
             spec_str = entry["spec"]
             external_path = entry.get("prefix", None)
             if external_path:
-                external_path = canonicalize_path(external_path)
+                external_path = canonicalize_path(
+                    external_path, replacements=spack.paths.path_replacements()
+                )
             external_modules = entry.get("modules", None)
             external_spec = spack.spec.Spec.from_detection(
                 spack.spec.Spec(
@@ -294,5 +299,16 @@ def get_package_group(spec):
     return group
 
 
+def set_permissions_by_spec(path, spec):
+    # Get permissions for spec
+    if os.path.isdir(path):
+        perms = get_package_dir_permissions(spec)
+    else:
+        perms = get_package_permissions(spec)
+    group = get_package_group(spec)
+
+    fp.set_permissions(path, perms, group)
+
+
 class VirtualInPackagesYAMLError(spack.error.SpackError):
     """Raised when a disallowed virtual is found in packages.yaml"""

lib/spack/spack/paths.py

@@ -9,11 +9,16 @@
 throughout Spack and should bring in a minimal number of external
 dependencies.
 """
+import getpass
 import os
+import tempfile
+from datetime import date
 from pathlib import PurePath
 
 import llnl.util.filesystem
 
+from spack.util.path import NOMATCH
+
 #: This file lives in $prefix/lib/spack/spack/__file__
 prefix = str(PurePath(llnl.util.filesystem.ancestor(__file__, 4)))
 
@@ -136,3 +141,50 @@ def _get_system_config_path():
 
 #: System configuration location
 system_config_path = _get_system_config_path()
+
+
+def architecture():
+    # break circular import
+    import spack.platforms
+    import spack.spec
+
+    host_platform = spack.platforms.host()
+    host_os = host_platform.operating_system("default_os")
+    host_target = host_platform.target("default_target")
+
+    return spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))
+
+
+def get_user():
+    # User pwd where available because it accounts for effective uids when using ksu and similar
+    try:
+        # user pwd for unix systems
+        import pwd
+
+        return pwd.getpwuid(os.geteuid()).pw_name
+    except ImportError:
+        # fallback on getpass
+        return getpass.getuser()
+
+
+def path_replacements():
+    # break circular imports
+    import spack.environment as ev
+
+    arch = architecture()
+
+    return {
+        "spack": lambda: prefix,
+        "user": lambda: get_user(),
+        "tempdir": lambda: tempfile.gettempdir(),
+        "user_cache_path": lambda: user_cache_path,
+        "architecture": lambda: arch,
+        "arch": lambda: arch,
+        "platform": lambda: arch.platform,
+        "operating_system": lambda: arch.os,
+        "os": lambda: arch.os,
+        "target": lambda: arch.target,
+        "target_family": lambda: arch.target.microarchitecture.family,
+        "date": lambda: date.today().strftime("%Y-%m-%d"),
+        "env": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH,
+    }

lib/spack/spack/platforms/__init__.py

@@ -8,7 +8,6 @@
 from ._platform import Platform
 from .cray import Cray
 from .darwin import Darwin
-from .freebsd import FreeBSD
 from .linux import Linux
 from .test import Test
 from .windows import Windows
@@ -18,7 +17,6 @@
     "Cray",
     "Darwin",
     "Linux",
-    "FreeBSD",
     "Test",
     "Windows",
     "platforms",

lib/spack/spack/platforms/_functions.py

@@ -10,13 +10,12 @@
 
 from .cray import Cray
 from .darwin import Darwin
-from .freebsd import FreeBSD
 from .linux import Linux
 from .test import Test
 from .windows import Windows
 
 #: List of all the platform classes known to Spack
-platforms = [Cray, Darwin, Linux, Windows, FreeBSD, Test]
+platforms = [Cray, Darwin, Linux, Windows, Test]
 
 
 @llnl.util.lang.memoized

lib/spack/spack/platforms/freebsd.py

@@ -1,37 +0,0 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import platform
-
-import archspec.cpu
-
-import spack.target
-from spack.operating_systems.freebsd import FreeBSDOs
-
-from ._platform import Platform
-
-
-class FreeBSD(Platform):
-    priority = 102
-
-    def __init__(self):
-        super().__init__("freebsd")
-
-        for name in archspec.cpu.TARGETS:
-            self.add_target(name, spack.target.Target(name))
-
-        # Get specific default
-        self.default = archspec.cpu.host().name
-        self.front_end = self.default
-        self.back_end = self.default
-
-        os = FreeBSDOs()
-        self.default_os = str(os)
-        self.front_os = self.default_os
-        self.back_os = self.default_os
-        self.add_operating_system(str(os), os)
-
-    @classmethod
-    def detect(cls):
-        return platform.system().lower() == "freebsd"

lib/spack/spack/repo.py

@@ -37,6 +37,7 @@
 import spack.config
 import spack.error
 import spack.patch
+import spack.paths
 import spack.provider_index
 import spack.spec
 import spack.tag
@@ -928,7 +929,9 @@ def __init__(self, root, cache=None):
         """
         # Root directory, containing _repo.yaml and package dirs
         # Allow roots to by spack-relative by starting with '$spack'
-        self.root = spack.util.path.canonicalize_path(root)
+        self.root = spack.util.path.canonicalize_path(
+            root, replacements=spack.paths.path_replacements()
+        )
 
         # check and raise BadRepoError on fail.
         def check(condition, msg):
@@ -1327,7 +1330,7 @@ def create_repo(root, namespace=None, subdir=packages_dir_name):
     If the namespace is not provided, use basename of root.
     Return the canonicalized path and namespace of the created repository.
     """
-    root = spack.util.path.canonicalize_path(root)
+    root = spack.util.path.canonicalize_path(root, replacements=spack.paths.path_replacements())
     if not namespace:
         namespace = os.path.basename(root)
 

lib/spack/spack/reporters/cdash.py

@@ -92,12 +92,11 @@ def __init__(self, configuration: CDashConfiguration):
         self.osname = platform.system()
         self.osrelease = platform.release()
         self.target = spack.platforms.host().target("default_target")
-        self.starttime = int(time.time())
-        self.endtime = self.starttime
+        self.endtime = int(time.time())
         self.buildstamp = (
             configuration.buildstamp
             if configuration.buildstamp
-            else build_stamp(configuration.track, self.starttime)
+            else build_stamp(configuration.track, self.endtime)
         )
         self.buildIds: Dict[str, str] = {}
         self.revision = ""
@@ -126,7 +125,7 @@ def build_report_for_package(self, report_dir, package, duration):
             report_data[phase] = {}
             report_data[phase]["loglines"] = []
             report_data[phase]["status"] = 0
-            report_data[phase]["starttime"] = self.starttime
+            report_data[phase]["endtime"] = self.endtime
 
         # Track the phases we perform so we know what reports to create.
         # We always report the update step because this is how we tell CDash
@@ -154,25 +153,6 @@ def build_report_for_package(self, report_dir, package, duration):
             elif cdash_phase:
                 report_data[cdash_phase]["loglines"].append(xml.sax.saxutils.escape(line))
 
-        # something went wrong pre-cdash "configure" phase b/c we have an exception and only
-        # "update" was encounterd.
-        # dump the report in the configure line so teams can see what the issue is
-        if len(phases_encountered) == 1 and package["exception"]:
-            # TODO this mapping is not ideal since these are pre-configure errors
-            # we need to determine if a more appropriate cdash phase can be utilized
-            # for now we will add a message to the log explaining this
-            cdash_phase = "configure"
-            phases_encountered.append(cdash_phase)
-
-            log_message = (
-                "Pre-configure errors occured in Spack's process that terminated the "
-                "build process prematurely.\nSpack output::\n{0}".format(
-                    xml.sax.saxutils.escape(package["exception"])
-                )
-            )
-
-            report_data[cdash_phase]["loglines"].append(log_message)
-
         # Move the build phase to the front of the list if it occurred.
         # This supports older versions of CDash that expect this phase
         # to be reported before all others.
@@ -180,9 +160,9 @@ def build_report_for_package(self, report_dir, package, duration):
             build_pos = phases_encountered.index("build")
             phases_encountered.insert(0, phases_encountered.pop(build_pos))
 
-        self.endtime = self.starttime + duration
+        self.starttime = self.endtime - duration
         for phase in phases_encountered:
-            report_data[phase]["endtime"] = self.endtime
+            report_data[phase]["starttime"] = self.starttime
             report_data[phase]["log"] = "\n".join(report_data[phase]["loglines"])
             errors, warnings = parse_log_events(report_data[phase]["loglines"])
 
@@ -329,7 +309,7 @@ def test_report_for_package(self, report_dir, package, duration):
             self.buildname = "{0}-{1}".format(self.current_package_name, package["id"])
         else:
             self.buildname = self.report_build_name(self.current_package_name)
-        self.endtime = self.starttime + duration
+        self.starttime = self.endtime - duration
 
         report_data = self.initialize_report(report_dir)
         report_data["hostname"] = socket.gethostname()
@@ -374,7 +354,7 @@ def concretization_report(self, report_dir, msg):
         self.buildname = self.base_buildname
         report_data = self.initialize_report(report_dir)
         report_data["update"] = {}
-        report_data["update"]["starttime"] = self.starttime
+        report_data["update"]["starttime"] = self.endtime
         report_data["update"]["endtime"] = self.endtime
         report_data["update"]["revision"] = self.revision
         report_data["update"]["log"] = msg

lib/spack/spack/solver/asp.py

@@ -41,6 +41,7 @@
 import spack.error
 import spack.package_base
 import spack.package_prefs
+import spack.paths
 import spack.platforms
 import spack.repo
 import spack.spec
@@ -2602,7 +2603,11 @@ def setup(
             dev_specs = tuple(
                 spack.spec.Spec(info["spec"]).constrained(
                     "dev_path=%s"
-                    % spack.util.path.canonicalize_path(info["path"], default_wd=env.path)
+                    % spack.util.path.canonicalize_path(
+                        info["path"],
+                        default_wd=env.path,
+                        replacements=spack.paths.path_replacements(),
+                    )
                 )
                 for name, info in env.dev_specs.items()
             )
@@ -3119,7 +3124,9 @@ def _develop_specs_from_env(spec, env):
     if not dev_info:
         return
 
-    path = spack.util.path.canonicalize_path(dev_info["path"], default_wd=env.path)
+    path = spack.util.path.canonicalize_path(
+        dev_info["path"], default_wd=env.path, replacements=spack.paths.path_replacements()
+    )
 
     if "dev_path" in spec.variants:
         error_msg = (

lib/spack/spack/stage.py

@@ -150,7 +150,7 @@ def _resolve_paths(candidates):
     Adjustments involve removing extra $user from $tempdir if $tempdir includes
     $user and appending $user if it is not present in the path.
     """
-    temp_path = sup.canonicalize_path("$tempdir")
+    temp_path = sup.canonicalize_path("$tempdir", replacements=spack.paths.path_replacements())
     user = getpass.getuser()
     tmp_has_usr = user in temp_path.split(os.path.sep)
 
@@ -162,7 +162,7 @@ def _resolve_paths(candidates):
             path = path.replace("/$user", "", 1)
 
         # Ensure the path is unique per user.
-        can_path = sup.canonicalize_path(path)
+        can_path = sup.canonicalize_path(path, replacements=spack.paths.path_replacements())
         # When multiple users share a stage root, we can avoid conflicts between
         # them by adding a per-user subdirectory.
         # Avoid doing this on Windows to keep stage absolute path as short as possible.
@@ -199,9 +199,10 @@ def get_stage_root():
 def _mirror_roots():
     mirrors = spack.config.get("mirrors")
     return [
-        sup.substitute_path_variables(root)
+        sup.substitute_path_variables(root, replacements=spack.paths.path_replacements())
         if root.endswith(os.sep)
-        else sup.substitute_path_variables(root) + os.sep
+        else sup.substitute_path_variables(root, replacemnts=spack.paths.path_replacements())
+        + os.sep
         for root in mirrors.values()
     ]
 

lib/spack/spack/store.py

@@ -77,7 +77,9 @@ def parse_install_tree(config_dict):
     if isinstance(install_tree, str):
         tty.warn("Using deprecated format for configuring install_tree")
         unpadded_root = install_tree
-        unpadded_root = spack.util.path.canonicalize_path(unpadded_root)
+        unpadded_root = spack.util.path.canonicalize_path(
+            unpadded_root, replacements=spack.paths.path_replacements()
+        )
         # construct projection from previous values for backwards compatibility
         all_projection = config_dict.get(
             "install_path_scheme", spack.directory_layout.default_projections["all"]
@@ -86,7 +88,9 @@ def parse_install_tree(config_dict):
         projections = {"all": all_projection}
     else:
         unpadded_root = install_tree.get("root", DEFAULT_INSTALL_TREE_ROOT)
-        unpadded_root = spack.util.path.canonicalize_path(unpadded_root)
+        unpadded_root = spack.util.path.canonicalize_path(
+            unpadded_root, replacements=spack.paths.path_replacements()
+        )
 
         padded_length = install_tree.get("padded_length", False)
         if padded_length is True:
@@ -267,7 +271,9 @@ def _construct_upstream_dbs_from_install_roots(
     for install_root in reversed(install_roots):
         upstream_dbs = list(accumulated_upstream_dbs)
         next_db = spack.database.Database(
-            spack.util.path.canonicalize_path(install_root),
+            spack.util.path.canonicalize_path(
+                install_root, replacements=spack.paths.path_replacements()
+            ),
             is_upstream=True,
             upstream_dbs=upstream_dbs,
         )

lib/spack/spack/tengine.py

@@ -10,6 +10,7 @@
 
 import spack.config
 import spack.extensions
+import spack.paths
 from spack.util.path import canonicalize_path
 
 
@@ -76,7 +77,10 @@ def make_environment(dirs: Optional[Tuple[str, ...]] = None):
         # Default directories where to search for templates
         builtins = spack.config.get("config:template_dirs", ["$spack/share/spack/templates"])
         extensions = spack.extensions.get_template_dirs()
-        dirs = tuple(canonicalize_path(d) for d in itertools.chain(builtins, extensions))
+        r = spack.paths.path_replacements()
+        dirs = tuple(
+            canonicalize_path(d, replacements=r) for d in itertools.chain(builtins, extensions)
+        )
 
     # Loader for the templates
     loader = jinja2.FileSystemLoader(dirs)

lib/spack/spack/test/architecture.py

@@ -30,8 +30,6 @@ def current_host_platform():
         current_platform = spack.platforms.Darwin()
     elif "Windows" in platform.system():
         current_platform = spack.platforms.Windows()
-    elif "FreeBSD" in platform.system():
-        current_platform = spack.platforms.FreeBSD()
     return current_platform
 
 

lib/spack/spack/test/bindist.py

@@ -25,12 +25,12 @@
 import spack.caches
 import spack.config
 import spack.fetch_strategy
+import spack.gpg
 import spack.hooks.sbang as sbang
 import spack.main
 import spack.mirror
 import spack.repo
 import spack.store
-import spack.util.gpg
 import spack.util.spack_yaml as syaml
 import spack.util.url as url_util
 import spack.util.web as web_util
@@ -344,10 +344,10 @@ def test_push_and_fetch_keys(mock_gnupghome):
 
     # dir 1: create a new key, record its fingerprint, and push it to a new
     #        mirror
-    with spack.util.gpg.gnupghome_override(gpg_dir1):
-        spack.util.gpg.create(name="test-key", email="fake@test.key", expires="0", comment=None)
+    with spack.gpg.gnupghome_override(gpg_dir1):
+        spack.gpg.create(name="test-key", email="fake@test.key", expires="0", comment=None)
 
-        keys = spack.util.gpg.public_keys()
+        keys = spack.gpg.public_keys()
         assert len(keys) == 1
         fpr = keys[0]
 
@@ -355,12 +355,12 @@ def test_push_and_fetch_keys(mock_gnupghome):
 
     # dir 2: import the key from the mirror, and confirm that its fingerprint
     #        matches the one created above
-    with spack.util.gpg.gnupghome_override(gpg_dir2):
-        assert len(spack.util.gpg.public_keys()) == 0
+    with spack.gpg.gnupghome_override(gpg_dir2):
+        assert len(spack.gpg.public_keys()) == 0
 
         bindist.get_keys(mirrors=mirrors, install=True, trust=True, force=True)
 
-        new_keys = spack.util.gpg.public_keys()
+        new_keys = spack.gpg.public_keys()
         assert len(new_keys) == 1
         assert new_keys[0] == fpr
 
@@ -672,7 +672,7 @@ def test_etag_fetching_304():
     # Test conditional fetch with etags. If the remote hasn't modified the file
     # it returns 304, which is an HTTPError in urllib-land. That should be
     # handled as success, since it means the local cache is up-to-date.
-    def response_304(request: urllib.request.Request):
+    def response_304(request: urllib.request.Request, verify_ssl=True, timeout=10):
         url = request.get_full_url()
         if url == "https://www.example.com/build_cache/index.json":
             assert request.get_header("If-none-match") == '"112a8bbc1b3f7f185621c1ee335f0502"'
@@ -694,7 +694,7 @@ def response_304(request: urllib.request.Request):
 
 def test_etag_fetching_200():
     # Test conditional fetch with etags. The remote has modified the file.
-    def response_200(request: urllib.request.Request):
+    def response_200(request: urllib.request.Request, verify_ssl=True, timeout=10):
         url = request.get_full_url()
         if url == "https://www.example.com/build_cache/index.json":
             assert request.get_header("If-none-match") == '"112a8bbc1b3f7f185621c1ee335f0502"'
@@ -722,7 +722,7 @@ def response_200(request: urllib.request.Request):
 
 def test_etag_fetching_404():
     # Test conditional fetch with etags. The remote has modified the file.
-    def response_404(request: urllib.request.Request):
+    def response_404(request: urllib.request.Request, verify_ssl=True, timeout=10):
         raise urllib.error.HTTPError(
             request.get_full_url(),
             404,
@@ -745,7 +745,7 @@ def test_default_index_fetch_200():
     index_json = '{"Hello": "World"}'
     index_json_hash = bindist.compute_hash(index_json)
 
-    def urlopen(request: urllib.request.Request):
+    def urlopen(request: urllib.request.Request, **kwargs):
         url = request.get_full_url()
         if url.endswith("index.json.hash"):
             return urllib.response.addinfourl(  # type: ignore[arg-type]
@@ -784,7 +784,7 @@ def test_default_index_dont_fetch_index_json_hash_if_no_local_hash():
     index_json = '{"Hello": "World"}'
     index_json_hash = bindist.compute_hash(index_json)
 
-    def urlopen(request: urllib.request.Request):
+    def urlopen(request: urllib.request.Request, **kwargs):
         url = request.get_full_url()
         if url.endswith("index.json"):
             return urllib.response.addinfourl(
@@ -813,7 +813,7 @@ def test_default_index_not_modified():
     index_json = '{"Hello": "World"}'
     index_json_hash = bindist.compute_hash(index_json)
 
-    def urlopen(request: urllib.request.Request):
+    def urlopen(request: urllib.request.Request, **kwargs):
         url = request.get_full_url()
         if url.endswith("index.json.hash"):
             return urllib.response.addinfourl(
@@ -838,7 +838,7 @@ def test_default_index_invalid_hash_file(index_json):
     # Test invalid unicode / invalid hash type
     index_json_hash = bindist.compute_hash(index_json)
 
-    def urlopen(request: urllib.request.Request):
+    def urlopen(request: urllib.request.Request, **kwargs):
         return urllib.response.addinfourl(
             io.BytesIO(),
             headers={},  # type: ignore[arg-type]
@@ -858,7 +858,7 @@ def test_default_index_json_404():
     index_json = '{"Hello": "World"}'
     index_json_hash = bindist.compute_hash(index_json)
 
-    def urlopen(request: urllib.request.Request):
+    def urlopen(request: urllib.request.Request, **kwargs):
         url = request.get_full_url()
         if url.endswith("index.json.hash"):
             return urllib.response.addinfourl(

lib/spack/spack/test/bootstrap.py

@@ -10,6 +10,7 @@
 import spack.bootstrap.core
 import spack.compilers
 import spack.environment
+import spack.paths
 import spack.store
 import spack.util.path
 
@@ -81,7 +82,9 @@ def test_store_path_customization(config_value, expected, mutable_config):
 
     # Check the store path
     current = spack.bootstrap.config.store_path()
-    assert current == spack.util.path.canonicalize_path(expected)
+    assert current == spack.util.path.canonicalize_path(
+        expected, replacements=spack.paths.path_replacements()
+    )
 
 
 def test_raising_exception_if_bootstrap_disabled(mutable_config):

lib/spack/spack/test/ci.py

@@ -16,9 +16,9 @@
 import spack.config
 import spack.environment as ev
 import spack.error
+import spack.gpg
 import spack.paths as spack_paths
 import spack.util.git
-import spack.util.gpg
 import spack.util.spack_yaml as syaml
 
 

lib/spack/spack/test/cmd/ci.py

@@ -18,11 +18,11 @@
 import spack.ci as ci
 import spack.config
 import spack.environment as ev
+import spack.gpg
 import spack.hash_types as ht
 import spack.main
 import spack.paths as spack_paths
 import spack.repo as repo
-import spack.util.gpg
 import spack.util.spack_yaml as syaml
 import spack.util.url as url_util
 from spack.schema.buildcache_spec import schema as specfile_schema
@@ -1210,7 +1210,7 @@ def test_push_mirror_contents(
             dl_dir = working_dir.join("download_dir")
             if not os.path.exists(dl_dir.strpath):
                 os.makedirs(dl_dir.strpath)
-            buildcache_cmd("download", "--spec", json_path, "--path", dl_dir.strpath)
+            buildcache_cmd("download", "--spec-file", json_path, "--path", dl_dir.strpath)
             dl_dir_list = os.listdir(dl_dir.strpath)
 
             assert len(dl_dir_list) == 2

lib/spack/spack/test/cmd/develop.py

@@ -10,6 +10,7 @@
 import llnl.util.filesystem as fs
 
 import spack.environment as ev
+import spack.paths
 import spack.spec
 from spack.main import SpackCommand
 
@@ -106,7 +107,9 @@ def test_develop_canonicalize_path(self, monkeypatch, config):
         env("create", "test")
         with ev.read("test") as e:
             path = "../$user"
-            abspath = spack.util.path.canonicalize_path(path, e.path)
+            abspath = spack.util.path.canonicalize_path(
+                path, e.path, replacements=spack.paths.path_replacements()
+            )
 
             def check_path(stage, dest):
                 assert dest == abspath
@@ -123,7 +126,9 @@ def test_develop_canonicalize_path_no_args(self, monkeypatch, config):
         env("create", "test")
         with ev.read("test") as e:
             path = "$user"
-            abspath = spack.util.path.canonicalize_path(path, e.path)
+            abspath = spack.util.path.canonicalize_path(
+                path, e.path, replacements=spack.paths.path_replacements()
+            )
 
             def check_path(stage, dest):
                 assert dest == abspath

lib/spack/spack/test/cmd/env.py

@@ -904,7 +904,9 @@ def test_env_with_included_config_var_path(tmpdir, packages_file):
     spack_yaml = env_path / ev.manifest_name
     spack_yaml.write_text(mpileaks_env_config(config_var_path))
 
-    config_real_path = substitute_path_variables(config_var_path)
+    config_real_path = substitute_path_variables(
+        config_var_path, replacements=spack.paths.path_replacements()
+    )
     shutil.move(included_file, config_real_path)
     assert os.path.exists(config_real_path)
 

lib/spack/spack/test/cmd/gpg.py

@@ -10,8 +10,9 @@
 import llnl.util.filesystem as fs
 
 import spack.bootstrap
+import spack.gpg
+import spack.paths
 import spack.util.executable
-import spack.util.gpg
 from spack.main import SpackCommand
 from spack.paths import mock_gpg_data_path, mock_gpg_keys_path
 from spack.util.executable import ProcessError
@@ -45,19 +46,19 @@ def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch):
 
     monkeypatch.setenv("PATH", str(tmpdir))
     if version == "undetectable" or version.endswith("1.3.4"):
-        with pytest.raises(spack.util.gpg.SpackGPGError):
-            spack.util.gpg.init(force=True)
+        with pytest.raises(spack.gpg.SpackGPGError):
+            spack.gpg.init(force=True, gpg_path=spack.paths.gpg_path)
     else:
-        spack.util.gpg.init(force=True)
-        assert spack.util.gpg.GPG is not None
-        assert spack.util.gpg.GPGCONF is not None
+        spack.gpg.init(force=True, gpg_path=spack.paths.gpg_path)
+        assert spack.gpg.GPG is not None
+        assert spack.gpg.GPGCONF is not None
 
 
 def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch, mutable_config):
     monkeypatch.setenv("PATH", str(tmpdir))
     bootstrap("disable")
     with pytest.raises(RuntimeError):
-        spack.util.gpg.init(force=True)
+        spack.gpg.init(force=True, gpg_path=spack.paths.gpg_path)
 
 
 @pytest.mark.maybeslow
@@ -105,7 +106,7 @@ def test_gpg(tmpdir, mutable_config, mock_gnupghome):
         "Spack testing 1",
         "spack@googlegroups.com",
     )
-    keyfp = spack.util.gpg.signing_keys()[0]
+    keyfp = spack.gpg.signing_keys()[0]
 
     # List the keys.
     # TODO: Test the output here.

lib/spack/spack/test/config.py

@@ -338,50 +338,51 @@ def __init__(self, path):
 
 def test_substitute_config_variables(mock_low_high_config, monkeypatch):
     prefix = spack.paths.prefix.lstrip("/")
+    r = spack.paths.path_replacements()
     assert cross_plat_join(
         os.sep + os.path.join("foo", "bar", "baz"), prefix
-    ) == spack_path.canonicalize_path("/foo/bar/baz/$spack")
+    ) == spack_path.canonicalize_path("/foo/bar/baz/$spack", replacements=r)
 
     assert cross_plat_join(
         spack.paths.prefix, os.path.join("foo", "bar", "baz")
-    ) == spack_path.canonicalize_path("$spack/foo/bar/baz/")
+    ) == spack_path.canonicalize_path("$spack/foo/bar/baz/", replacements=r)
 
     assert cross_plat_join(
         os.sep + os.path.join("foo", "bar", "baz"), prefix, os.path.join("foo", "bar", "baz")
-    ) == spack_path.canonicalize_path("/foo/bar/baz/$spack/foo/bar/baz/")
+    ) == spack_path.canonicalize_path("/foo/bar/baz/$spack/foo/bar/baz/", replacements=r)
 
     assert cross_plat_join(
         os.sep + os.path.join("foo", "bar", "baz"), prefix
-    ) == spack_path.canonicalize_path("/foo/bar/baz/${spack}")
+    ) == spack_path.canonicalize_path("/foo/bar/baz/${spack}", replacements=r)
 
     assert cross_plat_join(
         spack.paths.prefix, os.path.join("foo", "bar", "baz")
-    ) == spack_path.canonicalize_path("${spack}/foo/bar/baz/")
+    ) == spack_path.canonicalize_path("${spack}/foo/bar/baz/", replacements=r)
 
     assert cross_plat_join(
         os.sep + os.path.join("foo", "bar", "baz"), prefix, os.path.join("foo", "bar", "baz")
-    ) == spack_path.canonicalize_path("/foo/bar/baz/${spack}/foo/bar/baz/")
+    ) == spack_path.canonicalize_path("/foo/bar/baz/${spack}/foo/bar/baz/", replacements=r)
 
     assert cross_plat_join(
         os.sep + os.path.join("foo", "bar", "baz"), prefix, os.path.join("foo", "bar", "baz")
-    ) != spack_path.canonicalize_path("/foo/bar/baz/${spack/foo/bar/baz/")
+    ) != spack_path.canonicalize_path("/foo/bar/baz/${spack/foo/bar/baz/", replacements=r)
 
     # $env replacement is a no-op when no environment is active
     assert spack_path.canonicalize_path(
-        os.sep + os.path.join("foo", "bar", "baz", "$env")
+        os.sep + os.path.join("foo", "bar", "baz", "$env"), replacements=r
     ) == os.sep + os.path.join("foo", "bar", "baz", "$env")
 
     # Fake an active environment and $env is replaced properly
     fake_env_path = os.sep + os.path.join("quux", "quuux")
     monkeypatch.setattr(ev, "active_environment", lambda: MockEnv(fake_env_path))
-    assert spack_path.canonicalize_path("$env/foo/bar/baz") == os.path.join(
+    assert spack_path.canonicalize_path("$env/foo/bar/baz", replacements=r) == os.path.join(
         fake_env_path, os.path.join("foo", "bar", "baz")
     )
 
     # relative paths without source information are relative to cwd
-    assert spack_path.canonicalize_path(os.path.join("foo", "bar", "baz")) == os.path.abspath(
-        os.path.join("foo", "bar", "baz")
-    )
+    assert spack_path.canonicalize_path(
+        os.path.join("foo", "bar", "baz"), replacements=r
+    ) == os.path.abspath(os.path.join("foo", "bar", "baz"))
 
     # relative paths with source information are relative to the file
     spack.config.set(
@@ -389,19 +390,19 @@ def test_substitute_config_variables(mock_low_high_config, monkeypatch):
     )
     spack.config.CONFIG.clear_caches()
     path = spack.config.get("modules:default:roots:lmod")
-    assert spack_path.canonicalize_path(path) == os.path.normpath(
+    assert spack_path.canonicalize_path(path, replacements=r) == os.path.normpath(
         os.path.join(mock_low_high_config.scopes["low"].path, os.path.join("foo", "bar", "baz"))
     )
 
     # test architecture information is in replacements
     assert spack_path.canonicalize_path(
-        os.path.join("foo", "$platform", "bar")
+        os.path.join("foo", "$platform", "bar"), replacements=r
     ) == os.path.abspath(os.path.join("foo", "test", "bar"))
 
     host_target = spack.platforms.host().target("default_target")
     host_target_family = str(host_target.microarchitecture.family)
     assert spack_path.canonicalize_path(
-        os.path.join("foo", "$target_family", "bar")
+        os.path.join("foo", "$target_family", "bar"), replacements=r
     ) == os.path.abspath(os.path.join("foo", host_target_family, "bar"))
 
 
@@ -438,28 +439,33 @@ def test_substitute_user(mock_low_high_config):
     assert os.sep + os.path.join(
         "foo", "bar"
     ) + os.sep + user + os.sep + "baz" == spack_path.canonicalize_path(
-        os.sep + os.path.join("foo", "bar", "$user", "baz")
+        os.sep + os.path.join("foo", "bar", "$user", "baz"),
+        replacements=spack.paths.path_replacements(),
     )
 
 
 def test_substitute_user_cache(mock_low_high_config):
     user_cache_path = spack.paths.user_cache_path
     assert user_cache_path + os.sep + "baz" == spack_path.canonicalize_path(
-        os.path.join("$user_cache_path", "baz")
+        os.path.join("$user_cache_path", "baz"), replacements=spack.paths.path_replacements()
     )
 
 
 def test_substitute_tempdir(mock_low_high_config):
     tempdir = tempfile.gettempdir()
-    assert tempdir == spack_path.canonicalize_path("$tempdir")
+    assert tempdir == spack_path.canonicalize_path(
+        "$tempdir", replacements=spack.paths.path_replacements()
+    )
     assert tempdir + os.sep + os.path.join("foo", "bar", "baz") == spack_path.canonicalize_path(
-        os.path.join("$tempdir", "foo", "bar", "baz")
+        os.path.join("$tempdir", "foo", "bar", "baz"), replacements=spack.paths.path_replacements()
     )
 
 
 def test_substitute_date(mock_low_high_config):
     test_path = os.path.join("hello", "world", "on", "$date")
-    new_path = spack_path.canonicalize_path(test_path)
+    new_path = spack_path.canonicalize_path(
+        test_path, replacements=spack.paths.path_replacements()
+    )
     assert "$date" in test_path
     assert date.today().strftime("%Y-%m-%d") in new_path
 

lib/spack/spack/test/conftest.py

@@ -39,6 +39,7 @@
 import spack.directory_layout
 import spack.environment as ev
 import spack.error
+import spack.gpg
 import spack.package_base
 import spack.package_prefs
 import spack.paths
@@ -50,7 +51,6 @@
 import spack.test.cray_manifest
 import spack.util.executable
 import spack.util.git
-import spack.util.gpg
 import spack.util.spack_yaml as syaml
 import spack.util.url as url_util
 from spack.fetch_strategy import URLFetchStrategy
@@ -1074,13 +1074,13 @@ def mock_gnupghome(monkeypatch):
     # This comes up because tmp paths on macOS are already long-ish, and
     # pytest makes them longer.
     try:
-        spack.util.gpg.init()
-    except spack.util.gpg.SpackGPGError:
-        if not spack.util.gpg.GPG:
+        spack.gpg.init(gpg_path=spack.paths.gpg_path)
+    except spack.gpg.SpackGPGError:
+        if not spack.gpg.GPG:
             pytest.skip("This test requires gpg")
 
     short_name_tmpdir = tempfile.mkdtemp()
-    with spack.util.gpg.gnupghome_override(short_name_tmpdir):
+    with spack.gpg.gnupghome_override(short_name_tmpdir):
         yield short_name_tmpdir
 
     # clean up, since we are doing this manually

lib/spack/spack/test/llnl/util/lang.py

@@ -321,18 +321,3 @@ def inner():
     """
         ).format(__file__)
     )
-
-
-def test_grouped_exception_base_type():
-    h = llnl.util.lang.GroupedExceptionHandler()
-
-    with h.forward("catch-runtime-error", RuntimeError):
-        raise NotImplementedError()
-
-    with pytest.raises(NotImplementedError):
-        with h.forward("catch-value-error", ValueError):
-            raise NotImplementedError()
-
-    message = h.grouped_message(with_tracebacks=False)
-    assert "catch-runtime-error" in message
-    assert "catch-value-error" not in message

lib/spack/spack/test/package_hash.py

@@ -9,9 +9,9 @@
 import pytest
 
 import spack.directives
+import spack.package_hash as ph
 import spack.paths
 import spack.repo
-import spack.util.package_hash as ph
 from spack.spec import Spec
 from spack.util.unparse import unparse
 

lib/spack/spack/test/packaging.py

@@ -21,10 +21,10 @@
 import spack.binary_distribution as bindist
 import spack.cmd.buildcache as buildcache
 import spack.error
+import spack.gpg
 import spack.package_base
 import spack.repo
 import spack.store
-import spack.util.gpg
 import spack.util.url as url_util
 from spack.fetch_strategy import URLFetchStrategy
 from spack.paths import mock_gpg_keys_path
@@ -72,7 +72,7 @@ def test_buildcache(mock_archive, tmp_path, monkeypatch, mutable_config):
 
         create_args = ["create", "-f", "--rebuild-index", mirror_path, pkghash]
         # Create a private key to sign package with if gpg2 available
-        spack.util.gpg.create(
+        spack.gpg.create(
             name="test key 1",
             expires="0",
             email="spack@googlegroups.com",

lib/spack/spack/test/stage.py

@@ -734,7 +734,7 @@ def test_resolve_paths(self):
         assert spack.stage._resolve_paths(paths) == paths
 
         tempdir = "$tempdir"
-        can_tempdir = canonicalize_path(tempdir)
+        can_tempdir = canonicalize_path(tempdir, replacements=spack.paths.path_replacements())
         user = getpass.getuser()
         temp_has_user = user in can_tempdir.split(os.sep)
         paths = [
@@ -744,7 +744,8 @@ def test_resolve_paths(self):
             os.path.join(tempdir, "$user", "stage", "$user"),
         ]
 
-        res_paths = [canonicalize_path(p) for p in paths]
+        r = spack.paths.path_replacements()
+        res_paths = [canonicalize_path(p, replacements=r) for p in paths]
         if temp_has_user:
             res_paths[1] = can_tempdir
             res_paths[2] = os.path.join(can_tempdir, user)

lib/spack/spack/test/tengine.py

@@ -7,6 +7,7 @@
 import pytest
 
 import spack.config
+import spack.paths
 import spack.tengine as tengine
 from spack.util.path import canonicalize_path
 
@@ -70,8 +71,9 @@ class TestTengineEnvironment:
     def test_template_retrieval(self):
         """Tests the template retrieval mechanism hooked into config files"""
         # Check the directories are correct
+        r = spack.paths.path_replacements()
         template_dirs = spack.config.get("config:template_dirs")
-        template_dirs = tuple([canonicalize_path(x) for x in template_dirs])
+        template_dirs = tuple([canonicalize_path(x, replacements=r) for x in template_dirs])
         assert len(template_dirs) == 3
 
         env = tengine.make_environment(template_dirs)

lib/spack/spack/test/url_fetch.py

@@ -350,8 +350,8 @@ def _which(*args, **kwargs):
 
 
 def test_url_fetch_text_without_url(tmpdir):
-    with pytest.raises(spack.error.FetchError, match="URL is required"):
-        web_util.fetch_url_text(None)
+    with pytest.raises(web_util.WebError, match="URL is required"):
+        web_util.fetch_url_text(None, fetch_method=spack.config.get("config:url_fetch_method"))
 
 
 def test_url_fetch_text_curl_failures(tmpdir, monkeypatch):
@@ -367,18 +367,20 @@ def _which(*args, **kwargs):
     monkeypatch.setattr(spack.util.web, "which", _which)
 
     with spack.config.override("config:url_fetch_method", "curl"):
-        with pytest.raises(spack.error.FetchError, match="Missing required curl"):
-            web_util.fetch_url_text("https://github.com/")
+        with pytest.raises(web_util.WebError, match="Missing required curl"):
+            web_util.fetch_url_text(
+                "https://github.com/", fetch_method=spack.config.get("config:url_fetch_method")
+            )
 
 
 def test_url_check_curl_errors():
     """Check that standard curl error returncodes raise expected errors."""
     # Check returncode 22 (i.e., 404)
-    with pytest.raises(spack.error.FetchError, match="not found"):
+    with pytest.raises(web_util.WebError, match="not found"):
         web_util.check_curl_code(22)
 
     # Check returncode 60 (certificate error)
-    with pytest.raises(spack.error.FetchError, match="invalid certificate"):
+    with pytest.raises(web_util.WebError, match="invalid certificate"):
         web_util.check_curl_code(60)
 
 
@@ -395,8 +397,11 @@ def _which(*args, **kwargs):
     monkeypatch.setattr(spack.util.web, "which", _which)
 
     with spack.config.override("config:url_fetch_method", "curl"):
-        with pytest.raises(spack.error.FetchError, match="Missing required curl"):
-            web_util.url_exists("https://github.com/")
+        with pytest.raises(web_util.WebError, match="Missing required curl"):
+            web_util.url_exists(
+                "https://github.com/",
+                fetch_method=spack.config.get("config:url_fetch_method", "urllib"),
+            )
 
 
 def test_url_fetch_text_urllib_bad_returncode(tmpdir, monkeypatch):
@@ -410,16 +415,20 @@ def _read_from_url(*args, **kwargs):
     monkeypatch.setattr(spack.util.web, "read_from_url", _read_from_url)
 
     with spack.config.override("config:url_fetch_method", "urllib"):
-        with pytest.raises(spack.error.FetchError, match="failed with error code"):
-            web_util.fetch_url_text("https://github.com/")
+        with pytest.raises(web_util.WebError, match="failed with error code"):
+            web_util.fetch_url_text(
+                "https://github.com/", fetch_method=spack.config.get("config:url_fetch_method")
+            )
 
 
 def test_url_fetch_text_urllib_web_error(tmpdir, monkeypatch):
     def _raise_web_error(*args, **kwargs):
-        raise web_util.SpackWebError("bad url")
+        raise web_util.WebError("bad url")
 
     monkeypatch.setattr(spack.util.web, "read_from_url", _raise_web_error)
 
     with spack.config.override("config:url_fetch_method", "urllib"):
-        with pytest.raises(spack.error.FetchError, match="fetch failed to verify"):
-            web_util.fetch_url_text("https://github.com/")
+        with pytest.raises(web_util.WebError, match="fetch failed to verify"):
+            web_util.fetch_url_text(
+                "https://github.com/", fetch_method=spack.config.get("config:url_fetch_method")
+            )

lib/spack/spack/test/util/path.py

@@ -79,25 +79,26 @@ def test_output_filtering(self, capfd, install_mockery, mutable_config):
         # test filtering when padding is enabled
         with spack.config.override("config:install_tree", {"padded_length": 256}):
             # tty.msg with filtering on the first argument
-            with sup.filter_padding():
+            padding = spack.config.get("config:install_tree:padded_length", None)
+            with sup.filter_padding(padding=padding):
                 tty.msg("here is a long path: %s/with/a/suffix" % long_path)
             out, err = capfd.readouterr()
             assert padding_string in out
 
             # tty.msg with filtering on a laterargument
-            with sup.filter_padding():
+            with sup.filter_padding(padding=padding):
                 tty.msg("here is a long path:", "%s/with/a/suffix" % long_path)
             out, err = capfd.readouterr()
             assert padding_string in out
 
             # tty.error with filtering on the first argument
-            with sup.filter_padding():
+            with sup.filter_padding(padding=padding):
                 tty.error("here is a long path: %s/with/a/suffix" % long_path)
             out, err = capfd.readouterr()
             assert padding_string in err
 
             # tty.error with filtering on a later argument
-            with sup.filter_padding():
+            with sup.filter_padding(padding=padding):
                 tty.error("here is a long path:", "%s/with/a/suffix" % long_path)
             out, err = capfd.readouterr()
             assert padding_string in err

lib/spack/spack/test/util/spack_lock_wrapper.py

@@ -11,6 +11,7 @@
 from llnl.util.filesystem import getuid, group_ids
 
 import spack.config
+import spack.util.error
 import spack.util.lock as lk
 
 
@@ -54,7 +55,7 @@ def test_lock_checks_user(tmpdir):
 
     # unsafe
     tmpdir.chmod(0o777)
-    with pytest.raises(spack.error.SpackError):
+    with pytest.raises(spack.util.error.UtilityError):
         lk.check_lock_safety(path)
 
     # safe
@@ -85,12 +86,12 @@ def test_lock_checks_group(tmpdir):
 
     # unsafe
     tmpdir.chmod(0o774)
-    with pytest.raises(spack.error.SpackError):
+    with pytest.raises(spack.util.error.UtilityError):
         lk.check_lock_safety(path)
 
     # unsafe
     tmpdir.chmod(0o777)
-    with pytest.raises(spack.error.SpackError):
+    with pytest.raises(spack.util.error.UtilityError):
         lk.check_lock_safety(path)
 
     # safe

lib/spack/spack/test/util/util_gpg.py

@@ -7,13 +7,14 @@
 
 import pytest
 
-import spack.util.gpg
+import spack.gpg
+import spack.paths
 
 
 @pytest.fixture()
 def has_socket_dir():
-    spack.util.gpg.init()
-    return bool(spack.util.gpg.SOCKET_DIR)
+    spack.gpg.init(gpg_path=spack.paths.gpg_path)
+    return bool(spack.gpg.SOCKET_DIR)
 
 
 def test_parse_gpg_output_case_one():
@@ -27,7 +28,7 @@ def test_parse_gpg_output_case_one():
 uid:::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA::Joe (Test) <j.s@s.com>:
 ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA::::::::::
 """
-    keys = spack.util.gpg._parse_secret_keys_output(output)
+    keys = spack.gpg._parse_secret_keys_output(output)
 
     assert len(keys) == 2
     assert keys[0] == "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
@@ -44,7 +45,7 @@ def test_parse_gpg_output_case_two():
 fpr:::::::::YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY:
 grp:::::::::AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA:
 """
-    keys = spack.util.gpg._parse_secret_keys_output(output)
+    keys = spack.gpg._parse_secret_keys_output(output)
 
     assert len(keys) == 1
     assert keys[0] == "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
@@ -63,7 +64,7 @@ def test_parse_gpg_output_case_three():
 ssb::2048:1:AAAAAAAAAAAAAAAA:AAAAAAAAAA::::::::::
 fpr:::::::::ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ:"""
 
-    keys = spack.util.gpg._parse_secret_keys_output(output)
+    keys = spack.gpg._parse_secret_keys_output(output)
 
     assert len(keys) == 2
     assert keys[0] == "WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW"
@@ -83,8 +84,8 @@ def test_really_long_gnupghome_dir(tmpdir, has_socket_dir):
     tdir = tdir[:N].rstrip(os.sep)
     tdir += "0" * (N - len(tdir))
 
-    with spack.util.gpg.gnupghome_override(tdir):
-        spack.util.gpg.create(
+    with spack.gpg.gnupghome_override(tdir):
+        spack.gpg.create(
             name="Spack testing 1", email="test@spack.io", comment="Spack testing key", expires="0"
         )
-        spack.util.gpg.list(True, True)
+        spack.gpg.list(True, True)

lib/spack/spack/test/web.py

@@ -287,7 +287,7 @@ def test_gather_s3_information(monkeypatch, capfd):
         }
     )
 
-    session_args, client_args = spack.util.s3.get_mirror_s3_connection_info(mirror, "push")
+    session_args, client_args = spack.util.s3.get_mirror_s3_connection_info(mirror, "push", False)
 
     # Session args are used to create the S3 Session object
     assert "aws_session_token" in session_args
@@ -307,7 +307,7 @@ def test_gather_s3_information(monkeypatch, capfd):
 def test_remove_s3_url(monkeypatch, capfd):
     fake_s3_url = "s3://my-bucket/subdirectory/mirror"
 
-    def get_s3_session(url, method="fetch"):
+    def get_s3_session(url, method="fetch", verify_ssl=True):
         return MockS3Client()
 
     monkeypatch.setattr(spack.util.web, "get_s3_session", get_s3_session)
@@ -315,7 +315,9 @@ def get_s3_session(url, method="fetch"):
     current_debug_level = tty.debug_level()
     tty.set_debug(1)
 
-    spack.util.web.remove_url(fake_s3_url, recursive=True)
+    spack.util.web.remove_url(
+        fake_s3_url, recursive=True, verify_ssl=spack.config.get("config:verify_ssl", True)
+    )
     err = capfd.readouterr()[1]
 
     tty.set_debug(current_debug_level)
@@ -326,16 +328,26 @@ def get_s3_session(url, method="fetch"):
 
 
 def test_s3_url_exists(monkeypatch, capfd):
-    def get_s3_session(url, method="fetch"):
+    def get_s3_session(url, method="fetch", verify_ssl=True):
         return MockS3Client()
 
     monkeypatch.setattr(spack.util.s3, "get_s3_session", get_s3_session)
 
     fake_s3_url_exists = "s3://my-bucket/subdirectory/my-file"
-    assert spack.util.web.url_exists(fake_s3_url_exists)
+    assert spack.util.web.url_exists(
+        fake_s3_url_exists,
+        fetch_method=spack.config.get("config:url_fetch_method", "urllib"),
+        verify_ssl=spack.config.get("config:verify_ssl"),
+        timeout=spack.config.get("config:connect_timeout", 10),
+    )
 
     fake_s3_url_does_not_exist = "s3://my-bucket/subdirectory/my-notfound-file"
-    assert not spack.util.web.url_exists(fake_s3_url_does_not_exist)
+    assert not spack.util.web.url_exists(
+        fake_s3_url_does_not_exist,
+        fetch_method=spack.config.get("config:url_fetch_method", "urllib"),
+        verify_ssl=spack.config.get("config:verify_ssl"),
+        timeout=spack.config.get("config:connect_timeout", 10),
+    )
 
 
 def test_s3_url_parsing():

lib/spack/spack/traverse.py

@@ -251,7 +251,7 @@ def traverse_depth_first_edges_generator(edges, visitor, post_order=False, root=
         neighbors = [EdgeAndDepth(edge=n, depth=edge.depth + 1) for n in visitor.neighbors(edge)]
 
         # This extra branch is just for efficiency.
-        if len(neighbors) > 0:
+        if len(neighbors) >= 0:
             for item in traverse_depth_first_edges_generator(
                 neighbors, visitor, post_order, root, depth
             ):

lib/spack/spack/util/compression.py

@@ -13,7 +13,7 @@
 import llnl.url
 from llnl.util import tty
 
-from spack.error import SpackError
+from spack.util.error import UtilityError
 from spack.util.executable import CommandNotFoundError, which
 
 try:
@@ -400,7 +400,7 @@ def decompressor_for_win(extension):
     compression_extension = llnl.url.compression_ext_from_compressed_archive(extension)
     decompressor = _determine_py_decomp_archive_strategy(compression_extension)
     if not decompressor:
-        raise SpackError(
+        raise UtilityError(
             "Spack was unable to determine a proper decompression strategy for"
             f"valid extension: {extension}"
             "This is a bug, please file an issue at https://github.com/spack/spack/issues"

lib/spack/spack/util/cpus.py

@@ -7,8 +7,6 @@
 import os
 from typing import Optional
 
-import spack.config
-
 
 def cpus_available():
     """
@@ -27,7 +25,8 @@ def determine_number_of_jobs(
     *,
     parallel: bool = False,
     max_cpus: int = cpus_available(),
-    config: Optional["spack.config.Configuration"] = None,
+    config=None,
+    config_path: str = "config:build_jobs",
 ) -> int:
     """
     Packages that require sequential builds need 1 job. Otherwise we use the
@@ -39,18 +38,24 @@ def determine_number_of_jobs(
         parallel: true when package supports parallel builds
         max_cpus: maximum number of CPUs to use (defaults to cpus_available())
         config: configuration object (defaults to global config)
+        config_path: configuration path to read number of jobs from
     """
     if not parallel:
         return 1
 
-    cfg = config or spack.config.CONFIG
-
     # Command line overrides all
+    config_jobs = 16
     try:
-        command_line = cfg.get("config:build_jobs", default=None, scope="command_line")
+        command_line = None
+        if config is not None:
+            command_line = config.get(config_path, default=None, scope="command_line")
+
         if command_line is not None:
             return command_line
     except ValueError:
         pass
 
-    return min(max_cpus, cfg.get("config:build_jobs", 16))
+    if config is not None:
+        config_jobs = config.get(config_path, 16)
+
+    return min(max_cpus, config_jobs)

lib/spack/spack/util/editor.py

@@ -18,7 +18,6 @@
 
 import llnl.util.tty as tty
 
-import spack.config
 import spack.util.executable
 
 #: editors to try if VISUAL and EDITOR are not set
@@ -61,7 +60,9 @@ def executable(exe: str, args: List[str]) -> int:
     return cmd.returncode
 
 
-def editor(*args: str, exec_fn: Callable[[str, List[str]], int] = os.execv) -> bool:
+def editor(
+    *args: str, exec_fn: Callable[[str, List[str]], int] = os.execv, debug: bool = False
+) -> bool:
     """Invoke the user's editor.
 
     This will try to execute the following, in order:
@@ -100,7 +101,7 @@ def try_exec(exe, args, var=None):
             return exec_fn(exe, args) == 0
 
         except (OSError, spack.util.executable.ProcessError) as e:
-            if spack.config.get("config:debug"):
+            if debug:
                 raise
 
             # Show variable we were trying to use, if it's from one

lib/spack/spack/util/error.py

@@ -0,0 +1,87 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import inspect
+import sys
+
+import llnl.util.tty as tty
+
+#: at what level we should write stack traces or short error messages
+#: this is module-scoped because it needs to be set very early
+debug = 0
+
+
+class UtilityError(Exception):
+    """This is the superclass for all Spack errors.
+    Subclasses can be found in the modules they have to do with.
+    """
+
+    def __init__(self, message, long_message=None):
+        super().__init__()
+        self.message = message
+        self._long_message = long_message
+
+        # for exceptions raised from child build processes, we save the
+        # traceback as a string and print it in the parent.
+        self.traceback = None
+
+        # we allow exceptions to print debug info via print_context()
+        # before they are caught at the top level. If they *haven't*
+        # printed context early, we do it by default when die() is
+        # called, so we need to remember whether it's been called.
+        self.printed = False
+
+    @property
+    def long_message(self):
+        return self._long_message
+
+    def print_context(self):
+        """Print extended debug information about this exception.
+
+        This is usually printed when the top-level Spack error handler
+        calls ``die()``, but it can be called separately beforehand if a
+        lower-level error handler needs to print error context and
+        continue without raising the exception to the top level.
+        """
+        if self.printed:
+            return
+
+        # basic debug message
+        tty.error(self.message)
+        if self.long_message:
+            sys.stderr.write(self.long_message)
+            sys.stderr.write("\n")
+
+        # stack trace, etc. in debug mode.
+        if debug:
+            if self.traceback:
+                # exception came from a build child, already got
+                # traceback in child, so print it.
+                sys.stderr.write(self.traceback)
+            else:
+                # run parent exception hook.
+                sys.excepthook(*sys.exc_info())
+
+        sys.stderr.flush()
+        self.printed = True
+
+    def die(self):
+        self.print_context()
+        sys.exit(1)
+
+    def __str__(self):
+        msg = self.message
+        if self._long_message:
+            msg += "\n    %s" % self._long_message
+        return msg
+
+    def __repr__(self):
+        args = [repr(self.message), repr(self.long_message)]
+        args = ",".join(args)
+        qualified_name = inspect.getmodule(self).__name__ + "." + type(self).__name__
+        return qualified_name + "(" + args + ")"
+
+    def __reduce__(self):
+        return type(self), (self.message, self.long_message)

lib/spack/spack/util/executable.py

@@ -11,7 +11,7 @@
 
 import llnl.util.tty as tty
 
-import spack.error
+import spack.util.error
 
 __all__ = ["Executable", "which", "ProcessError"]
 
@@ -362,7 +362,11 @@ def which(*args, **kwargs):
     return Executable(exe) if exe else None
 
 
-class ProcessError(spack.error.SpackError):
+class EditorError(spack.util.error.UtilityError):
+    """Base error for all errors from the executable utility"""
+
+
+class ProcessError(EditorError):
     """ProcessErrors are raised when Executables exit with an error code."""
 
 
@@ -371,5 +375,5 @@ class ProcessTimeoutError(ProcessError):
     specified timeout exceed that time"""
 
 
-class CommandNotFoundError(spack.error.SpackError):
+class CommandNotFoundError(EditorError):
     """Raised when ``which()`` can't find a required executable."""

lib/spack/spack/util/file_cache.py

@@ -10,7 +10,6 @@
 
 from llnl.util.filesystem import mkdirp, rename
 
-from spack.error import SpackError
 from spack.util.lock import Lock, ReadTransaction, WriteTransaction
 
 
@@ -178,5 +177,5 @@ def remove(self, key):
             lock.release_write()
 
 
-class CacheError(SpackError):
+class CacheError(Exception):
     pass

lib/spack/spack/util/file_permissions.py

@@ -8,19 +8,7 @@
 
 import llnl.util.filesystem as fs
 
-import spack.package_prefs as pp
-from spack.error import SpackError
-
-
-def set_permissions_by_spec(path, spec):
-    # Get permissions for spec
-    if os.path.isdir(path):
-        perms = pp.get_package_dir_permissions(spec)
-    else:
-        perms = pp.get_package_permissions(spec)
-    group = pp.get_package_group(spec)
-
-    set_permissions(path, perms, group)
+from spack.util.error import UtilityError
 
 
 def set_permissions(path, perms, group=None):
@@ -44,5 +32,5 @@ def set_permissions(path, perms, group=None):
         fs.chgrp(path, group, follow_symlinks=False)
 
 
-class InvalidPermissionsError(SpackError):
+class InvalidPermissionsError(UtilityError):
     """Error class for invalid permission setters"""

lib/spack/spack/util/lock.py

@@ -18,8 +18,7 @@
 from llnl.util.lock import ReadTransaction  # noqa: F401
 from llnl.util.lock import WriteTransaction  # noqa: F401
 
-import spack.error
-import spack.paths
+import spack.util.error
 
 
 class Lock(llnl.util.lock.Lock):
@@ -100,4 +99,4 @@ def check_lock_safety(path: str) -> None:
                 f"Running a shared spack without locks is unsafe. You must "
                 f"restrict permissions on {path} or enable locks."
             )
-            raise spack.error.SpackError(msg, long_msg)
+            raise spack.util.error.UtilityError(msg, long_msg)

lib/spack/spack/util/naming.py

@@ -8,7 +8,7 @@
 import re
 import string
 
-import spack.error
+import spack.util.error
 
 __all__ = [
     "mod_to_class",
@@ -159,7 +159,7 @@ def validate_fully_qualified_module_name(mod_name):
         raise InvalidFullyQualifiedModuleNameError(mod_name)
 
 
-class InvalidModuleNameError(spack.error.SpackError):
+class InvalidModuleNameError(spack.util.error.UtilityError):
     """Raised when we encounter a bad module name."""
 
     def __init__(self, name):
@@ -167,7 +167,7 @@ def __init__(self, name):
         self.name = name
 
 
-class InvalidFullyQualifiedModuleNameError(spack.error.SpackError):
+class InvalidFullyQualifiedModuleNameError(spack.util.error.UtilityError):
     """Raised when we encounter a bad full package name."""
 
     def __init__(self, name):

lib/spack/spack/util/path.py

@@ -8,75 +8,27 @@
 TODO: this is really part of spack.config. Consolidate it.
 """
 import contextlib
-import getpass
 import os
 import re
 import subprocess
 import sys
-import tempfile
-from datetime import date
 
 import llnl.util.tty as tty
 from llnl.util.lang import memoized
 
 import spack.util.spack_yaml as syaml
 
-__all__ = ["substitute_config_variables", "substitute_path_variables", "canonicalize_path"]
-
-
-def architecture():
-    # break circular import
-    import spack.platforms
-    import spack.spec
-
-    host_platform = spack.platforms.host()
-    host_os = host_platform.operating_system("default_os")
-    host_target = host_platform.target("default_target")
-
-    return spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))
-
-
-def get_user():
-    # User pwd where available because it accounts for effective uids when using ksu and similar
-    try:
-        # user pwd for unix systems
-        import pwd
-
-        return pwd.getpwuid(os.geteuid()).pw_name
-    except ImportError:
-        # fallback on getpass
-        return getpass.getuser()
+__all__ = [
+    "substitute_config_variables",
+    "substitute_path_variables",
+    "canonicalize_path",
+    "NOMATCH",
+]
 
 
 # return value for replacements with no match
 NOMATCH = object()
 
-
-# Substitutions to perform
-def replacements():
-    # break circular imports
-    import spack.environment as ev
-    import spack.paths
-
-    arch = architecture()
-
-    return {
-        "spack": lambda: spack.paths.prefix,
-        "user": lambda: get_user(),
-        "tempdir": lambda: tempfile.gettempdir(),
-        "user_cache_path": lambda: spack.paths.user_cache_path,
-        "architecture": lambda: arch,
-        "arch": lambda: arch,
-        "platform": lambda: arch.platform,
-        "operating_system": lambda: arch.os,
-        "os": lambda: arch.os,
-        "target": lambda: arch.target,
-        "target_family": lambda: arch.target.microarchitecture.family,
-        "date": lambda: date.today().strftime("%Y-%m-%d"),
-        "env": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH,
-    }
-
-
 # This is intended to be longer than the part of the install path
 # spack generates from the root path we give it.  Included in the
 # estimate:
@@ -144,7 +96,7 @@ def get_system_path_max():
     return sys_max_path_length
 
 
-def substitute_config_variables(path):
+def substitute_config_variables(path, replacements={}):
     """Substitute placeholders into paths.
 
     Spack allows paths in configs to have some placeholders, as follows:
@@ -168,22 +120,21 @@ def substitute_config_variables(path):
     replaced if there is an active environment, and should only be used in
     environment yaml files.
     """
-    _replacements = replacements()
 
     # Look up replacements
     def repl(match):
         m = match.group(0)
         key = m.strip("${}").lower()
-        repl = _replacements.get(key, lambda: m)()
+        repl = replacements.get(key, lambda: m)()
         return m if repl is NOMATCH else str(repl)
 
     # Replace $var or ${var}.
     return re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path)
 
 
-def substitute_path_variables(path):
+def substitute_path_variables(path, replacements={}):
     """Substitute config vars, expand environment vars, expand user home."""
-    path = substitute_config_variables(path)
+    path = substitute_config_variables(path, replacements=replacements)
     path = os.path.expandvars(path)
     path = os.path.expanduser(path)
     return path
@@ -225,7 +176,7 @@ def add_padding(path, length):
     return os.path.join(path, padding)
 
 
-def canonicalize_path(path, default_wd=None):
+def canonicalize_path(path, default_wd=None, replacements=None):
     """Same as substitute_path_variables, but also take absolute path.
 
     If the string is a yaml object with file annotations, make absolute paths
@@ -234,6 +185,7 @@ def canonicalize_path(path, default_wd=None):
 
     Arguments:
         path (str): path being converted as needed
+        replacements (dict): dictionary of replacements to use
 
     Returns:
         (str): An absolute path with path variable substitution
@@ -245,7 +197,18 @@ def canonicalize_path(path, default_wd=None):
         filename = os.path.dirname(path._start_mark.name)
         assert path._start_mark.name == path._end_mark.name
 
-    path = substitute_path_variables(path)
+    if replacements is None:
+        _replacements = {}
+    else:
+        _replacements = replacements
+
+    if not isinstance(_replacements, dict):
+        tty.die(
+            "Replacements returned by replacements func are of type"
+            f"{type(replacements)} and not of the expected type of dict."
+        )
+
+    path = substitute_path_variables(path, replacements=_replacements)
     if not os.path.isabs(path):
         if filename:
             path = os.path.join(filename, path)
@@ -326,15 +289,12 @@ def replacer(match):
 
 
 @contextlib.contextmanager
-def filter_padding():
+def filter_padding(padding=None):
     """Context manager to safely disable path padding in all Spack output.
 
     This is needed because Spack's debug output gets extremely long when we use a
     long padded installation path.
     """
-    import spack.config
-
-    padding = spack.config.get("config:install_tree:padded_length", None)
     if padding:
         # filter out all padding from the intsall command output
         with tty.output_filter(padding_filter):