py-snakemake-{executor,storage}-plugin-{htcondor,xrootd}: new packages

2024-12-20 11:21:28 -06:00
9459 changed files with 23499 additions and 16456 deletions
--- a/.github/workflows/bin/setup_git.ps1
+++ b/.github/workflows/bin/setup_git.ps1
@@ -1,3 +1,5 @@
+# (c) 2022 Lawrence Livermore National Laboratory
+
 git config --global user.email "spack@example.com"
 git config --global user.name "Test User"
 git config --global core.longpaths true
--- a/.github/workflows/bootstrap.yml
+++ b/.github/workflows/bootstrap.yml
@@ -161,7 +161,11 @@ jobs:
          source share/spack/setup-env.sh
          spack -d gpg list
          tree $HOME/.spack/bootstrap/store/
-
+      - name: Bootstrap File
+        run: |
+          source share/spack/setup-env.sh
+          spack -d python share/spack/qa/bootstrap-file.py
+          tree $HOME/.spack/bootstrap/store/

  windows:
    runs-on: "windows-latest"
@@ -192,3 +196,9 @@ jobs:
          spack -d gpg list
          ./share/spack/qa/validate_last_exit.ps1
          tree $env:userprofile/.spack/bootstrap/store/
+      - name: Bootstrap File
+        run: |
+          ./share/spack/setup-env.ps1
+          spack -d python share/spack/qa/bootstrap-file.py
+          ./share/spack/qa/validate_last_exit.ps1
+          tree $env:userprofile/.spack/bootstrap/store/
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -29,7 +29,7 @@ jobs:
    - run: coverage xml

    - name: "Upload coverage report to CodeCov"
-      uses: codecov/codecov-action@1e68e06f1dbfde0e4cefc87efeba9e4643565303
+      uses: codecov/codecov-action@05f5a9cfad807516dbbef9929c4a42df3eb78766
      with:
        verbose: true
        fail_ci_if_error: false
--- a/.github/workflows/requirements/style/requirements.txt
+++ b/.github/workflows/requirements/style/requirements.txt
@@ -2,6 +2,6 @@ black==24.10.0
 clingo==5.7.1
 flake8==7.1.1
 isort==5.13.2
-mypy==1.11.2
+mypy==1.8.0
 types-six==1.17.0.20241205
 vermin==1.6.0
--- a/.github/workflows/unit_tests.yaml
+++ b/.github/workflows/unit_tests.yaml
@@ -140,7 +140,7 @@ jobs:
    - name: Install dependencies
      run: |
          dnf install -y \
-              bzip2 curl gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
+              bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
              make patch tcl unzip which xz
    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
    - name: Setup repo and non-root user
--- a/.github/workflows/valid-style.yml
+++ b/.github/workflows/valid-style.yml
@@ -20,7 +20,7 @@ jobs:
    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
    - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b
      with:
-        python-version: '3.13'
+        python-version: '3.11'
        cache: 'pip'
    - name: Install Python Packages
      run: |
@@ -39,7 +39,7 @@ jobs:
        fetch-depth: 0
    - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b
      with:
-        python-version: '3.13'
+        python-version: '3.11'
        cache: 'pip'
    - name: Install Python packages
      run: |
@@ -58,7 +58,7 @@ jobs:
    secrets: inherit
    with:
      with_coverage: ${{ inputs.with_coverage }}
-      python_version: '3.13'
+      python_version: '3.11'
  # Check that spack can bootstrap the development environment on Python 3.6 - RHEL8
  bootstrap-dev-rhel8:
    runs-on: ubuntu-latest
--- a/5
+++ b/5
@@ -8,9 +8,8 @@ or http://www.apache.org/licenses/LICENSE-2.0) or the MIT license,
 Copyrights and patents in the Spack project are retained by contributors.
 No copyright assignment is required to contribute to Spack.

-Spack was originally developed in 2013 by Lawrence Livermore National
-Security, LLC. It was originally distributed under the LGPL-2.1 license.
-Consent from contributors to relicense to Apache-2.0/MIT is documented at
+Spack was originally distributed under the LGPL-2.1 license. Consent from
+contributors to relicense to Apache-2.0/MIT is documented at
 https://github.com/spack/spack/issues/9137.


--- a/2
+++ b/2
@@ -1,6 +1,6 @@
 MIT License

-Copyright (c) Spack Project Developers.
+Copyright (c) 2013-2024 LLNS, LLC and other Spack Project Developers.

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
--- a/bin/haspywin.py
+++ b/bin/haspywin.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import subprocess
--- a/bin/sbang
+++ b/bin/sbang
@@ -1,6 +1,7 @@
 #!/bin/sh
 #
-# Copyright sbang project developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# sbang project developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/bin/spack
+++ b/bin/spack
@@ -1,7 +1,8 @@
 #!/bin/sh
 # -*- python -*-
 #
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/bin/spack-python
+++ b/bin/spack-python
@@ -1,6 +1,7 @@
 #!/bin/sh
 #
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/bin/spack.bat
+++ b/bin/spack.bat
@@ -1,4 +1,5 @@
-:: Copyright Spack Project Developers. See COPYRIGHT file for details.
+:: Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+:: Spack Project Developers. See the top-level COPYRIGHT file for details.
 ::
 :: SPDX-License-Identifier: (Apache-2.0 OR MIT)
 ::#######################################################################
--- a/bin/spack.ps1
+++ b/bin/spack.ps1
@@ -1,4 +1,5 @@
-#  Copyright Spack Project Developers. See COPYRIGHT file for details.
+#  Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+#  Spack Project Developers. See the top-level COPYRIGHT file for details.

 #  SPDX-License-Identifier: (Apache-2.0 OR MIT)
 # #######################################################################
--- a/bin/spack_pwsh.ps1
+++ b/bin/spack_pwsh.ps1
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/etc/spack/defaults/packages.yaml
+++ b/etc/spack/defaults/packages.yaml
@@ -65,7 +65,6 @@ packages:
      unwind: [libunwind]
      uuid: [util-linux-uuid, libuuid]
      wasi-sdk: [wasi-sdk-prebuilt]
-      xkbdata-api: [xkeyboard-config, xkbdata]
      xxd: [xxd-standalone, vim]
      yacc: [bison, byacc]
      ziglang: [zig]
--- a/lib/spack/docs/_pygments/style.py
+++ b/lib/spack/docs/_pygments/style.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/basic_usage.rst
+++ b/lib/spack/docs/basic_usage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/binary_caches.rst
+++ b/lib/spack/docs/binary_caches.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/bootstrapping.rst
+++ b/lib/spack/docs/bootstrapping.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_settings.rst
+++ b/lib/spack/docs/build_settings.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems.rst
+++ b/lib/spack/docs/build_systems.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/autotoolspackage.rst
+++ b/lib/spack/docs/build_systems/autotoolspackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/bundlepackage.rst
+++ b/lib/spack/docs/build_systems/bundlepackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/cachedcmakepackage.rst
+++ b/lib/spack/docs/build_systems/cachedcmakepackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/cmakepackage.rst
+++ b/lib/spack/docs/build_systems/cmakepackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/cudapackage.rst
+++ b/lib/spack/docs/build_systems/cudapackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/custompackage.rst
+++ b/lib/spack/docs/build_systems/custompackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/inteloneapipackage.rst
+++ b/lib/spack/docs/build_systems/inteloneapipackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/intelpackage.rst
+++ b/lib/spack/docs/build_systems/intelpackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/luapackage.rst
+++ b/lib/spack/docs/build_systems/luapackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/makefilepackage.rst
+++ b/lib/spack/docs/build_systems/makefilepackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/mavenpackage.rst
+++ b/lib/spack/docs/build_systems/mavenpackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/mesonpackage.rst
+++ b/lib/spack/docs/build_systems/mesonpackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/octavepackage.rst
+++ b/lib/spack/docs/build_systems/octavepackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/perlpackage.rst
+++ b/lib/spack/docs/build_systems/perlpackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/pythonpackage.rst
+++ b/lib/spack/docs/build_systems/pythonpackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/qmakepackage.rst
+++ b/lib/spack/docs/build_systems/qmakepackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/racketpackage.rst
+++ b/lib/spack/docs/build_systems/racketpackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/rocmpackage.rst
+++ b/lib/spack/docs/build_systems/rocmpackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/rpackage.rst
+++ b/lib/spack/docs/build_systems/rpackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/rubypackage.rst
+++ b/lib/spack/docs/build_systems/rubypackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/sconspackage.rst
+++ b/lib/spack/docs/build_systems/sconspackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/sippackage.rst
+++ b/lib/spack/docs/build_systems/sippackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/sourceforgepackage.rst
+++ b/lib/spack/docs/build_systems/sourceforgepackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/build_systems/wafpackage.rst
+++ b/lib/spack/docs/build_systems/wafpackage.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/chain.rst
+++ b/lib/spack/docs/chain.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/conf.py
+++ b/lib/spack/docs/conf.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/config_yaml.rst
+++ b/lib/spack/docs/config_yaml.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

@@ -25,23 +26,14 @@ These settings can be overridden in ``etc/spack/config.yaml`` or
 The location where Spack will install packages and their dependencies.
 Default is ``$spack/opt/spack``.

---------------
-``projections``
---------------
+---------------------------------------------------
+``install_hash_length`` and ``install_path_scheme``
+---------------------------------------------------

-.. warning::
-
-   Modifying projections of the install tree is strongly discouraged.
-
-By default Spack installs all packages into a unique directory relative to the install
-tree root with the following layout:
-
-.. code-block::
-
-   {architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}
-
-In very rare cases, it may be necessary to reduce the length of this path. For example,
-very old versions of the Intel compiler are known to segfault when input paths are too long:
+The default Spack installation path can be very long and can create problems
+for scripts with hardcoded shebangs. Additionally, when using the Intel
+compiler, and if there is also a long list of dependencies, the compiler may
+segfault. If you see the following:

     .. code-block:: console

@@ -49,25 +41,36 @@ very old versions of the Intel compiler are known to segfault when input paths a
       ** Segmentation violation signal raised. **
       Access violation or stack overflow. Please contact Intel Support for assistance.

-Another case is Python and R packages with many runtime dependencies, which can result
-in very large ``PYTHONPATH`` and ``R_LIBS`` environment variables. This can cause the
-``execve`` system call to fail with ``E2BIG``, preventing processes from starting.
+it may be because variables containing dependency specs may be too long. There
+are two parameters to help with long path names. Firstly, the
+``install_hash_length`` parameter can set the length of the hash in the
+installation path from 1 to 32. The default path uses the full 32 characters.

-For this reason, Spack allows users to modify the installation layout through custom
-projections. For example
+Secondly, it is also possible to modify the entire installation
+scheme. By default Spack uses
+``{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}``
+where the tokens that are available for use in this directive are the
+same as those understood by the :meth:`~spack.spec.Spec.format`
+method. Using this parameter it is possible to use a different package
+layout or reduce the depth of the installation paths. For example

     .. code-block:: yaml

       config:
-         install_tree:
-           root: $spack/opt/spack
-           projections:
-             all: "{name}/{version}/{hash:16}"
+         install_path_scheme: '{name}/{version}/{hash:7}'

-would install packages into sub-directories using only the package name, version and a
-hash length of 16 characters.
+would install packages into sub-directories using only the package
+name, version and a hash length of 7 characters.

-Notice that reducing the hash length increases the likelihood of hash collisions.
+When using either parameter to set the hash length it only affects the
+representation of the hash in the installation directory. You
+should be aware that the smaller the hash length the more likely
+naming conflicts will occur. These parameters are independent of those
+used to configure module names.
+
+.. warning:: Modifying the installation hash length or path scheme after
+   packages have been installed will prevent Spack from being
+   able to find the old installation directories.

 --------------------
 ``build_stage``
--- a/lib/spack/docs/configuration.rst
+++ b/lib/spack/docs/configuration.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/containers.rst
+++ b/lib/spack/docs/containers.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/contribution_guide.rst
+++ b/lib/spack/docs/contribution_guide.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/developer_guide.rst
+++ b/lib/spack/docs/developer_guide.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

@@ -543,10 +544,10 @@ With either interpreter you can run a single command:

 .. code-block:: console

-   $ spack python -c 'from spack.concretize import concretize_one; concretize_one("python")'
+   $ spack python -c 'from spack.spec import Spec; Spec("python").concretized()'
   ...

-   $ spack python -i ipython -c 'from spack.concretize import concretize_one; concretize_one("python")'
+   $ spack python -i ipython -c 'from spack.spec import Spec; Spec("python").concretized()'
   Out[1]: ...

 or a file:
--- a/lib/spack/docs/environments.rst
+++ b/lib/spack/docs/environments.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/extensions.rst
+++ b/lib/spack/docs/extensions.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/features.rst
+++ b/lib/spack/docs/features.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/frequently_asked_questions.rst
+++ b/lib/spack/docs/frequently_asked_questions.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/getting_started.rst
+++ b/lib/spack/docs/getting_started.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

@@ -34,7 +35,7 @@ A build matrix showing which packages are working on which systems is shown belo
      .. code-block:: console

         apt update
-         apt install bzip2 ca-certificates g++ gcc gfortran git gzip lsb-release patch python3 tar unzip xz-utils zstd
+         apt install bzip2 ca-certificates file g++ gcc gfortran git gzip lsb-release patch python3 tar unzip xz-utils zstd

   .. tab-item:: RHEL

@@ -147,22 +148,20 @@ The first time you concretize a spec, Spack will bootstrap automatically:
   --------------------------------
   zlib@1.2.13%gcc@9.4.0+optimize+pic+shared build_system=makefile arch=linux-ubuntu20.04-icelake

-The default bootstrap behavior is to use pre-built binaries. You can verify the
-active bootstrap repositories with:
-
-.. command-output:: spack bootstrap list
-
 If for security concerns you cannot bootstrap ``clingo`` from pre-built
 binaries, you have to disable fetching the binaries we generated with Github Actions.

 .. code-block:: console

-   $ spack bootstrap disable github-actions-v0.6
-   ==> "github-actions-v0.6" is now disabled and will not be used for bootstrapping
-   $ spack bootstrap disable github-actions-v0.5
-   ==> "github-actions-v0.5" is now disabled and will not be used for bootstrapping
+   $ spack bootstrap disable github-actions-v0.4
+   ==> "github-actions-v0.4" is now disabled and will not be used for bootstrapping
+   $ spack bootstrap disable github-actions-v0.3
+   ==> "github-actions-v0.3" is now disabled and will not be used for bootstrapping
+
+You can verify that the new settings are effective with:
+
+.. command-output:: spack bootstrap list

-You can verify that the new settings are effective with ``spack bootstrap list``.

 .. note::

--- a/lib/spack/docs/gpu_configuration.rst
+++ b/lib/spack/docs/gpu_configuration.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/index.rst
+++ b/lib/spack/docs/index.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/mirrors.rst
+++ b/lib/spack/docs/mirrors.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/module_file_support.rst
+++ b/lib/spack/docs/module_file_support.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

@@ -456,13 +457,14 @@ For instance, the following config options,
       tcl:
         all:
           suffixes:
-             ^python@3: 'python{^python.version.up_to_2}'
+             ^python@3: 'python{^python.version}'
             ^openblas: 'openblas'

-will add a ``python3.12`` to module names of packages compiled with Python 3.12, and similarly for
-all specs depending on ``python@3``. This is useful to know which version of Python a set of Python
-extensions is associated with. Likewise, the ``openblas`` string is attached to any program that
-has openblas in the spec, most likely via the ``+blas`` variant specification.
+will add a ``python-3.12.1`` version string to any packages compiled with
+Python matching the spec, ``python@3``. This is useful to know which
+version of Python a set of Python extensions is associated with. Likewise, the
+``openblas`` string is attached to any program that has openblas in the spec,
+most likely via the ``+blas`` variant specification.

 The most heavyweight solution to module naming is to change the entire
 naming convention for module files. This uses the projections format
--- a/lib/spack/docs/packages_yaml.rst
+++ b/lib/spack/docs/packages_yaml.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/packaging_guide.rst
+++ b/lib/spack/docs/packaging_guide.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

@@ -236,7 +237,8 @@ for details):
 .. code-block:: python
   :linenos:

-   # Copyright Spack Project Developers. See COPYRIGHT file for details.
+   # Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   # Spack Project Developers. See the top-level COPYRIGHT file for details.
   #
   # SPDX-License-Identifier: (Apache-2.0 OR MIT)

@@ -4898,15 +4900,15 @@ If your package has a virtual dependency like ``mpi``, then referring to
 ``spec["mpi"]`` within ``install()`` will get you the concrete ``mpi``
 implementation in your dependency DAG.  That is a spec object just like
 the one passed to install, only the MPI implementations all set some
-additional properties on it to help you out.  E.g., in openmpi, you'll
+additional properties on it to help you out.  E.g., in mvapich2, you'll
 find this:

-.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/openmpi/package.py
-   :pyobject: Openmpi.setup_dependent_package
+.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/mvapich2/package.py
+   :pyobject: Mvapich2.setup_dependent_package

-That code allows the ``openmpi`` package to associate an ``mpicc`` property
-with the ``openmpi`` node in the DAG, so that dependents can access it.
-``mvapich2`` and ``mpich`` do similar things.  So, no matter what MPI
+That code allows the mvapich2 package to associate an ``mpicc`` property
+with the ``mvapich2`` node in the DAG, so that dependents can access it.
+``openmpi`` and ``mpich`` do similar things.  So, no matter what MPI
 you're using, spec["mpi"].mpicc gets you the location of the MPI
 compilers. This allows us to have a fairly simple polymorphic interface
 for information about virtual dependencies like MPI.
--- a/lib/spack/docs/pipelines.rst
+++ b/lib/spack/docs/pipelines.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/replace_conda_homebrew.rst
+++ b/lib/spack/docs/replace_conda_homebrew.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/repositories.rst
+++ b/lib/spack/docs/repositories.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/requirements.txt
+++ b/lib/spack/docs/requirements.txt
@@ -4,8 +4,8 @@ sphinx_design==0.6.1
 sphinx-rtd-theme==3.0.2
 python-levenshtein==0.26.1
 docutils==0.21.2
-pygments==2.19.1
-urllib3==2.3.0
+pygments==2.18.0
+urllib3==2.2.3
 pytest==8.3.4
 isort==5.13.2
 black==24.10.0
--- a/lib/spack/docs/signing.rst
+++ b/lib/spack/docs/signing.rst
@@ -1,4 +1,5 @@
-.. Copyright Spack Project Developers. See COPYRIGHT file for details.
+.. Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.

   SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/spack.yaml
+++ b/lib/spack/docs/spack.yaml
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/docs/tables/system_prerequisites.csv
+++ b/lib/spack/docs/tables/system_prerequisites.csv
@@ -8,6 +8,7 @@ unzip, , , Compress/Decompress archives
 bzip2, , , Compress/Decompress archives
 xz, , , Compress/Decompress archives
 zstd, , Optional, Compress/Decompress archives
+file, , , Create/Use Buildcaches
 lsb-release, , , Linux: identify operating system version
 gnupg2, , , Sign/Verify Buildcaches
 git, , , Manage Software Repositories
--- a/lib/spack/env/cc
+++ b/lib/spack/env/cc
@@ -1,7 +1,8 @@
 #!/bin/sh -f
 # shellcheck disable=SC2034  # evals in this script fool shellcheck
 #
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/external/init.py
+++ b/lib/spack/external/init.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/llnl/init.py
+++ b/lib/spack/llnl/init.py
@@ -1,3 +1,4 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
--- a/lib/spack/llnl/path.py
+++ b/lib/spack/llnl/path.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Path primitives that just require Python standard library."""
--- a/lib/spack/llnl/string.py
+++ b/lib/spack/llnl/string.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """String manipulation functions that do not have other dependencies than Python
--- a/lib/spack/llnl/url.py
+++ b/lib/spack/llnl/url.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """URL primitives that just require Python standard library."""
--- a/lib/spack/llnl/util/init.py
+++ b/lib/spack/llnl/util/init.py
@@ -1,3 +1,4 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
--- a/lib/spack/llnl/util/argparsewriter.py
+++ b/lib/spack/llnl/util/argparsewriter.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import collections
@@ -1740,7 +1741,8 @@ def find(


 def _log_file_access_issue(e: OSError, path: str) -> None:
-    tty.debug(f"find must skip {path}: {e}")
+    errno_name = errno.errorcode.get(e.errno, "UNKNOWN")
+    tty.debug(f"find must skip {path}: {errno_name} {e}")


 def _file_id(s: os.stat_result) -> Tuple[int, int]:
--- a/lib/spack/llnl/util/lang.py
+++ b/lib/spack/llnl/util/lang.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

@@ -862,10 +863,8 @@ def elide_list(line_list: List[str], max_num: int = 10) -> List[str]:

 if sys.version_info >= (3, 9):
    PatternStr = re.Pattern[str]
-    PatternBytes = re.Pattern[bytes]
 else:
    PatternStr = typing.Pattern[str]
-    PatternBytes = typing.Pattern[bytes]


 def fnmatch_translate_multiple(named_patterns: Dict[str, str]) -> str:
--- a/lib/spack/llnl/util/link_tree.py
+++ b/lib/spack/llnl/util/link_tree.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/llnl/util/lock.py
+++ b/lib/spack/llnl/util/lock.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/llnl/util/multiproc.py
+++ b/lib/spack/llnl/util/multiproc.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/llnl/util/symlink.py
+++ b/lib/spack/llnl/util/symlink.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
--- a/lib/spack/llnl/util/tty/init.py
+++ b/lib/spack/llnl/util/tty/init.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/llnl/util/tty/colify.py
+++ b/lib/spack/llnl/util/tty/colify.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/llnl/util/tty/color.py
+++ b/lib/spack/llnl/util/tty/color.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/llnl/util/tty/log.py
+++ b/lib/spack/llnl/util/tty/log.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/llnl/util/tty/pty.py
+++ b/lib/spack/llnl/util/tty/pty.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/spack/init.py
+++ b/lib/spack/spack/init.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

--- a/lib/spack/spack/audit.py
+++ b/lib/spack/spack/audit.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Classes and functions to register audit checks for various parts of
@@ -1356,8 +1357,14 @@ def _test_detection_by_executable(pkgs, debug_log, error_cls):

            def _compare_extra_attribute(_expected, _detected, *, _spec):
                result = []
+                # Check items are of the same type
+                if not isinstance(_detected, type(_expected)):
+                    _summary = f'{pkg_name}: error when trying to detect "{_expected}"'
+                    _details = [f"{_detected} was detected instead"]
+                    return [error_cls(summary=_summary, details=_details)]
+
                # If they are string expected is a regex
-                if isinstance(_expected, str) and isinstance(_detected, str):
+                if isinstance(_expected, str):
                    try:
                        _regex = re.compile(_expected)
                    except re.error:
@@ -1373,7 +1380,7 @@ def _compare_extra_attribute(_expected, _detected, *, _spec):
                        _details = [f"{_detected} does not match the regex"]
                        return [error_cls(summary=_summary, details=_details)]

-                elif isinstance(_expected, dict) and isinstance(_detected, dict):
+                if isinstance(_expected, dict):
                    _not_detected = set(_expected.keys()) - set(_detected.keys())
                    if _not_detected:
                        _summary = f"{pkg_name}: cannot detect some attributes for spec {_spec}"
@@ -1388,10 +1395,6 @@ def _compare_extra_attribute(_expected, _detected, *, _spec):
                        result.extend(
                            _compare_extra_attribute(_expected[_key], _detected[_key], _spec=_spec)
                        )
-                else:
-                    _summary = f'{pkg_name}: error when trying to detect "{_expected}"'
-                    _details = [f"{_detected} was detected instead"]
-                    return [error_cls(summary=_summary, details=_details)]

                return result

--- a/lib/spack/spack/binary_distribution.py
+++ b/lib/spack/spack/binary_distribution.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

@@ -23,12 +24,13 @@
 import urllib.request
 import warnings
 from contextlib import closing
-from typing import IO, Callable, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple, Union
+from typing import Dict, Iterable, List, NamedTuple, Optional, Set, Tuple, Union

 import llnl.util.filesystem as fsys
 import llnl.util.lang
 import llnl.util.tty as tty
-from llnl.util.filesystem import mkdirp
+from llnl.util.filesystem import BaseDirectoryVisitor, mkdirp, visit_directory_tree
+from llnl.util.symlink import readlink

 import spack.caches
 import spack.config as config
@@ -52,6 +54,7 @@
 import spack.util.archive
 import spack.util.crypto
 import spack.util.file_cache as file_cache
+import spack.util.filesystem as ssys
 import spack.util.gpg
 import spack.util.parallel
 import spack.util.path
@@ -584,39 +587,173 @@ def read_buildinfo_file(prefix):
        return syaml.load(f)


-def file_matches(f: IO[bytes], regex: llnl.util.lang.PatternBytes) -> bool:
-    try:
-        return bool(regex.search(f.read()))
-    finally:
-        f.seek(0)
+class BuildManifestVisitor(BaseDirectoryVisitor):
+    """Visitor that collects a list of files and symlinks
+    that can be checked for need of relocation. It knows how
+    to dedupe hardlinks and deal with symlinks to files and
+    directories."""
+
+    def __init__(self):
+        # Save unique identifiers of hardlinks to avoid relocating them multiple times
+        self.visited = set()
+
+        # Lists of files we will check
+        self.files = []
+        self.symlinks = []
+
+    def seen_before(self, root, rel_path):
+        stat_result = os.lstat(os.path.join(root, rel_path))
+        if stat_result.st_nlink == 1:
+            return False
+        identifier = (stat_result.st_dev, stat_result.st_ino)
+        if identifier in self.visited:
+            return True
+        else:
+            self.visited.add(identifier)
+            return False
+
+    def visit_file(self, root, rel_path, depth):
+        if self.seen_before(root, rel_path):
+            return
+        self.files.append(rel_path)
+
+    def visit_symlinked_file(self, root, rel_path, depth):
+        # Note: symlinks *can* be hardlinked, but it is unclear if
+        # symlinks can be relinked in-place (preserving inode).
+        # Therefore, we do *not* de-dupe hardlinked symlinks.
+        self.symlinks.append(rel_path)
+
+    def before_visit_dir(self, root, rel_path, depth):
+        return os.path.basename(rel_path) not in (".spack", "man")
+
+    def before_visit_symlinked_dir(self, root, rel_path, depth):
+        # Treat symlinked directories simply as symlinks.
+        self.visit_symlinked_file(root, rel_path, depth)
+        # Never recurse into symlinked directories.
+        return False


-def specs_to_relocate(spec: spack.spec.Spec) -> List[spack.spec.Spec]:
-    """Return the set of specs that may be referenced in the install prefix of the provided spec.
-    We currently include non-external transitive link and direct run dependencies."""
-    specs = [
+def file_matches(path, regex):
+    with open(path, "rb") as f:
+        contents = f.read()
+    return bool(regex.search(contents))
+
+
+def get_buildfile_manifest(spec):
+    """
+    Return a data structure with information about a build, including
+    text_to_relocate, binary_to_relocate, binary_to_relocate_fullpath
+    link_to_relocate, and other, which means it doesn't fit any of previous
+    checks (and should not be relocated). We exclude docs (man) and
+    metadata (.spack). This can be used to find a particular kind of file
+    in spack, or to generate the build metadata.
+    """
+    data = {
+        "text_to_relocate": [],
+        "binary_to_relocate": [],
+        "link_to_relocate": [],
+        "other": [],
+        "binary_to_relocate_fullpath": [],
+        "hardlinks_deduped": True,
+    }
+
+    # Guard against filesystem footguns of hardlinks and symlinks by using
+    # a visitor to retrieve a list of files and symlinks, so we don't have
+    # to worry about hardlinks of symlinked dirs and what not.
+    visitor = BuildManifestVisitor()
+    root = spec.prefix
+    visit_directory_tree(root, visitor)
+
+    # Collect a list of prefixes for this package and it's dependencies, Spack will
+    # look for them to decide if text file needs to be relocated or not
+    prefixes = [d.prefix for d in spec.traverse(root=True, deptype="all") if not d.external]
+    prefixes.append(spack.hooks.sbang.sbang_install_path())
+    prefixes.append(str(spack.store.STORE.layout.root))
+
+    # Create a giant regex that matches all prefixes
+    regex = utf8_paths_to_single_binary_regex(prefixes)
+
+    # Symlinks.
+
+    # Obvious bugs:
+    #   1. relative links are not relocated.
+    #   2. paths are used as strings.
+    for rel_path in visitor.symlinks:
+        abs_path = os.path.join(root, rel_path)
+        link = readlink(abs_path)
+        if os.path.isabs(link) and link.startswith(spack.store.STORE.layout.root):
+            data["link_to_relocate"].append(rel_path)
+
+    # Non-symlinks.
+    for rel_path in visitor.files:
+        abs_path = os.path.join(root, rel_path)
+        m_type, m_subtype = ssys.mime_type(abs_path)
+
+        if relocate.needs_binary_relocation(m_type, m_subtype):
+            # Why is this branch not part of needs_binary_relocation? :(
+            if (
+                (
+                    m_subtype in ("x-executable", "x-sharedlib", "x-pie-executable")
+                    and sys.platform != "darwin"
+                )
+                or (m_subtype in ("x-mach-binary") and sys.platform == "darwin")
+                or (not rel_path.endswith(".o"))
+            ):
+                data["binary_to_relocate"].append(rel_path)
+                data["binary_to_relocate_fullpath"].append(abs_path)
+                continue
+
+        elif relocate.needs_text_relocation(m_type, m_subtype) and file_matches(abs_path, regex):
+            data["text_to_relocate"].append(rel_path)
+            continue
+
+        data["other"].append(abs_path)
+
+    return data
+
+
+def deps_to_relocate(spec):
+    """Return the transitive link and direct run dependencies of the spec.
+
+    This is a special traversal for dependencies we need to consider when relocating a package.
+
+    Package binaries, scripts, and other files may refer to the prefixes of  dependencies, so
+    we need to rewrite those locations when dependencies are in a different place at install time
+    than they were at build time.
+
+    This traversal covers transitive link dependencies and direct run dependencies because:
+
+    1. Spack adds RPATHs for transitive link dependencies so that packages can find needed
+       dependency libraries.
+    2. Packages may call any of their *direct* run dependencies (and may bake their paths into
+       binaries or scripts), so we also need to search for run dependency prefixes when relocating.
+
+    This returns a deduplicated list of transitive link dependencies and direct run dependencies.
+    """
+    deps = [
        s
        for s in itertools.chain(
-            spec.traverse(root=True, deptype="link", order="breadth", key=traverse.by_dag_hash),
-            spec.dependencies(deptype="run"),
+            spec.traverse(root=True, deptype="link"), spec.dependencies(deptype="run")
        )
        if not s.external
    ]
-    return list(llnl.util.lang.dedupe(specs, key=lambda s: s.dag_hash()))
+    return llnl.util.lang.dedupe(deps, key=lambda s: s.dag_hash())


 def get_buildinfo_dict(spec):
    """Create metadata for a tarball"""
+    manifest = get_buildfile_manifest(spec)
+
    return {
        "sbang_install_path": spack.hooks.sbang.sbang_install_path(),
        "buildpath": spack.store.STORE.layout.root,
        "spackprefix": spack.paths.prefix,
        "relative_prefix": os.path.relpath(spec.prefix, spack.store.STORE.layout.root),
-        # "relocate_textfiles": [],
-        # "relocate_binaries": [],
-        # "relocate_links": [],
-        "hardlinks_deduped": True,
-        "hash_to_prefix": {d.dag_hash(): str(d.prefix) for d in specs_to_relocate(spec)},
+        "relocate_textfiles": manifest["text_to_relocate"],
+        "relocate_binaries": manifest["binary_to_relocate"],
+        "relocate_links": manifest["link_to_relocate"],
+        "hardlinks_deduped": manifest["hardlinks_deduped"],
+        "hash_to_prefix": {d.dag_hash(): str(d.prefix) for d in deps_to_relocate(spec)},
    }


@@ -669,24 +806,19 @@ def sign_specfile(key: str, specfile_path: str) -> str:


 def _read_specs_and_push_index(
-    file_list: List[str],
-    read_method: Callable,
-    cache_prefix: str,
-    db: BuildCacheDatabase,
-    temp_dir: str,
-    concurrency: int,
+    file_list, read_method, cache_prefix, db: BuildCacheDatabase, temp_dir, concurrency
 ):
    """Read all the specs listed in the provided list, using thread given thread parallelism,
        generate the index, and push it to the mirror.

    Args:
-        file_list: List of urls or file paths pointing at spec files to read
+        file_list (list(str)): List of urls or file paths pointing at spec files to read
        read_method: A function taking a single argument, either a url or a file path,
            and which reads the spec file at that location, and returns the spec.
-        cache_prefix: prefix of the build cache on s3 where index should be pushed.
+        cache_prefix (str): prefix of the build cache on s3 where index should be pushed.
        db: A spack database used for adding specs and then writing the index.
-        temp_dir: Location to write index.json and hash for pushing
-        concurrency: Number of parallel processes to use when fetching
+        temp_dir (str): Location to write index.json and hash for pushing
+        concurrency (int): Number of parallel processes to use when fetching
    """
    for file in file_list:
        contents = read_method(file)
@@ -866,12 +998,9 @@ def _url_generate_package_index(url: str, tmpdir: str, concurrency: int = 32):
    tty.debug(f"Retrieving spec descriptor files from {url} to build index")

    db = BuildCacheDatabase(tmpdir)
-    db._write()

    try:
-        _read_specs_and_push_index(
-            file_list, read_fn, url, db, str(db.database_directory), concurrency
-        )
+        _read_specs_and_push_index(file_list, read_fn, url, db, db.database_directory, concurrency)
    except Exception as e:
        raise GenerateIndexError(f"Encountered problem pushing package index to {url}: {e}") from e

@@ -913,55 +1042,7 @@ def generate_key_index(key_prefix: str, tmpdir: str) -> None:
        ) from e


-class FileTypes:
-    BINARY = 0
-    TEXT = 1
-    UNKNOWN = 2
-
-
-NOT_ISO8859_1_TEXT = re.compile(b"[\x00\x7F-\x9F]")
-
-
-def file_type(f: IO[bytes]) -> int:
-    try:
-        # first check if this is an ELF or mach-o binary.
-        magic = f.read(8)
-        if len(magic) < 8:
-            return FileTypes.UNKNOWN
-        elif relocate.is_elf_magic(magic) or relocate.is_macho_magic(magic):
-            return FileTypes.BINARY
-
-        f.seek(0)
-
-        # Then try utf-8, which has a fast exponential decay in false positive rate with file size.
-        # Use chunked reads for fast early exit.
-        f_txt = io.TextIOWrapper(f, encoding="utf-8", errors="strict")
-        try:
-            while f_txt.read(1024):
-                pass
-            return FileTypes.TEXT
-        except UnicodeError:
-            f_txt.seek(0)
-            pass
-        finally:
-            f_txt.detach()
-        # Finally try iso-8859-1 heuristically. In Python, all possible 256 byte values are valid.
-        # We classify it as text if it does not contain any control characters / null bytes.
-        data = f.read(1024)
-        while data:
-            if NOT_ISO8859_1_TEXT.search(data):
-                break
-            data = f.read(1024)
-        else:
-            return FileTypes.TEXT
-        return FileTypes.UNKNOWN
-    finally:
-        f.seek(0)
-
-
-def tarfile_of_spec_prefix(
-    tar: tarfile.TarFile, prefix: str, prefixes_to_relocate: List[str]
-) -> dict:
+def tarfile_of_spec_prefix(tar: tarfile.TarFile, prefix: str) -> None:
    """Create a tarfile of an install prefix of a spec. Skips existing buildinfo file.

    Args:
@@ -977,33 +1058,6 @@ def tarfile_of_spec_prefix(
    except OSError:
        skip = lambda entry: False

-    binary_regex = utf8_paths_to_single_binary_regex(prefixes_to_relocate)
-
-    relocate_binaries = []
-    relocate_links = []
-    relocate_textfiles = []
-
-    # use callbacks to add files and symlinks, so we can register which files need relocation upon
-    # extraction.
-    def add_file(tar: tarfile.TarFile, info: tarfile.TarInfo, path: str):
-        with open(path, "rb") as f:
-            relpath = os.path.relpath(path, prefix)
-            # no need to relocate anything in the .spack directory
-            if relpath.split(os.sep, 1)[0] == ".spack":
-                tar.addfile(info, f)
-                return
-            f_type = file_type(f)
-            if f_type == FileTypes.BINARY:
-                relocate_binaries.append(os.path.relpath(path, prefix))
-            elif f_type == FileTypes.TEXT and file_matches(f, binary_regex):
-                relocate_textfiles.append(os.path.relpath(path, prefix))
-            tar.addfile(info, f)
-
-    def add_symlink(tar: tarfile.TarFile, info: tarfile.TarInfo, path: str):
-        if os.path.isabs(info.linkname) and binary_regex.match(info.linkname.encode("utf-8")):
-            relocate_links.append(os.path.relpath(path, prefix))
-        tar.addfile(info)
-
    spack.util.archive.reproducible_tarfile_from_prefix(
        tar,
        prefix,
@@ -1011,51 +1065,29 @@ def add_symlink(tar: tarfile.TarFile, info: tarfile.TarInfo, path: str):
        # used in runtimes like AWS lambda.
        include_parent_directories=True,
        skip=skip,
-        add_file=add_file,
-        add_symlink=add_symlink,
-    )
-
-    return {
-        "relocate_binaries": relocate_binaries,
-        "relocate_links": relocate_links,
-        "relocate_textfiles": relocate_textfiles,
-    }
-
-
-def create_tarball(spec: spack.spec.Spec, tarfile_path: str) -> Tuple[str, str]:
-    """Create a tarball of a spec and return the checksums of the compressed tarfile and the
-    uncompressed tarfile."""
-    return _do_create_tarball(
-        tarfile_path,
-        spec.prefix,
-        buildinfo=get_buildinfo_dict(spec),
-        prefixes_to_relocate=prefixes_to_relocate(spec),
    )


-def _do_create_tarball(
-    tarfile_path: str, prefix: str, buildinfo: dict, prefixes_to_relocate: List[str]
-) -> Tuple[str, str]:
+def _do_create_tarball(tarfile_path: str, binaries_dir: str, buildinfo: dict):
    with spack.util.archive.gzip_compressed_tarfile(tarfile_path) as (
        tar,
-        tar_gz_checksum,
-        tar_checksum,
+        inner_checksum,
+        outer_checksum,
    ):
        # Tarball the install prefix
-        files_to_relocate = tarfile_of_spec_prefix(tar, prefix, prefixes_to_relocate)
-        buildinfo.update(files_to_relocate)
+        tarfile_of_spec_prefix(tar, binaries_dir)

        # Serialize buildinfo for the tarball
        bstring = syaml.dump(buildinfo, default_flow_style=True).encode("utf-8")
        tarinfo = tarfile.TarInfo(
-            name=spack.util.archive.default_path_to_name(buildinfo_file_name(prefix))
+            name=spack.util.archive.default_path_to_name(buildinfo_file_name(binaries_dir))
        )
        tarinfo.type = tarfile.REGTYPE
        tarinfo.size = len(bstring)
        tarinfo.mode = 0o644
        tar.addfile(tarinfo, io.BytesIO(bstring))

-    return tar_gz_checksum.hexdigest(), tar_checksum.hexdigest()
+    return inner_checksum.hexdigest(), outer_checksum.hexdigest()


 class ExistsInBuildcache(NamedTuple):
@@ -1105,13 +1137,6 @@ def _exists_in_buildcache(spec: spack.spec.Spec, tmpdir: str, out_url: str) -> E
    return ExistsInBuildcache(signed, unsigned, tarball)


-def prefixes_to_relocate(spec):
-    prefixes = [s.prefix for s in specs_to_relocate(spec)]
-    prefixes.append(spack.hooks.sbang.sbang_install_path())
-    prefixes.append(str(spack.store.STORE.layout.root))
-    return prefixes
-
-
 def _url_upload_tarball_and_specfile(
    spec: spack.spec.Spec,
    tmpdir: str,
@@ -1121,7 +1146,7 @@ def _url_upload_tarball_and_specfile(
 ):
    files = BuildcacheFiles(spec, tmpdir, out_url)
    tarball = files.local_tarball()
-    checksum, _ = create_tarball(spec, tarball)
+    checksum, _ = _do_create_tarball(tarball, spec.prefix, get_buildinfo_dict(spec))
    spec_dict = spec.to_dict(hash=ht.dag_hash)
    spec_dict["buildcache_layout_version"] = CURRENT_BUILD_CACHE_LAYOUT_VERSION
    spec_dict["binary_cache_checksum"] = {"hash_algorithm": "sha256", "hash": checksum}
@@ -1445,11 +1470,13 @@ def _oci_push_pkg_blob(
    filename = os.path.join(tmpdir, f"{spec.dag_hash()}.tar.gz")

    # Create an oci.image.layer aka tarball of the package
-    tar_gz_checksum, tar_checksum = create_tarball(spec, filename)
+    compressed_tarfile_checksum, tarfile_checksum = _do_create_tarball(
+        filename, spec.prefix, get_buildinfo_dict(spec)
+    )

    blob = spack.oci.oci.Blob(
-        Digest.from_sha256(tar_gz_checksum),
-        Digest.from_sha256(tar_checksum),
+        Digest.from_sha256(compressed_tarfile_checksum),
+        Digest.from_sha256(tarfile_checksum),
        os.path.getsize(filename),
    )

@@ -2133,9 +2160,10 @@ def fetch_url_to_mirror(url):


 def dedupe_hardlinks_if_necessary(root, buildinfo):
-    """Updates a buildinfo dict for old archives that did not dedupe hardlinks. De-duping hardlinks
-    is necessary when relocating files in parallel and in-place. This means we must preserve inodes
-    when relocating."""
+    """Updates a buildinfo dict for old archives that did
+    not dedupe hardlinks. De-duping hardlinks is necessary
+    when relocating files in parallel and in-place. This
+    means we must preserve inodes when relocating."""

    # New archives don't need this.
    if buildinfo.get("hardlinks_deduped", False):
@@ -2164,47 +2192,65 @@ def dedupe_hardlinks_if_necessary(root, buildinfo):
        buildinfo[key] = new_list


-def relocate_package(spec: spack.spec.Spec) -> None:
-    """Relocate binaries and text files in the given spec prefix, based on its buildinfo file."""
-    buildinfo = read_buildinfo_file(spec.prefix)
+def relocate_package(spec):
+    """
+    Relocate the given package
+    """
+    workdir = str(spec.prefix)
+    buildinfo = read_buildinfo_file(workdir)
+    new_layout_root = str(spack.store.STORE.layout.root)
+    new_prefix = str(spec.prefix)
+    new_rel_prefix = str(os.path.relpath(new_prefix, new_layout_root))
+    new_spack_prefix = str(spack.paths.prefix)
+
+    old_sbang_install_path = None
+    if "sbang_install_path" in buildinfo:
+        old_sbang_install_path = str(buildinfo["sbang_install_path"])
    old_layout_root = str(buildinfo["buildpath"])
+    old_spack_prefix = str(buildinfo.get("spackprefix"))
+    old_rel_prefix = buildinfo.get("relative_prefix")
+    old_prefix = os.path.join(old_layout_root, old_rel_prefix)
+    rel = buildinfo.get("relative_rpaths", False)

-    # Warn about old style tarballs created with the --rel flag (removed in Spack v0.20)
-    if buildinfo.get("relative_rpaths", False):
-        tty.warn(
-            f"Tarball for {spec} uses relative rpaths, which can cause library loading issues."
-        )
-
-    # In Spack 0.19 and older prefix_to_hash was the default and externals were not dropped, so
-    # prefixes were not unique.
+    # In the past prefix_to_hash was the default and externals were not dropped, so prefixes
+    # were not unique.
    if "hash_to_prefix" in buildinfo:
        hash_to_old_prefix = buildinfo["hash_to_prefix"]
    elif "prefix_to_hash" in buildinfo:
-        hash_to_old_prefix = {v: k for (k, v) in buildinfo["prefix_to_hash"].items()}
+        hash_to_old_prefix = dict((v, k) for (k, v) in buildinfo["prefix_to_hash"].items())
    else:
-        raise NewLayoutException(
-            "Package tarball was created from an install prefix with a different directory layout "
-            "and an older buildcache create implementation. It cannot be relocated."
-        )
+        hash_to_old_prefix = dict()

-    prefix_to_prefix = {}
+    if old_rel_prefix != new_rel_prefix and not hash_to_old_prefix:
+        msg = "Package tarball was created from an install "
+        msg += "prefix with a different directory layout and an older "
+        msg += "buildcache create implementation. It cannot be relocated."
+        raise NewLayoutException(msg)

-    if "sbang_install_path" in buildinfo:
-        old_sbang_install_path = str(buildinfo["sbang_install_path"])
-        prefix_to_prefix[old_sbang_install_path] = spack.hooks.sbang.sbang_install_path()
+    # Spurious replacements (e.g. sbang) will cause issues with binaries
+    # For example, the new sbang can be longer than the old one.
+    # Hence 2 dictionaries are maintained here.
+    prefix_to_prefix_text = collections.OrderedDict()
+    prefix_to_prefix_bin = collections.OrderedDict()

-    # First match specific prefix paths. Possibly the *local* install prefix of some dependency is
-    # in an upstream, so we cannot assume the original spack store root can be mapped uniformly to
-    # the new spack store root.
+    if old_sbang_install_path:
+        install_path = spack.hooks.sbang.sbang_install_path()
+        prefix_to_prefix_text[old_sbang_install_path] = install_path

-    # If the spec is spliced, we need to handle the simultaneous mapping from the old install_tree
-    # to the new install_tree and from the build_spec to the spliced spec. Because foo.build_spec
-    # is foo for any non-spliced spec, we can simplify by checking for spliced-in nodes by checking
-    # for nodes not in the build_spec without any explicit check for whether the spec is spliced.
-    # An analog in this algorithm is any spec that shares a name or provides the same virtuals in
-    # the context of the relevant root spec. This ensures that the analog for a spec s is the spec
-    # that s replaced when we spliced.
-    relocation_specs = specs_to_relocate(spec)
+    # First match specific prefix paths. Possibly the *local* install prefix
+    # of some dependency is in an upstream, so we cannot assume the original
+    # spack store root can be mapped uniformly to the new spack store root.
+    #
+    # If the spec is spliced, we need to handle the simultaneous mapping
+    # from the old install_tree to the new install_tree and from the build_spec
+    # to the spliced spec.
+    # Because foo.build_spec is foo for any non-spliced spec, we can simplify
+    # by checking for spliced-in nodes by checking for nodes not in the build_spec
+    # without any explicit check for whether the spec is spliced.
+    # An analog in this algorithm is any spec that shares a name or provides the same virtuals
+    # in the context of the relevant root spec. This ensures that the analog for a spec s
+    # is the spec that s replaced when we spliced.
+    relocation_specs = deps_to_relocate(spec)
    build_spec_ids = set(id(s) for s in spec.build_spec.traverse(deptype=dt.ALL & ~dt.BUILD))
    for s in relocation_specs:
        analog = s
@@ -2223,48 +2269,98 @@ def relocate_package(spec: spack.spec.Spec) -> None:
        lookup_dag_hash = analog.dag_hash()
        if lookup_dag_hash in hash_to_old_prefix:
            old_dep_prefix = hash_to_old_prefix[lookup_dag_hash]
-            prefix_to_prefix[old_dep_prefix] = str(s.prefix)
+            prefix_to_prefix_bin[old_dep_prefix] = str(s.prefix)
+            prefix_to_prefix_text[old_dep_prefix] = str(s.prefix)

    # Only then add the generic fallback of install prefix -> install prefix.
-    prefix_to_prefix[old_layout_root] = str(spack.store.STORE.layout.root)
+    prefix_to_prefix_text[old_prefix] = new_prefix
+    prefix_to_prefix_bin[old_prefix] = new_prefix
+    prefix_to_prefix_text[old_layout_root] = new_layout_root
+    prefix_to_prefix_bin[old_layout_root] = new_layout_root

-    # Delete identity mappings from prefix_to_prefix
-    prefix_to_prefix = {k: v for k, v in prefix_to_prefix.items() if k != v}
+    # This is vestigial code for the *old* location of sbang. Previously,
+    # sbang was a bash script, and it lived in the spack prefix. It is
+    # now a POSIX script that lives in the install prefix. Old packages
+    # will have the old sbang location in their shebangs.
+    orig_sbang = "#!/bin/bash {0}/bin/sbang".format(old_spack_prefix)
+    new_sbang = spack.hooks.sbang.sbang_shebang_line()
+    prefix_to_prefix_text[orig_sbang] = new_sbang

-    # If there's nothing to relocate, we're done.
-    if not prefix_to_prefix:
-        return
+    tty.debug("Relocating package from", "%s to %s." % (old_layout_root, new_layout_root))

-    for old, new in prefix_to_prefix.items():
-        tty.debug(f"Relocating: {old} => {new}.")
+    # Old archives maybe have hardlinks repeated.
+    dedupe_hardlinks_if_necessary(workdir, buildinfo)

-    # Old archives may have hardlinks repeated.
-    dedupe_hardlinks_if_necessary(spec.prefix, buildinfo)
+    def is_backup_file(file):
+        return file.endswith("~")

    # Text files containing the prefix text
-    textfiles = [os.path.join(spec.prefix, f) for f in buildinfo["relocate_textfiles"]]
-    binaries = [os.path.join(spec.prefix, f) for f in buildinfo.get("relocate_binaries")]
-    links = [os.path.join(spec.prefix, f) for f in buildinfo.get("relocate_links", [])]
+    text_names = list()
+    for filename in buildinfo["relocate_textfiles"]:
+        text_name = os.path.join(workdir, filename)
+        # Don't add backup files generated by filter_file during install step.
+        if not is_backup_file(text_name):
+            text_names.append(text_name)

-    platform = spack.platforms.by_name(spec.platform)
-    if "macho" in platform.binary_formats:
-        relocate.relocate_macho_binaries(binaries, prefix_to_prefix)
-    elif "elf" in platform.binary_formats:
-        relocate.relocate_elf_binaries(binaries, prefix_to_prefix)
+    # If we are not installing back to the same install tree do the relocation
+    if old_prefix != new_prefix:
+        files_to_relocate = [
+            os.path.join(workdir, filename) for filename in buildinfo.get("relocate_binaries")
+        ]
+        # If the buildcache was not created with relativized rpaths
+        # do the relocation of path in binaries
+        platform = spack.platforms.by_name(spec.platform)
+        if "macho" in platform.binary_formats:
+            relocate.relocate_macho_binaries(
+                files_to_relocate,
+                old_layout_root,
+                new_layout_root,
+                prefix_to_prefix_bin,
+                rel,
+                old_prefix,
+                new_prefix,
+            )
+        elif "elf" in platform.binary_formats and not rel:
+            # The new ELF dynamic section relocation logic only handles absolute to
+            # absolute relocation.
+            relocate.new_relocate_elf_binaries(files_to_relocate, prefix_to_prefix_bin)
+        elif "elf" in platform.binary_formats and rel:
+            relocate.relocate_elf_binaries(
+                files_to_relocate,
+                old_layout_root,
+                new_layout_root,
+                prefix_to_prefix_bin,
+                rel,
+                old_prefix,
+                new_prefix,
+            )

-    relocate.relocate_links(links, prefix_to_prefix)
-    relocate.relocate_text(textfiles, prefix_to_prefix)
-    changed_files = relocate.relocate_text_bin(binaries, prefix_to_prefix)
+        # Relocate links to the new install prefix
+        links = [os.path.join(workdir, f) for f in buildinfo.get("relocate_links", [])]
+        relocate.relocate_links(links, prefix_to_prefix_bin)

-    # Add ad-hoc signatures to patched macho files when on macOS.
-    if "macho" in platform.binary_formats and sys.platform == "darwin":
-        codesign = which("codesign")
-        if not codesign:
-            return
-        for binary in changed_files:
-            # preserve the original inode by running codesign on a copy
-            with fsys.edit_in_place_through_temporary_file(binary) as tmp_binary:
-                codesign("-fs-", tmp_binary)
+        # For all buildcaches
+        # relocate the install prefixes in text files including dependencies
+        relocate.relocate_text(text_names, prefix_to_prefix_text)
+
+        # relocate the install prefixes in binary files including dependencies
+        changed_files = relocate.relocate_text_bin(files_to_relocate, prefix_to_prefix_bin)
+
+        # Add ad-hoc signatures to patched macho files when on macOS.
+        if "macho" in platform.binary_formats and sys.platform == "darwin":
+            codesign = which("codesign")
+            if not codesign:
+                return
+            for binary in changed_files:
+                # preserve the original inode by running codesign on a copy
+                with fsys.edit_in_place_through_temporary_file(binary) as tmp_binary:
+                    codesign("-fs-", tmp_binary)
+
+    # If we are installing back to the same location
+    # relocate the sbang location if the spack directory changed
+    else:
+        if old_spack_prefix != new_spack_prefix:
+            relocate.relocate_text(text_names, prefix_to_prefix_text)


 def _extract_inner_tarball(spec, filename, extract_to, signature_required: bool, remote_checksum):
@@ -2339,14 +2435,6 @@ def _tar_strip_component(tar: tarfile.TarFile, prefix: str):
        yield m


-def extract_buildcache_tarball(tarfile_path: str, destination: str) -> None:
-    with closing(tarfile.open(tarfile_path, "r")) as tar:
-        # Remove common prefix from tarball entries and directly extract them to the install dir.
-        tar.extractall(
-            path=destination, members=_tar_strip_component(tar, prefix=_ensure_common_prefix(tar))
-        )
-
-
 def extract_tarball(spec, download_result, force=False, timer=timer.NULL_TIMER):
    """
    extract binary tarball for given package into install area
@@ -2416,7 +2504,12 @@ def extract_tarball(spec, download_result, force=False, timer=timer.NULL_TIMER):
                tarfile_path, size, contents, "sha256", expected, local_checksum
            )
    try:
-        extract_buildcache_tarball(tarfile_path, destination=spec.prefix)
+        with closing(tarfile.open(tarfile_path, "r")) as tar:
+            # Remove install prefix from tarfil to extract directly into spec.prefix
+            tar.extractall(
+                path=spec.prefix,
+                members=_tar_strip_component(tar, prefix=_ensure_common_prefix(tar)),
+            )
    except Exception:
        shutil.rmtree(spec.prefix, ignore_errors=True)
        _delete_staged_downloads(download_result)
--- a/lib/spack/spack/bootstrap/init.py
+++ b/lib/spack/spack/bootstrap/init.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Function and classes needed to bootstrap Spack itself."""
@@ -8,6 +9,7 @@
    all_core_root_specs,
    ensure_clingo_importable_or_raise,
    ensure_core_dependencies,
+    ensure_file_in_path_or_raise,
    ensure_gpg_in_path_or_raise,
    ensure_patchelf_in_path_or_raise,
 )
@@ -18,6 +20,7 @@
    "is_bootstrapping",
    "ensure_bootstrap_configuration",
    "ensure_core_dependencies",
+    "ensure_file_in_path_or_raise",
    "ensure_gpg_in_path_or_raise",
    "ensure_clingo_importable_or_raise",
    "ensure_patchelf_in_path_or_raise",
--- a/lib/spack/spack/bootstrap/_common.py
+++ b/lib/spack/spack/bootstrap/_common.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Common basic functions used through the spack.bootstrap package"""
@@ -10,9 +11,7 @@
 import sys
 import sysconfig
 import warnings
-from typing import Optional, Sequence, Union
-
-from typing_extensions import TypedDict
+from typing import Dict, Optional, Sequence, Union

 import archspec.cpu

@@ -20,17 +19,13 @@
 from llnl.util import tty

 import spack.platforms
-import spack.spec
 import spack.store
 import spack.util.environment
 import spack.util.executable

 from .config import spec_for_current_python

-
-class QueryInfo(TypedDict, total=False):
-    spec: spack.spec.Spec
-    command: spack.util.executable.Executable
+QueryInfo = Dict[str, "spack.spec.Spec"]


 def _python_import(module: str) -> bool:
@@ -217,9 +212,7 @@ def _executables_in_store(
            ):
                spack.util.environment.path_put_first("PATH", [bin_dir])
                if query_info is not None:
-                    query_info["command"] = spack.util.executable.which(
-                        *executables, path=bin_dir, required=True
-                    )
+                    query_info["command"] = spack.util.executable.which(*executables, path=bin_dir)
                    query_info["spec"] = concrete_spec
                return True
    return False
--- a/lib/spack/spack/bootstrap/clingo.py
+++ b/lib/spack/spack/bootstrap/clingo.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Bootstrap concrete specs for clingo
--- a/lib/spack/spack/bootstrap/config.py
+++ b/lib/spack/spack/bootstrap/config.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Manage configuration swapping for bootstrapping purposes"""
--- a/lib/spack/spack/bootstrap/core.py
+++ b/lib/spack/spack/bootstrap/core.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Bootstrap Spack core dependencies from binaries.
@@ -34,7 +35,6 @@
 from llnl.util.lang import GroupedExceptionHandler

 import spack.binary_distribution
-import spack.concretize
 import spack.config
 import spack.detection
 import spack.mirrors.mirror
@@ -48,13 +48,7 @@
 import spack.version
 from spack.installer import PackageInstaller

-from ._common import (
-    QueryInfo,
-    _executables_in_store,
-    _python_import,
-    _root_spec,
-    _try_import_from_store,
-)
+from ._common import _executables_in_store, _python_import, _root_spec, _try_import_from_store
 from .clingo import ClingoBootstrapConcretizer
 from .config import spack_python_interpreter, spec_for_current_python

@@ -141,7 +135,7 @@ class BuildcacheBootstrapper(Bootstrapper):

    def __init__(self, conf) -> None:
        super().__init__(conf)
-        self.last_search: Optional[QueryInfo] = None
+        self.last_search: Optional[ConfigDictionary] = None
        self.config_scope_name = f"bootstrap_buildcache-{uuid.uuid4()}"

    @staticmethod
@@ -218,14 +212,14 @@ def _install_and_test(
                for _, pkg_hash, pkg_sha256 in item["binaries"]:
                    self._install_by_hash(pkg_hash, pkg_sha256, bincache_platform)

-                info: QueryInfo = {}
+                info: ConfigDictionary = {}
                if test_fn(query_spec=abstract_spec, query_info=info):
                    self.last_search = info
                    return True
        return False

    def try_import(self, module: str, abstract_spec_str: str) -> bool:
-        info: QueryInfo
+        info: ConfigDictionary
        test_fn, info = functools.partial(_try_import_from_store, module), {}
        if test_fn(query_spec=abstract_spec_str, query_info=info):
            return True
@@ -238,7 +232,7 @@ def try_import(self, module: str, abstract_spec_str: str) -> bool:
        return self._install_and_test(abstract_spec, bincache_platform, data, test_fn)

    def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool:
-        info: QueryInfo
+        info: ConfigDictionary
        test_fn, info = functools.partial(_executables_in_store, executables), {}
        if test_fn(query_spec=abstract_spec_str, query_info=info):
            self.last_search = info
@@ -256,11 +250,11 @@ class SourceBootstrapper(Bootstrapper):

    def __init__(self, conf) -> None:
        super().__init__(conf)
-        self.last_search: Optional[QueryInfo] = None
+        self.last_search: Optional[ConfigDictionary] = None
        self.config_scope_name = f"bootstrap_source-{uuid.uuid4()}"

    def try_import(self, module: str, abstract_spec_str: str) -> bool:
-        info: QueryInfo = {}
+        info: ConfigDictionary = {}
        if _try_import_from_store(module, abstract_spec_str, query_info=info):
            self.last_search = info
            return True
@@ -277,10 +271,10 @@ def try_import(self, module: str, abstract_spec_str: str) -> bool:
                bootstrapper = ClingoBootstrapConcretizer(configuration=spack.config.CONFIG)
                concrete_spec = bootstrapper.concretize()
            else:
-                abstract_spec = spack.spec.Spec(
+                concrete_spec = spack.spec.Spec(
                    abstract_spec_str + " ^" + spec_for_current_python()
                )
-                concrete_spec = spack.concretize.concretize_one(abstract_spec)
+                concrete_spec.concretize()

        msg = "[BOOTSTRAP MODULE {0}] Try installing '{1}' from sources"
        tty.debug(msg.format(module, abstract_spec_str))
@@ -295,7 +289,7 @@ def try_import(self, module: str, abstract_spec_str: str) -> bool:
        return False

    def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bool:
-        info: QueryInfo = {}
+        info: ConfigDictionary = {}
        if _executables_in_store(executables, abstract_spec_str, query_info=info):
            self.last_search = info
            return True
@@ -306,7 +300,7 @@ def try_search_path(self, executables: Tuple[str], abstract_spec_str: str) -> bo
        # might reduce compilation time by a fair amount
        _add_externals_if_missing()

-        concrete_spec = spack.concretize.concretize_one(abstract_spec_str)
+        concrete_spec = spack.spec.Spec(abstract_spec_str).concretized()
        msg = "[BOOTSTRAP] Try installing '{0}' from sources"
        tty.debug(msg.format(abstract_spec_str))
        with spack.config.override(self.mirror_scope):
@@ -421,7 +415,6 @@ def ensure_executables_in_path_or_raise(
                    current_bootstrapper.last_search["spec"],
                    current_bootstrapper.last_search["command"],
                )
-                assert cmd is not None, "expected an Executable"
                cmd.add_default_envmod(
                    spack.user_environment.environment_modifications_for_specs(
                        concrete_spec, set_package_py_globals=False
@@ -488,6 +481,19 @@ def ensure_gpg_in_path_or_raise() -> None:
    )


+def file_root_spec() -> str:
+    """Return the root spec used to bootstrap file"""
+    root_spec_name = "win-file" if IS_WINDOWS else "file"
+    return _root_spec(root_spec_name)
+
+
+def ensure_file_in_path_or_raise() -> None:
+    """Ensure file is in the PATH or raise"""
+    return ensure_executables_in_path_or_raise(
+        executables=["file"], abstract_spec=file_root_spec()
+    )
+
+
 def patchelf_root_spec() -> str:
    """Return the root spec used to bootstrap patchelf"""
    # 0.13.1 is the last version not to require C++17.
@@ -556,9 +562,10 @@ def ensure_winsdk_external_or_raise() -> None:
            missing_packages_lst.append("win-sdk")
        missing_packages = " & ".join(missing_packages_lst)
        raise RuntimeError(
-            f"Unable to find the {missing_packages}, please install these packages via the Visual "
-            "Studio installer before proceeding with Spack or provide the path to a non standard "
-            "install with 'spack external find --path'"
+            f"Unable to find the {missing_packages}, please install these packages \
+via the Visual Studio installer \
+before proceeding with Spack or provide the path to a non standard install with \
+'spack external find --path'"
        )
    # wgl/sdk are not required for bootstrapping Spack, but
    # are required for building anything non trivial
@@ -570,13 +577,15 @@ def ensure_core_dependencies() -> None:
    """Ensure the presence of all the core dependencies."""
    if sys.platform.lower() == "linux":
        ensure_patchelf_in_path_or_raise()
+    elif sys.platform == "win32":
+        ensure_file_in_path_or_raise()
    ensure_gpg_in_path_or_raise()
    ensure_clingo_importable_or_raise()


 def all_core_root_specs() -> List[str]:
    """Return a list of all the core root specs that may be used to bootstrap Spack"""
-    return [clingo_root_spec(), gnupg_root_spec(), patchelf_root_spec()]
+    return [clingo_root_spec(), gnupg_root_spec(), patchelf_root_spec(), file_root_spec()]


 def bootstrapping_sources(scope: Optional[str] = None):
--- a/lib/spack/spack/bootstrap/environment.py
+++ b/lib/spack/spack/bootstrap/environment.py
@@ -1,4 +1,5 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Bootstrap non-core Spack dependencies from an environment."""
--- a/lib/spack/spack/bootstrap/status.py
+++ b/lib/spack/spack/bootstrap/status.py
@@ -1,9 +1,10 @@
-# Copyright Spack Project Developers. See COPYRIGHT file for details.
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Query the status of bootstrapping on this machine"""
-import sys
-from typing import Dict, List, Optional, Sequence, Tuple, Union
+import platform
+from typing import List, Optional, Sequence, Tuple, Union

 import spack.util.executable

@@ -71,7 +72,7 @@ def _core_requirements() -> List[RequiredResponseType]:
        "bzip2": _missing("bzip2", "required to compress/decompress code archives"),
        "git": _missing("git", "required to fetch/manage git repositories"),
    }
-    if sys.platform == "linux":
+    if platform.system().lower() == "linux":
        _core_system_exes["xz"] = _missing("xz", "required to compress/decompress code archives")

    # Executables that are not bootstrapped yet
@@ -86,16 +87,17 @@ def _core_requirements() -> List[RequiredResponseType]:


 def _buildcache_requirements() -> List[RequiredResponseType]:
-    _buildcache_exes: Dict[ExecutablesType, str] = {
-        ("gpg2", "gpg"): _missing("gpg2", "required to sign/verify buildcaches", False)
+    _buildcache_exes = {
+        "file": _missing("file", "required to analyze files for buildcaches", system_only=False),
+        ("gpg2", "gpg"): _missing("gpg2", "required to sign/verify buildcaches", False),
    }
-    if sys.platform == "darwin":
+    if platform.system().lower() == "darwin":
        _buildcache_exes["otool"] = _missing("otool", "required to relocate binaries")

    # Executables that are not bootstrapped yet
    result = [_required_system_executable(exe, msg) for exe, msg in _buildcache_exes.items()]

-    if sys.platform == "linux":
+    if platform.system().lower() == "linux":
        result.append(
            _required_executable(
                "patchelf",
--- a/Show More
+++ b/Show More