Revert "Revert "Revert "gitlab: Add shared PR mirror to places pipelines look for binaries. (#33746)" (#34087)" (#34153)"

This reverts commit efa1dba9e4.

.github/workflows/audit.yaml

@@ -25,7 +25,7 @@ jobs:
         python-version: ${{inputs.python_version}}
     - name: Install Python packages
       run: |
-        pip install --upgrade pip six setuptools pytest codecov 'coverage[toml]<=6.2'
+        pip install --upgrade pip six setuptools pytest codecov coverage[toml]
     - name: Package audits (with coverage)
       if: ${{ inputs.with_coverage == 'true' }}
       run: |

.github/workflows/bootstrap.yml

@@ -214,7 +214,7 @@ jobs:
       - name: Bootstrap clingo
         run: |
           set -ex
-          for ver in '2.7' '3.6' '3.7' '3.8' '3.9' '3.10' ; do
+          for ver in '3.6' '3.7' '3.8' '3.9' '3.10' ; do
             not_found=1
             ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)"
             echo "Testing $ver_dir"

.github/workflows/build-containers.yml

@@ -80,16 +80,16 @@ jobs:
           fi
 
       - name: Upload Dockerfile
-        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
         with:
           name: dockerfiles
           path: dockerfiles
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 # @v1
+        uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # @v1
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@c74574e6c82eeedc46366be1b0d287eff9085eb6 # @v1
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # @v1
 
       - name: Log in to GitHub Container Registry
         uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # @v1
@@ -106,7 +106,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build & Deploy ${{ matrix.dockerfile[0] }}
-        uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 # @v2
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # @v2
         with:
           context: dockerfiles/${{ matrix.dockerfile[0] }}
           platforms: ${{ matrix.dockerfile[1] }}

.github/workflows/ci.yaml

@@ -20,12 +20,6 @@ jobs:
     uses: ./.github/workflows/valid-style.yml
     with:
       with_coverage: ${{ needs.changes.outputs.core }}
-  audit-ancient-python:
-    uses: ./.github/workflows/audit.yaml
-    needs: [ changes ]
-    with:
-      with_coverage: ${{ needs.changes.outputs.core }}
-      python_version: 2.7
   all-prechecks:
     needs: [ prechecks ]
     runs-on: ubuntu-latest
@@ -46,7 +40,7 @@ jobs:
         with:
           fetch-depth: 0
             # For pull requests it's not necessary to checkout the code
-      - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
+      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50
         id: filter
         with:
           # See https://github.com/dorny/paths-filter/issues/56 for the syntax used below
@@ -85,7 +79,7 @@ jobs:
     needs: [ prechecks ]
     uses: ./.github/workflows/windows_python.yml
   all:
-    needs: [ windows, unit-tests, bootstrap, audit-ancient-python ]
+    needs: [ windows, unit-tests, bootstrap ]
     runs-on: ubuntu-latest
     steps:
     - name: Success

.github/workflows/unit_tests.yaml

@@ -11,28 +11,38 @@ concurrency:
 jobs:
   # Run unit tests with different configurations on linux
   ubuntu:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10']
+        os: [ubuntu-latest]
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
         concretizer: ['clingo']
         on_develop:
         - ${{ github.ref == 'refs/heads/develop' }}
         include:
-        - python-version: 2.7
+        - python-version: '3.11'
+          os: ubuntu-latest
           concretizer: original
           on_develop: ${{ github.ref == 'refs/heads/develop' }}
-        - python-version: '3.10'
-          concretizer: original
+        - python-version: '3.6'
+          os: ubuntu-20.04
+          concretizer: clingo
           on_develop: ${{ github.ref == 'refs/heads/develop' }}
         exclude:
         - python-version: '3.7'
+          os: ubuntu-latest
           concretizer: 'clingo'
           on_develop: false
         - python-version: '3.8'
+          os: ubuntu-latest
           concretizer: 'clingo'
           on_develop: false
         - python-version: '3.9'
+          os: ubuntu-latest
+          concretizer: 'clingo'
+          on_develop: false
+        - python-version: '3.10'
+          os: ubuntu-latest
           concretizer: 'clingo'
           on_develop: false
 
@@ -49,24 +59,11 @@ jobs:
           # Needed for unit tests
           sudo apt-get -y install \
               coreutils cvs gfortran graphviz gnupg2 mercurial ninja-build \
-              patchelf cmake bison libbison-dev kcov
+              cmake bison libbison-dev kcov
     - name: Install Python packages
       run: |
-          pip install --upgrade pip six setuptools pytest codecov[toml] pytest-xdist 
-          # Install pytest-cov only on recent Python, to avoid stalling on Python 2.7 due
-          # to bugs on an unmaintained version of the package when used with xdist.
-          if [[ ${{ matrix.python-version }} != "2.7" ]]; then
-            pip install --upgrade pytest-cov
-          fi
-          # ensure style checks are not skipped in unit tests for python >= 3.6
-          # note that true/false (i.e., 1/0) are opposite in conditions in python and bash
-          if python -c 'import sys; sys.exit(not sys.version_info >= (3, 6))'; then
-              pip install --upgrade flake8 "isort>=4.3.5" "mypy>=0.900" "click==8.0.4" "black<=21.12b0"
-          fi
-    - name: Pin pathlib for Python 2.7
-      if: ${{ matrix.python-version == 2.7 }}
-      run: |
-          pip install -U pathlib2==2.3.6 toml
+          pip install --upgrade pip six setuptools pytest codecov[toml] pytest-xdist pytest-cov
+          pip install --upgrade flake8 "isort>=4.3.5" "mypy>=0.900" "click" "black"
     - name: Setup git configuration
       run: |
           # Need this for the git tests to succeed.
@@ -79,6 +76,7 @@ jobs:
       run: |
           . share/spack/setup-env.sh
           spack bootstrap disable spack-install
+          spack bootstrap now
           spack -v solve zlib
     - name: Run unit tests
       env:
@@ -86,7 +84,7 @@ jobs:
           SPACK_TEST_SOLVER: ${{ matrix.concretizer }}
           SPACK_TEST_PARALLEL: 2
           COVERAGE: true
-          UNIT_TEST_COVERAGE: ${{ (matrix.python-version == '3.10') }}
+          UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }}
       run: |
           share/spack/qa/run-unit-tests
     - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70
@@ -101,7 +99,7 @@ jobs:
         fetch-depth: 0
     - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # @v2
       with:
-        python-version: '3.10'
+        python-version: '3.11'
     - name: Install System packages
       run: |
           sudo apt-get -y update
@@ -109,7 +107,7 @@ jobs:
           sudo apt-get install -y coreutils kcov csh zsh tcsh fish dash bash
     - name: Install Python packages
       run: |
-          pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 pytest-xdist
+          pip install --upgrade pip six setuptools pytest codecov coverage[toml] pytest-xdist
     - name: Setup git configuration
       run: |
           # Need this for the git tests to succeed.
@@ -158,14 +156,11 @@ jobs:
         fetch-depth: 0
     - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # @v2
       with:
-        python-version: '3.10'
+        python-version: '3.11'
     - name: Install System packages
       run: |
           sudo apt-get -y update
-          # Needed for unit tests
-          sudo apt-get -y install \
-              coreutils cvs gfortran graphviz gnupg2 mercurial ninja-build \
-              patchelf kcov
+          sudo apt-get -y install coreutils cvs gfortran graphviz gnupg2 mercurial ninja-build kcov
     - name: Install Python packages
       run: |
           pip install --upgrade pip six setuptools pytest codecov coverage[toml] pytest-cov clingo pytest-xdist

.github/workflows/valid-style.yml

@@ -21,16 +21,16 @@ jobs:
     - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # @v2
     - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # @v2
       with:
-        python-version: '3.10'
+        python-version: '3.11'
         cache: 'pip'
     - name: Install Python Packages
       run: |
         pip install --upgrade pip
         pip install --upgrade vermin
     - name: vermin (Spack's Core)
-      run: vermin --backport argparse --violations --backport typing -t=2.7- -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
+      run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
     - name: vermin (Repositories)
-      run: vermin --backport argparse --violations --backport typing -t=2.7- -t=3.6- -vvv var/spack/repos
+      run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv var/spack/repos
   # Run style checks on the files that have been changed
   style:
     runs-on: ubuntu-latest
@@ -40,11 +40,11 @@ jobs:
         fetch-depth: 0
     - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # @v2
       with:
-        python-version: '3.10'
+        python-version: '3.11'
         cache: 'pip'
     - name: Install Python packages
       run: |
-        python3 -m pip install --upgrade pip six setuptools types-six click==8.0.2 'black==21.12b0' mypy isort clingo flake8
+        python3 -m pip install --upgrade pip six setuptools types-six black mypy isort clingo flake8
     - name: Setup git configuration
       run: |
         # Need this for the git tests to succeed.
@@ -57,4 +57,4 @@ jobs:
     uses: ./.github/workflows/audit.yaml
     with:
       with_coverage: ${{ inputs.with_coverage }}
-      python_version: '3.10'
+      python_version: '3.11'

.github/workflows/windows_python.yml

@@ -109,11 +109,11 @@ jobs:
         echo "installer_root=$((pwd).Path)" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
       env:
         ProgressPreference: SilentlyContinue
-    - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+    - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
       with:
         name: Windows Spack Installer Bundle
         path: ${{ env.installer_root }}\pkg\Spack.exe
-    - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+    - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
       with:
         name: Windows Spack Installer
         path: ${{ env.installer_root}}\pkg\Spack.msi

CHANGELOG.md

@@ -1,16 +1,284 @@
+# v0.19.0 (2022-11-11)
+
+`v0.19.0` is a major feature release.
+
+## Major features in this release
+
+1. **Package requirements**
+
+   Spack's traditional [package preferences](
+     https://spack.readthedocs.io/en/latest/build_settings.html#package-preferences)
+   are soft, but we've added hard requriements to `packages.yaml` and `spack.yaml`
+   (#32528, #32369). Package requirements use the same syntax as specs:
+
+   ```yaml
+   packages:
+     libfabric:
+       require: "@1.13.2"
+     mpich:
+       require:
+       - one_of: ["+cuda", "+rocm"]
+   ```
+
+   More details in [the docs](
+     https://spack.readthedocs.io/en/latest/build_settings.html#package-requirements).
+
+2. **Environment UI Improvements**
+
+   * Fewer surprising modifications to `spack.yaml` (#33711):
+
+     * `spack install` in an environment will no longer add to the `specs:` list; you'll
+       need to either use `spack add <spec>` or `spack install --add <spec>`.
+
+     * Similarly, `spack uninstall` will not remove from your environment's `specs:`
+       list; you'll need to use `spack remove` or `spack uninstall --remove`.
+
+     This will make it easier to manage an environment, as there is clear separation
+     between the stack to be installed (`spack.yaml`/`spack.lock`) and which parts of
+     it should be installed (`spack install` / `spack uninstall`).
+
+   * `concretizer:unify:true` is now the default mode for new environments (#31787)
+
+     We see more users creating `unify:true` environments now. Users who need
+     `unify:false` can add it to their environment to get the old behavior. This will
+     concretize every spec in the environment independently.
+
+   * Include environment configuration from URLs (#29026, [docs](
+       https://spack.readthedocs.io/en/latest/environments.html#included-configurations))
+
+     You can now include configuration in your environment directly from a URL:
+
+     ```yaml
+     spack:
+       include:
+       - https://github.com/path/to/raw/config/compilers.yaml
+     ```
+
+4. **Multiple Build Systems**
+
+   An increasing number of packages in the ecosystem need the ability to support
+   multiple build systems (#30738, [docs](
+     https://spack.readthedocs.io/en/latest/packaging_guide.html#multiple-build-systems)),
+   either across versions, across platforms, or within the same version of the software.
+   This has been hard to support through multiple inheritance, as methods from different
+   build system superclasses would conflict. `package.py` files can now define separate
+   builder classes with installation logic for different build systems, e.g.:
+
+   ```python
+   class ArpackNg(CMakePackage, AutotoolsPackage):
+
+       build_system(
+           conditional("cmake", when="@0.64:"),
+           conditional("autotools", when="@:0.63"),
+           default="cmake",
+       )
+
+   class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder):
+       def cmake_args(self):
+           pass
+
+   class Autotoolsbuilder(spack.build_systems.autotools.AutotoolsBuilder):
+       def configure_args(self):
+           pass
+   ```
+
+5. **Compiler and variant propagation**
+
+   Currently, compiler flags and variants are inconsistent: compiler flags set for a
+   package are inherited by its dependencies, while variants are not. We should have
+   these be consistent by allowing for inheritance to be enabled or disabled for both
+   variants and compiler flags.
+
+   Example syntax:
+   - `package ++variant`:
+         enabled variant that will be propagated to dependencies
+   - `package +variant`:
+         enabled variant that will NOT be propagated to dependencies
+   - `package ~~variant`:
+         disabled variant that will be propagated to dependencies
+   - `package ~variant`:
+         disabled variant that will NOT be propagated to dependencies
+   - `package cflags==-g`:
+         `cflags` will be propagated to dependencies
+   - `package cflags=-g`:
+         `cflags` will NOT be propagated to dependencies
+
+   Syntax for non-boolan variants is similar to compiler flags. More in the docs for
+   [variants](
+     https://spack.readthedocs.io/en/latest/basic_usage.html#variants) and [compiler flags](
+     https://spack.readthedocs.io/en/latest/basic_usage.html#compiler-flags).
+
+6. **Enhancements to git version specifiers**
+
+   * `v0.18.0` added the ability to use git commits as versions. You can now use the
+     `git.` prefix to specify git tags or branches as versions. All of these are valid git
+     versions in `v0.19` (#31200):
+
+     ```console
+     foo@abcdef1234abcdef1234abcdef1234abcdef1234      # raw commit
+     foo@git.abcdef1234abcdef1234abcdef1234abcdef1234  # commit with git prefix
+     foo@git.develop                                   # the develop branch
+     foo@git.0.19                                      # use the 0.19 tag
+     ```
+
+   * `v0.19` also gives you more control over how Spack interprets git versions, in case
+     Spack cannot detect the version from the git repository. You can suffix a git
+     version with `=<version>` to force Spack to concretize it as a particular version
+     (#30998, #31914, #32257):
+
+     ```console
+     # use mybranch, but treat it as version 3.2 for version comparison
+     foo@git.mybranch=3.2
+
+     # use the given commit, but treat it as develop for version comparison
+     foo@git.abcdef1234abcdef1234abcdef1234abcdef1234=develop
+     ```
+
+     More in [the docs](
+       https://spack.readthedocs.io/en/latest/basic_usage.html#version-specifier)
+
+7. **Changes to Cray EX Support**
+
+   Cray machines have historically had their own "platform" within Spack, because we
+   needed to go through the module system to leverage compilers and MPI installations on
+   these machines. The Cray EX programming environment now provides standalone `craycc`
+   executables and proper `mpicc` wrappers, so Spack can treat EX machines like Linux
+   with extra packages (#29392).
+
+   We expect this to greatly reduce bugs, as external packages and compilers can now be
+   used by prefix instead of through modules. We will also no longer be subject to
+   reproducibility issues when modules change from Cray PE release to release and from
+   site to site. This also simplifies dealing with the underlying Linux OS on cray
+   systems, as Spack will properly model the machine's OS as either SuSE or RHEL.
+
+8. **Improvements to tests and testing in CI**
+
+   * `spack ci generate --tests` will generate a `.gitlab-ci.yml` file that not only does
+     builds but also runs tests for built packages (#27877). Public GitHub pipelines now
+     also run tests in CI.
+
+   * `spack test run --explicit` will only run tests for packages that are explicitly
+     installed, instead of all packages.
+
+9. **Experimental binding link model**
+
+   You can add a new option to `config.yaml` to make Spack embed absolute paths to
+   needed shared libraries in ELF executables and shared libraries on Linux (#31948, [docs](
+     https://spack.readthedocs.io/en/latest/config_yaml.html#shared-linking-bind)):
+
+   ```yaml
+   config:
+     shared_linking:
+       type: rpath
+       bind: true
+   ```
+
+   This can improve launch time at scale for parallel applications, and it can make
+   installations less susceptible to environment variables like `LD_LIBRARY_PATH`, even
+   especially when dealing with external libraries that use `RUNPATH`. You can think of
+   this as a faster, even higher-precedence version of `RPATH`.
+
+## Other new features of note
+
+* `spack spec` prints dependencies more legibly. Dependencies in the output now appear
+  at the *earliest* level of indentation possible (#33406)
+* You can override `package.py` attributes like `url`, directly in `packages.yaml`
+  (#33275, [docs](
+    https://spack.readthedocs.io/en/latest/build_settings.html#assigning-package-attributes))
+* There are a number of new architecture-related format strings you can use in Spack
+  configuration files to specify paths (#29810, [docs](
+    https://spack.readthedocs.io/en/latest/configuration.html#config-file-variables))
+* Spack now supports bootstrapping Clingo on Windows (#33400)
+* There is now support for an `RPATH`-like library model on Windows (#31930)
+
+## Performance Improvements
+
+* Major performance improvements for installation from binary caches (#27610, #33628,
+  #33636, #33608, #33590, #33496)
+* Test suite can now be parallelized using `xdist` (used in GitHub Actions) (#32361)
+* Reduce lock contention for parallel builds in environments (#31643)
+
+## New binary caches and stacks
+
+* We now build nearly all of E4S with `oneapi` in our buildcache (#31781, #31804,
+  #31804, #31803, #31840, #31991, #32117, #32107, #32239)
+* Added 3 new machine learning-centric stacks to binary cache: `x86_64_v3`, CUDA, ROCm
+  (#31592, #33463)
+
+## Removals and Deprecations
+
+* Support for Python 3.5 is dropped (#31908). Only Python 2.7 and 3.6+ are officially
+  supported.
+
+* This is the last Spack release that will support Python 2 (#32615). Spack `v0.19`
+  will emit a deprecation warning if you run it with Python 2, and Python 2 support will
+  soon be removed from the `develop` branch.
+
+* `LD_LIBRARY_PATH` is no longer set by default by `spack load` or module loads.
+
+  Setting `LD_LIBRARY_PATH` in Spack environments/modules can cause binaries from
+  outside of Spack to crash, and Spack's own builds use `RPATH` and do not need
+  `LD_LIBRARY_PATH` set in order to run. If you still want the old behavior, you
+  can run these commands to configure Spack to set `LD_LIBRARY_PATH`:
+
+  ```console
+  spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]
+  spack config add modules:prefix_inspections:lib:[LD_LIBRARY_PATH]
+  ```
+
+* The `spack:concretization:[together|separately]` has been removed after being
+  deprecated in `v0.18`. Use `concretizer:unify:[true|false]`.
+* `config:module_roots` is no longer supported after being deprecated in `v0.18`. Use
+  configuration in module sets instead (#28659, [docs](
+    https://spack.readthedocs.io/en/latest/module_file_support.html)).
+* `spack activate` and `spack deactivate` are no longer supported, having been
+  deprecated in `v0.18`. Use an environment with a view instead of
+  activating/deactivating ([docs](
+    https://spack.readthedocs.io/en/latest/environments.html#configuration-in-spack-yaml)).
+* The old YAML format for buildcaches is now deprecated (#33707). If you are using an
+  old buildcache with YAML metadata you will need to regenerate it with JSON metadata.
+* `spack bootstrap trust` and `spack bootstrap untrust` are deprecated in favor of
+  `spack bootstrap enable` and `spack bootstrap disable` and will be removed in `v0.20`.
+  (#33600)
+* The `graviton2` architecture has been renamed to `neoverse_n1`, and `graviton3`
+  is now `neoverse_v1`. Buildcaches using the old architecture names will need to be rebuilt.
+* The terms `blacklist` and `whitelist` have been replaced with `include` and `exclude`
+  in all configuration files (#31569). You can use `spack config update` to
+  automatically fix your configuration files.
+
+## Notable Bugfixes
+
+* Permission setting on installation now handles effective uid properly (#19980)
+* `buildable:true` for an MPI implementation now overrides `buildable:false` for `mpi` (#18269)
+* Improved error messages when attempting to use an unconfigured compiler (#32084)
+* Do not punish explicitly requested compiler mismatches in the solver (#30074)
+* `spack stage`: add missing --fresh and --reuse (#31626)
+* Fixes for adding build system executables like `cmake` to package scope (#31739)
+* Bugfix for binary relocation with aliased strings produced by newer `binutils` (#32253)
+
+## Spack community stats
+
+* 6,751 total packages, 335 new since `v0.18.0`
+    * 141 new Python packages
+    * 89 new R packages
+* 303 people contributed to this release
+    * 287 committers to packages
+    * 57 committers to core
+
+
 # v0.18.1 (2022-07-19)
 
 ### Spack Bugfixes
 * Fix several bugs related to bootstrapping (#30834,#31042,#31180)
-* Fix a regression that was causing spec hashes to differ between 
+* Fix a regression that was causing spec hashes to differ between
   Python 2 and Python 3 (#31092)
 * Fixed compiler flags for oneAPI and DPC++ (#30856)
 * Fixed several issues related to concretization (#31142,#31153,#31170,#31226)
 * Improved support for Cray manifest file and `spack external find` (#31144,#31201,#31173,#31186)
 * Assign a version to openSUSE Tumbleweed according to the GLIBC version
-  in the system (#19895) 
+  in the system (#19895)
 * Improved Dockerfile generation for `spack containerize` (#29741,#31321)
-* Fixed a few bugs related to concurrent execution of commands (#31509,#31493,#31477)  
+* Fixed a few bugs related to concurrent execution of commands (#31509,#31493,#31477)
 
 ### Package updates
 * WarpX: add v22.06, fixed libs property (#30866,#31102)

SECURITY.md

@@ -10,8 +10,8 @@ For more on Spack's release structure, see
 | Version | Supported          |
 | ------- | ------------------ |
 | develop | :white_check_mark: |
-| 0.17.x  | :white_check_mark: |
-| 0.16.x  | :white_check_mark: |
+| 0.19.x  | :white_check_mark: |
+| 0.18.x  | :white_check_mark: |
 
 ## Reporting a Vulnerability
 

bin/spack

@@ -31,13 +31,11 @@ import os
 import os.path
 import sys
 
-min_python3 = (3, 5)
+min_python3 = (3, 6)
 
-if sys.version_info[:2] < (2, 7) or (
-    sys.version_info[:2] >= (3, 0) and sys.version_info[:2] < min_python3
-):
+if sys.version_info[:2] < min_python3:
     v_info = sys.version_info[:3]
-    msg = "Spack requires Python 2.7 or %d.%d or higher " % min_python3
+    msg = "Spack requires Python %d.%d or higher " % min_python3
     msg += "You are running spack with Python %d.%d.%d." % v_info
     sys.exit(msg)
 
@@ -49,52 +47,8 @@ spack_prefix = os.path.dirname(os.path.dirname(spack_file))
 spack_lib_path = os.path.join(spack_prefix, "lib", "spack")
 sys.path.insert(0, spack_lib_path)
 
-# Add external libs
-spack_external_libs = os.path.join(spack_lib_path, "external")
-
-if sys.version_info[:2] <= (2, 7):
-    sys.path.insert(0, os.path.join(spack_external_libs, "py2"))
-
-sys.path.insert(0, spack_external_libs)
-
-# Here we delete ruamel.yaml in case it has been already imported from site
-# (see #9206 for a broader description of the issue).
-#
-# Briefly: ruamel.yaml produces a .pth file when installed with pip that
-# makes the site installed package the preferred one, even though sys.path
-# is modified to point to another version of ruamel.yaml.
-if "ruamel.yaml" in sys.modules:
-    del sys.modules["ruamel.yaml"]
-
-if "ruamel" in sys.modules:
-    del sys.modules["ruamel"]
-
-# The following code is here to avoid failures when updating
-# the develop version, due to spurious argparse.pyc files remaining
-# in the libs/spack/external directory, see:
-# https://github.com/spack/spack/pull/25376
-# TODO: Remove in v0.18.0 or later
-try:
-    import argparse
-except ImportError:
-    argparse_pyc = os.path.join(spack_external_libs, "argparse.pyc")
-    if not os.path.exists(argparse_pyc):
-        raise
-    try:
-        os.remove(argparse_pyc)
-        import argparse  # noqa: F401
-    except Exception:
-        msg = (
-            "The file\n\n\t{0}\n\nis corrupted and cannot be deleted by Spack. "
-            "Either delete it manually or ask some administrator to "
-            "delete it for you."
-        )
-        print(msg.format(argparse_pyc))
-        sys.exit(1)
-
-
-import spack.main  # noqa: E402
+from spack_installable.main import main  # noqa: E402
 
 # Once we've set up the system path, run the spack main method
 if __name__ == "__main__":
-    sys.exit(spack.main.main())
+    sys.exit(main())

etc/spack/defaults/concretizer.yaml

@@ -33,4 +33,4 @@ concretizer:
   # environments can always be activated. When "false" perform concretization separately
   # on each root spec, allowing different versions and variants of the same package in
   # an environment.
-  unify: false
+  unify: true

etc/spack/defaults/config.yaml

@@ -19,7 +19,7 @@ config:
   install_tree:
     root: $spack/opt/spack
     projections:
-      all: "${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}"
+      all: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}"
     # install_tree can include an optional padded length (int or boolean)
     # default is False (do not pad)
     # if padded_length is True, Spack will pad as close to the system max path
@@ -214,4 +214,8 @@ config:
 
   # Number of seconds a buildcache's index.json is cached locally before probing
   # for updates, within a single Spack invocation. Defaults to 10 minutes.
-  binary_index_ttl: 600
+  binary_index_ttl: 600
+
+  flags:
+    # Whether to keep -Werror flags active in package builds.
+    keep_werror: 'none'

lib/spack/docs/basic_usage.rst

@@ -1114,21 +1114,21 @@ set of arbitrary versions, such as ``@1.0,1.5,1.7`` (``1.0``, ``1.5``,
 or ``1.7``).  When you supply such a specifier to ``spack install``,
 it constrains the set of versions that Spack will install.
 
-For packages with a ``git`` attribute, ``git`` references 
-may be specified instead of a numerical version i.e. branches, tags 
-and commits. Spack will stage and build based off the ``git`` 
+For packages with a ``git`` attribute, ``git`` references
+may be specified instead of a numerical version i.e. branches, tags
+and commits. Spack will stage and build based off the ``git``
 reference provided.  Acceptable syntaxes for this are:
 
 .. code-block:: sh
-   
+
     # branches and tags
    foo@git.develop # use the develop branch
    foo@git.0.19 # use the 0.19 tag
-   
+
     # commit hashes
    foo@abcdef1234abcdef1234abcdef1234abcdef1234    # 40 character hashes are automatically treated as git commits
    foo@git.abcdef1234abcdef1234abcdef1234abcdef1234
-   
+
 Spack versions from git reference either have an associated version supplied by the user,
 or infer a relationship to known versions from the structure of the git repository. If an
 associated version is supplied by the user, Spack treats the git version as equivalent to that
@@ -1244,8 +1244,8 @@ For example, for the ``stackstart`` variant:
 
 .. code-block:: sh
 
-    mpileaks stackstart=4    # variant will be propagated to dependencies
-    mpileaks stackstart==4   # only mpileaks will have this variant value
+    mpileaks stackstart==4   # variant will be propagated to dependencies
+    mpileaks stackstart=4    # only mpileaks will have this variant value
 
 ^^^^^^^^^^^^^^
 Compiler Flags
@@ -1672,9 +1672,13 @@ own install prefix.  However, certain packages are typically installed
 `Python <https://www.python.org>`_ packages are typically installed in the
 ``$prefix/lib/python-2.7/site-packages`` directory.
 
-Spack has support for this type of installation as well.  In Spack,
-a package that can live inside the prefix of another package is called
-an *extension*.  Suppose you have Python installed like so:
+In Spack, installation prefixes are immutable, so this type of installation
+is not directly supported. However, it is possible to create views that
+allow you to merge install prefixes of multiple packages into a single new prefix.
+Views are a convenient way to get a more traditional filesystem structure.
+Using *extensions*, you can ensure that Python packages always share the
+same prefix in the view as Python itself. Suppose you have
+Python installed like so:
 
 .. code-block:: console
 
@@ -1712,8 +1716,6 @@ You can find extensions for your Python installation like this:
    py-ipython@2.3.1     py-pygments@2.0.1   py-setuptools@11.3.1
    py-matplotlib@1.4.2  py-pyparsing@2.0.3  py-six@1.9.0
 
-   ==> None activated.
-
 The extensions are a subset of what's returned by ``spack list``, and
 they are packages like any other.  They are installed into their own
 prefixes, and you can see this with ``spack find --paths``:
@@ -1741,32 +1743,72 @@ directly when you run ``python``:
    ImportError: No module named numpy
    >>>
 
-^^^^^^^^^^^^^^^^
-Using Extensions
-^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Using Extensions in Environments
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-There are four ways to get ``numpy`` working in Python.  The first is
-to use :ref:`shell-support`.  You can simply ``load`` the extension,
-and it will be added to the ``PYTHONPATH`` in your current shell:
+The recommended way of working with extensions such as ``py-numpy``
+above is through :ref:`Environments <environments>`. For example,
+the following creates an environment in the current working directory
+with a filesystem view in the ``./view`` directory:
 
 .. code-block:: console
 
-   $ spack load python
-   $ spack load py-numpy
+   $ spack env create --with-view view --dir .
+   $ spack -e . add py-numpy
+   $ spack -e . concretize
+   $ spack -e . install
+
+We recommend environments for two reasons. Firstly, environments
+can be activated (requires :ref:`shell-support`):
+
+.. code-block:: console
+
+   $ spack env activate .
+
+which sets all the right environment variables such as ``PATH`` and
+``PYTHONPATH``. This ensures that
+
+.. code-block:: console
+
+   $ python
+   >>> import numpy
+
+works. Secondly, even without shell support, the view ensures
+that Python can locate its extensions:
+
+.. code-block:: console
+
+   $ ./view/bin/python
+   >>> import numpy
+
+See :ref:`environments` for a more in-depth description of Spack
+environments and customizations to views.
+
+^^^^^^^^^^^^^^^^^^^^
+Using ``spack load``
+^^^^^^^^^^^^^^^^^^^^
+
+A more traditional way of using Spack and extensions is ``spack load``
+(requires :ref:`shell-support`). This will add the extension to ``PYTHONPATH``
+in your current shell, and Python itself will be available in the ``PATH``:
+
+.. code-block:: console
+
+   $ spack load py-numpy
+   $ python
+   >>> import numpy
 
-Now ``import numpy`` will succeed for as long as you keep your current
-session open.
 The loaded packages can be checked using ``spack find --loaded``
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Loading Extensions via Modules
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Instead of using Spack's environment modification capabilities through
-the ``spack load`` command, you can load numpy through your
-environment modules (using ``environment-modules`` or ``lmod``). This
-will also add the extension to the ``PYTHONPATH`` in your current
-shell.
+Apart from ``spack env activate`` and ``spack load``, you can load numpy
+through your environment modules (using ``environment-modules`` or
+``lmod``). This will also add the extension to the ``PYTHONPATH`` in
+your current shell.
 
 .. code-block:: console
 
@@ -1776,130 +1818,6 @@ If you do not know the name of the specific numpy module you wish to
 load, you can use the ``spack module tcl|lmod loads`` command to get
 the name of the module from the Spack spec.
 
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Activating Extensions in a View
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Another way to use extensions is to create a view, which merges the
-python installation along with the extensions into a single prefix.
-See :ref:`configuring_environment_views` for a more in-depth description
-of views.
-
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Activating Extensions Globally
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-As an alternative to creating a merged prefix with Python and its extensions,
-and prior to support for views, Spack has provided a means to install the
-extension into the Spack installation prefix for the extendee. This has
-typically been useful since extendable packages typically search their own
-installation path for addons by default.
-
-Global activations are performed with the ``spack activate`` command:
-
-.. _cmd-spack-activate:
-
-^^^^^^^^^^^^^^^^^^
-``spack activate``
-^^^^^^^^^^^^^^^^^^
-
-.. code-block:: console
-
-   $ spack activate py-numpy
-   ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7 arch=linux-debian7-x86_64-3c74eb69 for python@2.7.8%gcc@4.4.7.
-   ==> Activated extension py-nose@1.3.4%gcc@4.4.7 arch=linux-debian7-x86_64-5f70f816 for python@2.7.8%gcc@4.4.7.
-   ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=linux-debian7-x86_64-66733244 for python@2.7.8%gcc@4.4.7.
-
-Several things have happened here.  The user requested that
-``py-numpy`` be activated in the ``python`` installation it was built
-with.  Spack knows that ``py-numpy`` depends on ``py-nose`` and
-``py-setuptools``, so it activated those packages first.  Finally,
-once all dependencies were activated in the ``python`` installation,
-``py-numpy`` was activated as well.
-
-If we run ``spack extensions`` again, we now see the three new
-packages listed as activated:
-
-.. code-block:: console
-
-   $ spack extensions python
-   ==> python@2.7.8%gcc@4.4.7  arch=linux-debian7-x86_64-703c7a96
-   ==> 36 extensions:
-   geos          py-ipython     py-pexpect    py-pyside            py-sip
-   py-basemap    py-libxml2     py-pil        py-pytz              py-six
-   py-biopython  py-mako        py-pmw        py-rpy2              py-sympy
-   py-cython     py-matplotlib  py-pychecker  py-scientificpython  py-virtualenv
-   py-dateutil   py-mpi4py      py-pygments   py-scikit-learn
-   py-epydoc     py-mx          py-pylint     py-scipy
-   py-gnuplot    py-nose        py-pyparsing  py-setuptools
-   py-h5py       py-numpy       py-pyqt       py-shiboken
-
-   ==> 12 installed:
-   -- linux-debian7-x86_64 / gcc@4.4.7 --------------------------------
-   py-dateutil@2.4.0    py-nose@1.3.4       py-pyside@1.2.2
-   py-dateutil@2.4.0    py-numpy@1.9.1      py-pytz@2014.10
-   py-ipython@2.3.1     py-pygments@2.0.1   py-setuptools@11.3.1
-   py-matplotlib@1.4.2  py-pyparsing@2.0.3  py-six@1.9.0
-
-   ==> 3 currently activated:
-   -- linux-debian7-x86_64 / gcc@4.4.7 --------------------------------
-   py-nose@1.3.4  py-numpy@1.9.1  py-setuptools@11.3.1
-
-Now, when a user runs python, ``numpy`` will be available for import
-*without* the user having to explicitly load it.  ``python@2.7.8`` now
-acts like a system Python installation with ``numpy`` installed inside
-of it.
-
-Spack accomplishes this by symbolically linking the *entire* prefix of
-the ``py-numpy`` package into the prefix of the ``python`` package.  To the
-python interpreter, it looks like ``numpy`` is installed in the
-``site-packages`` directory.
-
-The only limitation of global activation is that you can only have a *single*
-version of an extension activated at a time.  This is because multiple
-versions of the same extension would conflict if symbolically linked
-into the same prefix.  Users who want a different version of a package
-can still get it by using environment modules or views, but they will have to
-explicitly load their preferred version.
-
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-``spack activate --force``
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-If, for some reason, you want to activate a package *without* its
-dependencies, you can use ``spack activate --force``:
-
-.. code-block:: console
-
-   $ spack activate --force py-numpy
-   ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=linux-debian7-x86_64-66733244 for python@2.7.8%gcc@4.4.7.
-
-.. _cmd-spack-deactivate:
-
-^^^^^^^^^^^^^^^^^^^^
-``spack deactivate``
-^^^^^^^^^^^^^^^^^^^^
-
-We've seen how activating an extension can be used to set up a default
-version of a Python module.  Obviously, you may want to change that at
-some point.  ``spack deactivate`` is the command for this.  There are
-several variants:
-
-* ``spack deactivate <extension>`` will deactivate a single
-  extension.  If another activated extension depends on this one,
-  Spack will warn you and exit with an error.
-* ``spack deactivate --force <extension>`` deactivates an extension
-  regardless of packages that depend on it.
-* ``spack deactivate --all <extension>`` deactivates an extension and
-  all of its dependencies.  Use ``--force`` to disregard dependents.
-* ``spack deactivate --all <extendee>`` deactivates *all* activated
-  extensions of a package.  For example, to deactivate *all* python
-  extensions, use:
-
-  .. code-block:: console
-
-     $ spack deactivate --all python
-
 -----------------------
 Filesystem requirements
 -----------------------

lib/spack/docs/build_settings.rst

@@ -302,88 +302,31 @@ microarchitectures considered during the solve are constrained to be compatible
 host Spack is currently running on. For instance, if this option is set to ``true``, a
 user cannot concretize for ``target=icelake`` while running on an Haswell node.
 
-.. _package-preferences:
-
--------------------
-Package Preferences
--------------------
-
-Spack can be configured to prefer certain compilers, package
-versions, dependencies, and variants during concretization.
-The preferred configuration can be controlled via the
-``~/.spack/packages.yaml`` file for user configurations, or the
-``etc/spack/packages.yaml`` site configuration.
-
-Here's an example ``packages.yaml`` file that sets preferred packages:
-
-.. code-block:: yaml
-
-   packages:
-     opencv:
-       compiler: [gcc@4.9]
-       variants: +debug
-     gperftools:
-       version: [2.2, 2.4, 2.3]
-     all:
-       compiler: [gcc@4.4.7, 'gcc@4.6:', intel, clang, pgi]
-       target: [sandybridge]
-       providers:
-         mpi: [mvapich2, mpich, openmpi]
-
-At a high level, this example is specifying how packages should be
-concretized.  The opencv package should prefer using GCC 4.9 and
-be built with debug options.  The gperftools package should prefer version
-2.2 over 2.4.  Every package on the system should prefer mvapich2 for
-its MPI and GCC 4.4.7 (except for opencv, which overrides this by preferring GCC 4.9).
-These options are used to fill in implicit defaults.  Any of them can be overwritten
-on the command line if explicitly requested.
-
-Each ``packages.yaml`` file begins with the string ``packages:`` and
-package names are specified on the next level. The special string ``all``
-applies settings to *all* packages. Underneath each package name is one
-or more components: ``compiler``, ``variants``, ``version``,
-``providers``, and ``target``.  Each component has an ordered list of
-spec ``constraints``, with earlier entries in the list being preferred
-over later entries.
-
-Sometimes a package installation may have constraints that forbid
-the first concretization rule, in which case Spack will use the first
-legal concretization rule.  Going back to the example, if a user
-requests gperftools 2.3 or later, then Spack will install version 2.4
-as the 2.4 version of gperftools is preferred over 2.3.
-
-An explicit concretization rule in the preferred section will always
-take preference over unlisted concretizations.  In the above example,
-xlc isn't listed in the compiler list.  Every listed compiler from
-gcc to pgi will thus be preferred over the xlc compiler.
-
-The syntax for the ``provider`` section differs slightly from other
-concretization rules.  A provider lists a value that packages may
-``depend_on`` (e.g, MPI) and a list of rules for fulfilling that
-dependency.
-
 .. _package-requirements:
 
 --------------------
 Package Requirements
 --------------------
 
-You can use the configuration to force the concretizer to choose
-specific properties for packages when building them. Like preferences,
-these are only applied when the package is required by some other
-request (e.g. if the package is needed as a dependency of a
-request to ``spack install``).
+Spack can be configured to always use certain compilers, package
+versions, and variants during concretization through package
+requirements.
 
-An example of where this is useful is if you have a package that
-is normally built as a dependency but only under certain circumstances
-(e.g. only when a variant on a dependent is active): you can make
-sure that it always builds the way you want it to; this distinguishes
-package configuration requirements from constraints that you add to
-``spack install`` or to environments (in those cases, the associated
-packages are always built).
+Package requirements are useful when you find yourself repeatedly
+specifying the same constraints on the command line, and wish that
+Spack respects these constraints whether you mention them explicitly
+or not. Another use case is specifying constraints that should apply
+to all root specs in an environment, without having to repeat the
+constraint everywhere.
 
-The following is an example of how to enforce package properties in
-``packages.yaml``:
+Apart from that, requirements config is more flexible than constraints
+on the command line, because it can specify constraints on packages
+*when they occur* as a dependency. In contrast, on the command line it
+is not possible to specify constraints on dependencies while also keeping
+those dependencies optional.
+
+The package requirements configuration is specified in ``packages.yaml``
+keyed by package name:
 
 .. code-block:: yaml
 
@@ -452,15 +395,15 @@ under ``all`` are disregarded. For example, with a configuration like this:
      cmake:
        require: '%gcc'
 
-Spack requires ``cmake`` to use ``gcc`` and all other nodes (including cmake dependencies)
-to use ``clang``.
+Spack requires ``cmake`` to use ``gcc`` and all other nodes (including ``cmake``
+dependencies) to use ``clang``.
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Setting requirements on virtual specs
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-A requirement on a virtual spec applies whenever that virtual is present in the DAG. This
-can be useful for fixing which virtual provider you want to use:
+A requirement on a virtual spec applies whenever that virtual is present in the DAG.
+This can be useful for fixing which virtual provider you want to use:
 
 .. code-block:: yaml
 
@@ -470,8 +413,8 @@ can be useful for fixing which virtual provider you want to use:
 
 With the configuration above the only allowed ``mpi`` provider is ``mvapich2 %gcc``.
 
-Requirements on the virtual spec and on the specific provider are both applied, if present. For
-instance with a configuration like:
+Requirements on the virtual spec and on the specific provider are both applied, if
+present. For instance with a configuration like:
 
 .. code-block:: yaml
 
@@ -483,6 +426,66 @@ instance with a configuration like:
 
 you will use ``mvapich2~cuda %gcc`` as an ``mpi`` provider.
 
+.. _package-preferences:
+
+-------------------
+Package Preferences
+-------------------
+
+In some cases package requirements can be too strong, and package
+preferences are the better option. Package preferences do not impose
+constraints on packages for particular versions or variants values,
+they rather only set defaults -- the concretizer is free to change
+them if it must due to other constraints. Also note that package
+preferences are of lower priority than reuse of already installed
+packages.
+
+Here's an example ``packages.yaml`` file that sets preferred packages:
+
+.. code-block:: yaml
+
+   packages:
+     opencv:
+       compiler: [gcc@4.9]
+       variants: +debug
+     gperftools:
+       version: [2.2, 2.4, 2.3]
+     all:
+       compiler: [gcc@4.4.7, 'gcc@4.6:', intel, clang, pgi]
+       target: [sandybridge]
+       providers:
+         mpi: [mvapich2, mpich, openmpi]
+
+At a high level, this example is specifying how packages are preferably
+concretized.  The opencv package should prefer using GCC 4.9 and
+be built with debug options.  The gperftools package should prefer version
+2.2 over 2.4.  Every package on the system should prefer mvapich2 for
+its MPI and GCC 4.4.7 (except for opencv, which overrides this by preferring GCC 4.9).
+These options are used to fill in implicit defaults.  Any of them can be overwritten
+on the command line if explicitly requested.
+
+Package preferences accept the follow keys or components under
+the specific package (or ``all``) section: ``compiler``, ``variants``,
+``version``, ``providers``, and ``target``.  Each component has an
+ordered list of spec ``constraints``, with earlier entries in the
+list being preferred over later entries.
+
+Sometimes a package installation may have constraints that forbid
+the first concretization rule, in which case Spack will use the first
+legal concretization rule.  Going back to the example, if a user
+requests gperftools 2.3 or later, then Spack will install version 2.4
+as the 2.4 version of gperftools is preferred over 2.3.
+
+An explicit concretization rule in the preferred section will always
+take preference over unlisted concretizations.  In the above example,
+xlc isn't listed in the compiler list.  Every listed compiler from
+gcc to pgi will thus be preferred over the xlc compiler.
+
+The syntax for the ``provider`` section differs slightly from other
+concretization rules.  A provider lists a value that packages may
+``depends_on`` (e.g, MPI) and a list of rules for fulfilling that
+dependency.
+
 .. _package_permissions:
 
 -------------------

lib/spack/docs/build_systems/cachedcmakepackage.rst

@@ -5,9 +5,9 @@
 
 .. _cachedcmakepackage:
 
-------------------
-CachedCMakePackage
-------------------
+-----------
+CachedCMake
+-----------
 
 The CachedCMakePackage base class is used for CMake-based workflows
 that create a CMake cache file prior to running ``cmake``. This is

lib/spack/docs/build_systems/cudapackage.rst

@@ -5,9 +5,9 @@
 
 .. _cudapackage:
 
------------
-CudaPackage
------------
+----
+Cuda
+----
 
 Different from other packages, ``CudaPackage`` does not represent a build system.
 Instead its goal is to simplify and unify usage of ``CUDA`` in other packages by providing a `mixin-class <https://en.wikipedia.org/wiki/Mixin>`_.
@@ -80,7 +80,7 @@ standard CUDA compiler flags.
 
 **cuda_flags**
 
-    This built-in static method returns a list of command line flags 
+    This built-in static method returns a list of command line flags
     for the chosen ``cuda_arch`` value(s).  The flags are intended to
     be passed to the CUDA compiler driver (i.e., ``nvcc``).
 

lib/spack/docs/build_systems/inteloneapipackage.rst

@@ -6,9 +6,9 @@
 .. _inteloneapipackage:
 
 
-====================
- IntelOneapiPackage
-====================
+===========
+IntelOneapi
+===========
 
 
 .. contents::
@@ -36,7 +36,7 @@ For more information on a specific package, do::
 
 Intel no longer releases new versions of Parallel Studio, which can be
 used in Spack via the :ref:`intelpackage`. All of its components can
-now be found in oneAPI. 
+now be found in oneAPI.
 
 Examples
 ========

lib/spack/docs/build_systems/intelpackage.rst

@@ -5,9 +5,9 @@
 
 .. _intelpackage:
 
-------------
-IntelPackage
-------------
+-----
+Intel
+-----
 
 .. contents::
 

lib/spack/docs/build_systems/pythonpackage.rst

@@ -5,9 +5,9 @@
 
 .. _pythonpackage:
 
--------------
-PythonPackage
--------------
+------
+Python
+------
 
 Python packages and modules have their own special build system. This
 documentation covers everything you'll need to know in order to write
@@ -724,10 +724,9 @@ extends vs. depends_on
 
 This is very similar to the naming dilemma above, with a slight twist.
 As mentioned in the :ref:`Packaging Guide <packaging_extensions>`,
-``extends`` and ``depends_on`` are very similar, but ``extends`` adds
-the ability to *activate* the package. Activation involves symlinking
-everything in the installation prefix of the package to the installation
-prefix of Python. This allows the user to import a Python module without
+``extends`` and ``depends_on`` are very similar, but ``extends`` ensures
+that the extension and extendee share the same prefix in views.
+This allows the user to import a Python module without
 having to add that module to ``PYTHONPATH``.
 
 When deciding between ``extends`` and ``depends_on``, the best rule of
@@ -735,7 +734,7 @@ thumb is to check the installation prefix. If Python libraries are
 installed to ``<prefix>/lib/pythonX.Y/site-packages``, then you
 should use ``extends``. If Python libraries are installed elsewhere
 or the only files that get installed reside in ``<prefix>/bin``, then
-don't use ``extends``, as symlinking the package wouldn't be useful.
+don't use ``extends``.
 
 ^^^^^^^^^^^^^^^^^^^^^
 Alternatives to Spack

lib/spack/docs/build_systems/rocmpackage.rst

@@ -5,9 +5,9 @@
 
 .. _rocmpackage:
 
------------
-ROCmPackage
------------
+----
+ROCm
+----
 
 The ``ROCmPackage`` is not a build system but a helper package. Like ``CudaPackage``,
 it provides standard variants, dependencies, and conflicts to facilitate building
@@ -25,7 +25,7 @@ This package provides the following variants:
 
 * **rocm**
 
-  This variant is used to enable/disable building with ``rocm``.  
+  This variant is used to enable/disable building with ``rocm``.
   The default is disabled (or ``False``).
 
 * **amdgpu_target**

lib/spack/docs/build_systems/rpackage.rst

@@ -5,9 +5,9 @@
 
 .. _rpackage:
 
---------
-RPackage
---------
+--
+R
+--
 
 Like Python, R has its own built-in build system.
 
@@ -193,10 +193,10 @@ Build system dependencies
 
 As an extension of the R ecosystem, your package will obviously depend
 on R to build and run. Normally, we would use ``depends_on`` to express
-this, but for R packages, we use ``extends``. ``extends`` is similar to
-``depends_on``, but adds an additional feature: the ability to "activate"
-the package by symlinking it to the R installation directory. Since
-every R package needs this, the ``RPackage`` base class contains:
+this, but for R packages, we use ``extends``. This implies a special
+dependency on R, which is used to set environment variables such as
+``R_LIBS`` uniformly. Since every R package needs this, the ``RPackage``
+base class contains:
 
 .. code-block:: python
 

lib/spack/docs/build_systems/sourceforgepackage.rst

@@ -5,15 +5,15 @@
 
 .. _sourceforgepackage:
 
-------------------
-SourceforgePackage
-------------------
+-----------
+Sourceforge
+-----------
 
-``SourceforgePackage`` is a 
+``SourceforgePackage`` is a
 `mixin-class <https://en.wikipedia.org/wiki/Mixin>`_. It automatically
 sets the URL based on a list of Sourceforge mirrors listed in
 `sourceforge_mirror_path`, which defaults to a half dozen known mirrors.
-Refer to the package source 
+Refer to the package source
 (`<https://github.com/spack/spack/blob/develop/lib/spack/spack/build_systems/sourceforge.py>`__) for the current list of mirrors used by Spack.
 
 
@@ -29,7 +29,7 @@ This package provides a method for populating mirror URLs.
     It is decorated with `property` so its results are treated as
     a package attribute.
 
-    Refer to 
+    Refer to
     `<https://spack.readthedocs.io/en/latest/packaging_guide.html#mirrors-of-the-main-url>`__
     for information on how Spack uses the `urls` attribute during
     fetching.

lib/spack/docs/conf.py

@@ -37,12 +37,6 @@
     os.symlink(os.path.abspath("../../.."), link_name, target_is_directory=True)
 sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external"))
 sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external/pytest-fallback"))
-
-if sys.version_info[0] < 3:
-    sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external/yaml/lib"))
-else:
-    sys.path.insert(0, os.path.abspath("_spack_root/lib/spack/external/yaml/lib3"))
-
 sys.path.append(os.path.abspath("_spack_root/lib/spack/"))
 
 # Add the Spack bin directory to the path so that we can use its output in docs.
@@ -160,8 +154,8 @@ def setup(sphinx):
 master_doc = "index"
 
 # General information about the project.
-project = u"Spack"
-copyright = u"2013-2021, Lawrence Livermore National Laboratory."
+project = "Spack"
+copyright = "2013-2021, Lawrence Livermore National Laboratory."
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -350,7 +344,7 @@ class SpackStyle(DefaultStyle):
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-    ("index", "Spack.tex", u"Spack Documentation", u"Todd Gamblin", "manual"),
+    ("index", "Spack.tex", "Spack Documentation", "Todd Gamblin", "manual"),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -378,7 +372,7 @@ class SpackStyle(DefaultStyle):
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [("index", "spack", u"Spack Documentation", [u"Todd Gamblin"], 1)]
+man_pages = [("index", "spack", "Spack Documentation", ["Todd Gamblin"], 1)]
 
 # If true, show URL addresses after external links.
 # man_show_urls = False
@@ -393,8 +387,8 @@ class SpackStyle(DefaultStyle):
     (
         "index",
         "Spack",
-        u"Spack Documentation",
-        u"Todd Gamblin",
+        "Spack Documentation",
+        "Todd Gamblin",
         "Spack",
         "One line description of project.",
         "Miscellaneous",

lib/spack/docs/configuration.rst

@@ -405,6 +405,19 @@ Spack understands several special variables. These are:
 * ``$user``: name of the current user
 * ``$user_cache_path``: user cache directory (``~/.spack`` unless
   :ref:`overridden <local-config-overrides>`)
+* ``$architecture``: the architecture triple of the current host, as
+  detected by Spack.
+* ``$arch``: alias for ``$architecture``.
+* ``$platform``: the platform of the current host, as detected by Spack.
+* ``$operating_system``: the operating system of the current host, as
+  detected by the ``distro`` python module.
+* ``$os``: alias for ``$operating_system``.
+* ``$target``: the ISA target for the current host, as detected by
+  ArchSpec. E.g. ``skylake`` or ``neoverse-n1``.
+* ``$target_family``. The target family for the current host, as
+  detected by ArchSpec. E.g. ``x86_64`` or ``aarch64``.
+* ``$date``: the current date in the format YYYY-MM-DD
+
 
 Note that, as with shell variables, you can write these as ``$varname``
 or with braces to distinguish the variable from surrounding characters:

lib/spack/docs/contribution_guide.rst

@@ -253,27 +253,6 @@ to update them.
    multiple runs of ``spack style`` just to re-compute line numbers and
    makes it much easier to fix errors directly off of the CI output.
 
-.. warning::
-
-   Flake8 and ``pep8-naming`` require a number of dependencies in order
-   to run.  If you installed ``py-flake8`` and ``py-pep8-naming``, the
-   easiest way to ensure the right packages are on your ``PYTHONPATH`` is
-   to run::
-
-     spack activate py-flake8
-     spack activate pep8-naming
-
-   so that all of the dependencies are symlinked to a central
-   location. If you see an error message like:
-
-   .. code-block:: console
-
-      Traceback (most recent call last):
-        File: "/usr/bin/flake8", line 5, in <module>
-          from pkg_resources import load_entry_point
-      ImportError: No module named pkg_resources
-
-   that means Flake8 couldn't find setuptools in your ``PYTHONPATH``.
 
 ^^^^^^^^^^^^^^^^^^^
 Documentation Tests
@@ -309,13 +288,9 @@ All of these can be installed with Spack, e.g.
 
    .. code-block:: console
 
-      $ spack activate py-sphinx
-      $ spack activate py-sphinx-rtd-theme
-      $ spack activate py-sphinxcontrib-programoutput
+      $ spack load py-sphinx py-sphinx-rtd-theme py-sphinxcontrib-programoutput
 
-   so that all of the dependencies are symlinked into that Python's
-   tree.  Alternatively, you could arrange for their library
-   directories to be added to PYTHONPATH.  If you see an error message
+   so that all of the dependencies are added to PYTHONPATH.  If you see an error message
    like:
 
    .. code-block:: console

lib/spack/docs/environments.rst

@@ -233,8 +233,8 @@ packages will be listed as roots of the Environment.
 
 All of the Spack commands that act on the list of installed specs are
 Environment-sensitive in this way, including ``install``,
-``uninstall``, ``activate``, ``deactivate``, ``find``, ``extensions``,
-and more. In the :ref:`environment-configuration` section we will discuss
+``uninstall``, ``find``, ``extensions``, and more. In the
+:ref:`environment-configuration` section we will discuss
 Environment-sensitive commands further.
 
 ^^^^^^^^^^^^^^^^^^^^^
@@ -519,27 +519,33 @@ available from the yaml file.
 ^^^^^^^^^^^^^^^^^^^
 Spec concretization
 ^^^^^^^^^^^^^^^^^^^
-An environment can be concretized in three different modes and the behavior active under any environment
-is determined by the ``concretizer:unify`` property. By default specs are concretized *separately*, one after the other:
+An environment can be concretized in three different modes and the behavior active under
+any environment is determined by the ``concretizer:unify`` configuration option.
+
+The *default* mode is to unify all specs:
 
 .. code-block:: yaml
 
    spack:
        specs:
-         - hdf5~mpi
          - hdf5+mpi
          - zlib@1.2.8
        concretizer:
-         unify: false
+         unify: true
 
-This mode of operation permits to deploy a full software stack where multiple configurations of the same package
-need to be installed alongside each other using the best possible selection of transitive dependencies. The downside
-is that redundancy of installations is disregarded completely, and thus environments might be more bloated than
-strictly needed. In the example above, for instance, if a version of ``zlib`` newer than ``1.2.8`` is known to Spack,
-then it will be used for both ``hdf5`` installations.
+This means that any package in the environment corresponds to a single concrete spec. In
+the above example, when ``hdf5`` depends down the line of ``zlib``, it is required to
+take ``zlib@1.2.8`` instead of a newer version. This mode of concretization is
+particularly useful when environment views are used: if every package occurs in
+only one flavor, it is usually possible to merge all install directories into a view.
 
-If redundancy of the environment is a concern, Spack provides a way to install it *together where possible*,
-i.e. trying to maximize reuse of dependencies across different specs:
+A downside of unified concretization is that it can be overly strict. For example, a
+concretization error would happen when both ``hdf5+mpi`` and ``hdf5~mpi`` are specified
+in an environment.
+
+The second mode is to *unify when possible*: this makes concretization of root specs
+more independendent. Instead of requiring reuse of dependencies across different root
+specs, it is only maximized:
 
 .. code-block:: yaml
 
@@ -551,26 +557,27 @@ i.e. trying to maximize reuse of dependencies across different specs:
        concretizer:
          unify: when_possible
 
-Also in this case Spack allows having multiple configurations of the same package, but privileges the reuse of
-specs over other factors. Going back to our example, this means that both ``hdf5`` installations will use
-``zlib@1.2.8`` as a dependency even if newer versions of that library are available.
-Central installations done at HPC centers by system administrators or user support groups are a common case
-that fits either of these two modes.
+This means that both ``hdf5`` installations will use ``zlib@1.2.8`` as a dependency even
+if newer versions of that library are available.
 
-Environments can also be configured to concretize all the root specs *together*, in a self-consistent way, to
-ensure that each package in the environment comes with a single configuration:
+The third mode of operation is to concretize root specs entirely independently by
+disabling unified concretization:
 
 .. code-block:: yaml
 
    spack:
        specs:
+         - hdf5~mpi
          - hdf5+mpi
          - zlib@1.2.8
        concretizer:
-         unify: true
+         unify: false
 
-This mode of operation is usually what is required by software developers that want to deploy their development
-environment and have a single view of it in the filesystem.
+In this example ``hdf5`` is concretized separately, and does not consider ``zlib@1.2.8``
+as a constraint or preference. Instead, it will take the latest possible version.
+
+The last two concretization options are typically useful for system administrators and
+user support groups providing a large software stack for their HPC center.
 
 .. note::
 
@@ -581,10 +588,10 @@ environment and have a single view of it in the filesystem.
 
 .. admonition:: Re-concretization of user specs
 
-   When concretizing specs *together* or *together where possible* the entire set of specs will be
+   When using *unified* concretization (when possible), the entire set of specs will be
    re-concretized after any addition of new user specs, to ensure that
-   the environment remains consistent / minimal. When instead the specs are concretized
-   separately only the new specs will be re-concretized after any addition.
+   the environment remains consistent / minimal. When instead unified concretization is
+   disabled, only the new specs will be concretized after any addition.
 
 ^^^^^^^^^^^^^
 Spec Matrices
@@ -1063,19 +1070,23 @@ the include is conditional.
 Building a subset of the environment
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The generated ``Makefile``\s contain install targets for each spec. Given the hash
-of a particular spec, you can use the ``.install/<hash>`` target to install the
-spec with its dependencies. There is also ``.install-deps/<hash>`` to *only* install
+The generated ``Makefile``\s contain install targets for each spec, identified
+by ``<name>-<version>-<hash>``. This allows you to install only a subset of the
+packages in the environment. When packages are unique in the environment, it's
+enough to know the name and let tab-completion fill out the version and hash.
+
+The following phony targets are available: ``install/<spec>`` to install the
+spec with its dependencies, and ``install-deps/<spec>`` to *only* install
 its dependencies. This can be useful when certain flags should only apply to
 dependencies. Below we show a use case where a spec is installed with verbose
 output (``spack install --verbose``) while its dependencies are installed silently:
 
 .. code:: console
 
-   $ spack env depfile -o Makefile --make-target-prefix my_env
+   $ spack env depfile -o Makefile
 
    # Install dependencies in parallel, only show a log on error.
-   $ make -j16 my_env/.install-deps/<hash> SPACK_INSTALL_FLAGS=--show-log-on-error
+   $ make -j16 install-deps/python-3.11.0-<hash> SPACK_INSTALL_FLAGS=--show-log-on-error
 
    # Install the root spec with verbose output.
-   $ make -j16 my_env/.install/<hash> SPACK_INSTALL_FLAGS=--verbose
+   $ make -j16 install/python-3.11.0-<hash> SPACK_INSTALL_FLAGS=--verbose

lib/spack/docs/getting_started.rst

@@ -21,8 +21,9 @@ be present on the machine where Spack is run:
    :header-rows: 1
 
 These requirements can be easily installed on most modern Linux systems;
-on macOS, XCode is required.  Spack is designed to run on HPC
-platforms like Cray.  Not all packages should be expected
+on macOS, the Command Line Tools package is required, and a full XCode suite 
+may be necessary for some packages such as Qt and apple-gl. Spack is designed
+to run on HPC platforms like Cray.  Not all packages should be expected
 to work on all platforms.
 
 A build matrix showing which packages are working on which systems is shown below.
@@ -44,7 +45,7 @@ A build matrix showing which packages are working on which systems is shown belo
          yum install -y epel-release
          yum update -y
          yum --enablerepo epel groupinstall -y "Development Tools"
-         yum --enablerepo epel install -y curl findutils gcc-c++ gcc gcc-gfortran git gnupg2 hostname iproute make patch python3 python3-pip python3-setuptools unzip
+         yum --enablerepo epel install -y curl findutils gcc-c++ gcc gcc-gfortran git gnupg2 hostname iproute redhat-lsb-core make patch python3 python3-pip python3-setuptools unzip
          python3 -m pip install boto3
 
    .. tab-item:: macOS Brew
@@ -1704,9 +1705,11 @@ dependencies or incompatible build tools like autoconf. Here are several
 packages known to work on Windows:
 
 * abseil-cpp
+* bzip2
 * clingo
 * cpuinfo
 * cmake
+* hdf5
 * glm
 * nasm
 * netlib-lapack (requires Intel Fortran)

lib/spack/docs/index.rst

@@ -56,7 +56,6 @@ or refer to the full manual below.
    basic_usage
    Tutorial: Spack 101 <https://spack-tutorial.readthedocs.io>
    replace_conda_homebrew
-   known_issues
 
 .. toctree::
    :maxdepth: 2

lib/spack/docs/known_issues.rst

@@ -1,40 +0,0 @@
-.. Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
-   Spack Project Developers. See the top-level COPYRIGHT file for details.
-
-   SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-============
-Known Issues
-============
-
-This is a list of known issues in Spack. It provides ways of getting around these
-problems if you encounter them.
-
-------------------------------------------------
-Spack does not seem to respect ``packages.yaml``
-------------------------------------------------
-
-.. note::
-
-   This issue is **resolved** as of v0.19.0.dev0 commit
-   `8281a0c5feabfc4fe180846d6fe95cfe53420bc5`, through the introduction of package
-   requirements. See :ref:`package-requirements`.
-
-A common problem in Spack v0.18.0 up to v0.19.0.dev0 is that package, compiler and target
-preferences specified in ``packages.yaml`` do not seem to be respected. Spack picks the
-"wrong" compilers and their versions, package versions and variants, and
-micro-architectures.
-
-This is however not a bug. In order to reduce the number of builds of the same
-packages, the concretizer values reuse of installed packages higher than preferences
-set in ``packages.yaml``. Note that ``packages.yaml`` specifies only preferences, not
-hard constraints.
-
-There are multiple workarounds:
-
-1. Disable reuse during concretization: ``spack install --fresh <spec>`` when installing
-   from the command line, or ``spack concretize --fresh --force`` when using
-   environments.  
-2. Turn preferences into constrains, by moving them to the input spec. For example,
-   use ``spack spec zlib%gcc@12`` when you want to force GCC 12 even if ``zlib`` was
-   already installed with GCC 10.

lib/spack/docs/packaging_guide.rst

@@ -2634,9 +2634,12 @@ extendable package:
        extends('python')
        ...
 
-Now, the ``py-numpy`` package can be used as an argument to ``spack
-activate``.  When it is activated, all the files in its prefix will be
-symbolically linked into the prefix of the python package.
+This accomplishes a few things. Firstly, the Python package can set special
+variables such as ``PYTHONPATH`` for all extensions when the run or build
+environment is set up. Secondly, filesystem views can ensure that extensions
+are put in the same prefix as their extendee. This ensures that Python in
+a view can always locate its Python packages, even without environment
+variables set.
 
 A package can only extend one other package at a time.  To support packages
 that may extend one of a list of other packages, Spack supports multiple
@@ -2684,9 +2687,8 @@ variant(s) are selected.  This may be accomplished with conditional
        ...
 
 Sometimes, certain files in one package will conflict with those in
-another, which means they cannot both be activated (symlinked) at the
-same time.  In this case, you can tell Spack to ignore those files
-when it does the activation:
+another, which means they cannot both be used in a view at the
+same time.  In this case, you can tell Spack to ignore those files:
 
 .. code-block:: python
 
@@ -2698,7 +2700,7 @@ when it does the activation:
        ...
 
 The code above will prevent everything in the ``$prefix/bin/`` directory
-from being linked in at activation time.
+from being linked in a view.
 
 .. note::
 
@@ -2722,67 +2724,6 @@ extensions; as a consequence python extension packages (those inheriting from
 ``PythonPackage``) likewise override ``add_files_to_view`` in order to rewrite
 shebang lines which point to the Python interpreter.
 
-^^^^^^^^^^^^^^^^^^^^^^^^^
-Activation & deactivation
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Adding an extension to a view is referred to as an activation. If the view is
-maintained in the Spack installation prefix of the extendee this is called a
-global activation. Activations may involve updating some centralized state
-that is maintained by the extendee package, so there can be additional work
-for adding extensions compared with non-extension packages.
-
-Spack's ``Package`` class has default ``activate`` and ``deactivate``
-implementations that handle symbolically linking extensions' prefixes
-into a specified view. Extendable packages can override these methods
-to add custom activate/deactivate logic of their own.  For example,
-the ``activate`` and ``deactivate`` methods in the Python class handle
-symbolic linking of extensions, but they also handle details surrounding
-Python's ``.pth`` files, and other aspects of Python packaging.
-
-Spack's extensions mechanism is designed to be extensible, so that
-other packages (like Ruby, R, Perl, etc.)  can provide their own
-custom extension management logic, as they may not handle modules the
-same way that Python does.
-
-Let's look at Python's activate function:
-
-.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/python/package.py
-   :pyobject: Python.activate
-   :linenos:
-
-This function is called on the *extendee* (Python).  It first calls
-``activate`` in the superclass, which handles symlinking the
-extension package's prefix into the specified view.  It then does
-some special handling of the ``easy-install.pth`` file, part of
-Python's setuptools.
-
-Deactivate behaves similarly to activate, but it unlinks files:
-
-.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/python/package.py
-   :pyobject: Python.deactivate
-   :linenos:
-
-Both of these methods call some custom functions in the Python
-package.  See the source for Spack's Python package for details.
-
-^^^^^^^^^^^^^^^^^^^^
-Activation arguments
-^^^^^^^^^^^^^^^^^^^^
-
-You may have noticed that the ``activate`` function defined above
-takes keyword arguments.  These are the keyword arguments from
-``extends()``, and they are passed to both activate and deactivate.
-
-This capability allows an extension to customize its own activation by
-passing arguments to the extendee.  Extendees can likewise implement
-custom ``activate()`` and ``deactivate()`` functions to suit their
-needs.
-
-The only keyword argument supported by default is the ``ignore``
-argument, which can take a regex, list of regexes, or a predicate to
-determine which files *not* to symlink during activation.
-
 .. _virtual-dependencies:
 
 --------------------
@@ -3584,7 +3525,7 @@ will likely contain some overriding of default builder methods:
        def cmake_args(self):
            pass
 
-   class Autotoolsbuilder(spack.build_systems.autotools.AutotoolsBuilder):
+   class AutotoolsBuilder(spack.build_systems.autotools.AutotoolsBuilder):
        def configure_args(self):
            pass
 

lib/spack/docs/tables/system_prerequisites.csv

@@ -1,5 +1,5 @@
 Name, Supported Versions, Notes, Requirement Reason
-Python, 2.7/3.6-3.10, , Interpreter for Spack
+Python, 3.6--3.11, , Interpreter for Spack
 C/C++ Compilers, , , Building software
 make, , , Build software
 patch, , , Build software
@@ -11,6 +11,7 @@ bzip2, , , Compress/Decompress archives
 xz, , , Compress/Decompress archives
 zstd, , Optional, Compress/Decompress archives
 file, , , Create/Use Buildcaches
+lsb-release, , , Linux: identify operating system version
 gnupg2, , , Sign/Verify Buildcaches
 git, , , Manage Software Repositories
 svn, , Optional, Manage Software Repositories

lib/spack/env/cc

@@ -440,6 +440,47 @@ while [ $# -ne 0 ]; do
         continue
     fi
 
+    if [ -n "${SPACK_COMPILER_FLAGS_KEEP}" ] ; then
+        # NOTE: the eval is required to allow `|` alternatives inside the variable
+        eval "\
+        case \"\$1\" in
+            $SPACK_COMPILER_FLAGS_KEEP)
+                append other_args_list \"\$1\"
+                shift
+                continue
+                ;;
+        esac
+        "
+    fi
+    # the replace list is a space-separated list of pipe-separated pairs,
+    # the first in each pair is the original prefix to be matched, the
+    # second is the replacement prefix
+    if [ -n "${SPACK_COMPILER_FLAGS_REPLACE}" ] ; then
+        for rep in ${SPACK_COMPILER_FLAGS_REPLACE} ; do
+            before=${rep%|*}
+            after=${rep#*|}
+            eval "\
+            stripped=\"\${1##$before}\"
+            "
+            if [ "$stripped" = "$1" ] ; then
+                continue
+            fi
+
+            replaced="$after$stripped" 
+
+            # it matched, remove it
+            shift
+
+            if [ -z "$replaced" ] ; then
+                # completely removed, continue OUTER loop
+                continue 2
+            fi
+
+            # re-build argument list with replacement
+            set -- "$replaced" "$@"
+        done
+    fi
+
     case "$1" in
         -isystem*)
             arg="${1#-isystem}"

lib/spack/external/__init__.py

@@ -18,7 +18,7 @@
 
 * Homepage: https://pypi.python.org/pypi/archspec
 * Usage: Labeling, comparison and detection of microarchitectures
-* Version: 0.1.4 (commit e2cfdc266174488dee78b8c9058e36d60dc1b548)
+* Version: 0.2.0 (commit 77640e572725ad97f18e63a04857155752ace045)
 
 argparse
 --------

lib/spack/external/archspec/cpu/detect.py

@@ -132,9 +132,15 @@ def sysctl(*args):
             "model name": sysctl("-n", "machdep.cpu.brand_string"),
         }
     else:
-        model = (
-            "m1" if "Apple" in sysctl("-n", "machdep.cpu.brand_string") else "unknown"
-        )
+        model = "unknown"
+        model_str = sysctl("-n", "machdep.cpu.brand_string").lower()
+        if "m2" in model_str:
+            model = "m2"
+        elif "m1" in model_str:
+            model = "m1"
+        elif "apple" in model_str:
+            model = "m1"
+
         info = {
             "vendor_id": "Apple",
             "flags": [],
@@ -322,14 +328,26 @@ def compatibility_check_for_aarch64(info, target):
     features = set(info.get("Features", "").split())
     vendor = info.get("CPU implementer", "generic")
 
+    # At the moment it's not clear how to detect compatibility with
+    # a specific version of the architecture
+    if target.vendor == "generic" and target.name != "aarch64":
+        return False
+
     arch_root = TARGETS[basename]
-    return (
-        (target == arch_root or arch_root in target.ancestors)
-        and target.vendor in (vendor, "generic")
-        # On macOS it seems impossible to get all the CPU features with syctl info
-        and (target.features.issubset(features) or platform.system() == "Darwin")
+    arch_root_and_vendor = arch_root == target.family and target.vendor in (
+        vendor,
+        "generic",
     )
 
+    # On macOS it seems impossible to get all the CPU features
+    # with syctl info, but for ARM we can get the exact model
+    if platform.system() == "Darwin":
+        model_key = info.get("model", basename)
+        model = TARGETS[model_key]
+        return arch_root_and_vendor and (target == model or target in model.ancestors)
+
+    return arch_root_and_vendor and target.features.issubset(features)
+
 
 @compatibility_check(architecture_family="riscv64")
 def compatibility_check_for_riscv64(info, target):

lib/spack/external/archspec/json/cpu/microarchitectures.json

@@ -85,7 +85,7 @@
         "intel": [
           {
             "versions": ":",
-            "name": "x86-64",
+            "name": "pentium4",
             "flags": "-march={name} -mtune=generic"
           }
         ],
@@ -2093,8 +2093,163 @@
         ]
       }
     },
-    "thunderx2": {
+    "armv8.1a": {
       "from": ["aarch64"],
+      "vendor": "generic",
+      "features": [],
+      "compilers": {
+        "gcc": [
+          {
+            "versions": "5:",
+            "flags": "-march=armv8.1-a -mtune=generic"
+          }
+        ],
+        "clang": [
+          {
+            "versions": ":",
+            "flags": "-march=armv8.1-a -mtune=generic"
+          }
+        ],
+        "apple-clang": [
+          {
+            "versions": ":",
+            "flags": "-march=armv8.1-a -mtune=generic"
+          }
+        ],
+        "arm": [
+          {
+            "versions": ":",
+            "flags": "-march=armv8.1-a -mtune=generic"
+          }
+        ]
+      }
+    },
+    "armv8.2a": {
+      "from": ["armv8.1a"],
+      "vendor": "generic",
+      "features": [],
+      "compilers": {
+        "gcc": [
+          {
+            "versions": "6:",
+            "flags": "-march=armv8.2-a -mtune=generic"
+          }
+        ],
+        "clang": [
+          {
+            "versions": ":",
+            "flags": "-march=armv8.2-a -mtune=generic"
+          }
+        ],
+        "apple-clang": [
+          {
+            "versions": ":",
+            "flags": "-march=armv8.2-a -mtune=generic"
+          }
+        ],
+        "arm": [
+          {
+            "versions": ":",
+            "flags": "-march=armv8.2-a -mtune=generic"
+          }
+        ]
+      }
+    },
+    "armv8.3a": {
+      "from": ["armv8.2a"],
+      "vendor": "generic",
+      "features": [],
+      "compilers": {
+        "gcc": [
+          {
+            "versions": "6:",
+            "flags": "-march=armv8.3-a -mtune=generic"
+          }
+        ],
+        "clang": [
+          {
+            "versions": "6:",
+            "flags": "-march=armv8.3-a -mtune=generic"
+          }
+        ],
+        "apple-clang": [
+          {
+            "versions": ":",
+            "flags": "-march=armv8.3-a -mtune=generic"
+          }
+        ],
+        "arm": [
+          {
+            "versions": ":",
+            "flags": "-march=armv8.3-a -mtune=generic"
+          }
+        ]
+      }
+    },
+    "armv8.4a": {
+      "from": ["armv8.3a"],
+      "vendor": "generic",
+      "features": [],
+      "compilers": {
+        "gcc": [
+          {
+            "versions": "8:",
+            "flags": "-march=armv8.4-a -mtune=generic"
+          }
+        ],
+        "clang": [
+          {
+            "versions": "8:",
+            "flags": "-march=armv8.4-a -mtune=generic"
+          }
+        ],
+        "apple-clang": [
+          {
+            "versions": ":",
+            "flags": "-march=armv8.4-a -mtune=generic"
+          }
+        ],
+        "arm": [
+          {
+            "versions": ":",
+            "flags": "-march=armv8.4-a -mtune=generic"
+          }
+        ]
+      }
+    },
+    "armv8.5a": {
+      "from": ["armv8.4a"],
+      "vendor": "generic",
+      "features": [],
+      "compilers": {
+        "gcc": [
+          {
+            "versions": "9:",
+            "flags": "-march=armv8.5-a -mtune=generic"
+          }
+        ],
+        "clang": [
+          {
+            "versions": "11:",
+            "flags": "-march=armv8.5-a -mtune=generic"
+          }
+        ],
+        "apple-clang": [
+          {
+            "versions": ":",
+            "flags": "-march=armv8.5-a -mtune=generic"
+          }
+        ],
+        "arm": [
+          {
+            "versions": ":",
+            "flags": "-march=armv8.5-a -mtune=generic"
+          }
+        ]
+      }
+    },
+    "thunderx2": {
+      "from": ["armv8.1a"],
       "vendor": "Cavium",
       "features": [
         "fp",
@@ -2141,7 +2296,7 @@
       }
     },
     "a64fx": {
-      "from": ["aarch64"],
+      "from": ["armv8.2a"],
       "vendor": "Fujitsu",
       "features": [
         "fp",
@@ -2209,7 +2364,7 @@
         ]
       }
     },
-    "graviton": {
+    "cortex_a72": {
       "from": ["aarch64"],
       "vendor": "ARM",
       "features": [
@@ -2235,19 +2390,19 @@
               },
               {
                   "versions": "6:",
-                  "flags" : "-march=armv8-a+crc+crypto -mtune=cortex-a72"
+                  "flags" : "-mcpu=cortex-a72"
               }
           ],
           "clang" : [
               {
                   "versions": "3.9:",
-                  "flags" : "-march=armv8-a+crc+crypto"
+                  "flags" : "-mcpu=cortex-a72"
               }
           ]
       }
     },
-    "graviton2": {
-      "from": ["graviton"],
+    "neoverse_n1": {
+      "from": ["cortex_a72", "armv8.2a"],
       "vendor": "ARM",
       "features": [
           "fp",
@@ -2296,7 +2451,7 @@
               },
               {
                   "versions": "9.0:",
-                  "flags" : "-march=armv8.2-a+fp16+rcpc+dotprod+crypto -mtune=neoverse-n1"
+                  "flags" : "-mcpu=neoverse-n1"
               }
           ],
           "clang" : [
@@ -2307,6 +2462,10 @@
               {
                   "versions": "5:",
                   "flags" : "-march=armv8.2-a+fp16+rcpc+dotprod+crypto"
+              },
+              {
+                  "versions": "10:",
+                  "flags" : "-mcpu=neoverse-n1"
               }
           ],
           "arm" : [
@@ -2317,11 +2476,11 @@
           ]
       }
     },
-    "graviton3": {
-      "from": ["graviton2"],
+    "neoverse_v1": {
+      "from": ["neoverse_n1", "armv8.4a"],
       "vendor": "ARM",
       "features": [
-	  "fp",
+          "fp",
           "asimd",
           "evtstrm",
           "aes",
@@ -2384,11 +2543,11 @@
               },
               {
                   "versions": "9.0:9.9",
-                  "flags" : "-march=armv8.4-a+crypto+rcpc+sha3+sm4+sve+rng+nodotprod -mtune=neoverse-v1"
+                  "flags" : "-mcpu=neoverse-v1"
               },
 	            {
                   "versions": "10.0:",
-                  "flags" : "-march=armv8.4-a+crypto+rcpc+sha3+sm4+sve+rng+ssbs+i8mm+bf16+nodotprod -mtune=neoverse-v1"
+                  "flags" : "-mcpu=neoverse-v1"
               }
 
           ],
@@ -2404,6 +2563,10 @@
               {
                   "versions": "11:",
                   "flags" : "-march=armv8.4-a+sve+ssbs+fp16+bf16+crypto+i8mm+rng"
+              },
+              {
+                  "versions": "12:",
+                  "flags" : "-mcpu=neoverse-v1"
               }
           ],
           "arm" : [
@@ -2419,7 +2582,7 @@
       }
     },
     "m1": {
-      "from": ["aarch64"],
+      "from": ["armv8.4a"],
       "vendor": "Apple",
       "features": [
           "fp",
@@ -2484,6 +2647,76 @@
         ]
       }
     },
+    "m2": {
+      "from": ["m1", "armv8.5a"],
+      "vendor": "Apple",
+      "features": [
+          "fp",
+          "asimd",
+          "evtstrm",
+          "aes",
+          "pmull",
+          "sha1",
+          "sha2",
+          "crc32",
+          "atomics",
+          "fphp",
+          "asimdhp",
+          "cpuid",
+          "asimdrdm",
+          "jscvt",
+          "fcma",
+          "lrcpc",
+          "dcpop",
+          "sha3",
+          "asimddp",
+          "sha512",
+          "asimdfhm",
+          "dit",
+          "uscat",
+          "ilrcpc",
+          "flagm",
+          "ssbs",
+          "sb",
+          "paca",
+          "pacg",
+          "dcpodp",
+          "flagm2",
+          "frint",
+          "ecv",
+          "bf16",
+          "i8mm",
+          "bti"
+      ],
+      "compilers": {
+        "gcc": [
+          {
+            "versions": "8.0:",
+            "flags" : "-march=armv8.5-a -mtune=generic"
+          }
+        ],
+        "clang" : [
+          {
+            "versions": "9.0:12.0",
+            "flags" : "-march=armv8.5-a"
+          },
+          {
+            "versions": "13.0:",
+            "flags" : "-mcpu=apple-m1"
+          }
+        ],
+        "apple-clang": [
+          {
+            "versions": "11.0:12.5",
+            "flags" : "-march=armv8.5-a"
+          },
+          {
+            "versions": "13.0:",
+            "flags" : "-mcpu=vortex"
+          }
+        ]
+      }
+    },
     "arm": {
       "from": [],
       "vendor": "generic",

lib/spack/external/ctest_log_parser.py

@@ -71,13 +71,12 @@
 import re
 import math
 import multiprocessing
+import io
 import sys
 import threading
 import time
 from contextlib import contextmanager
 
-from six import StringIO
-from six import string_types
 
 _error_matches = [
     "^FAIL: ",
@@ -246,7 +245,7 @@ def __getitem__(self, line_no):
 
     def __str__(self):
         """Returns event lines and context."""
-        out = StringIO()
+        out = io.StringIO()
         for i in range(self.start, self.end):
             if i == self.line_no:
                 out.write('  >> %-6d%s' % (i, self[i]))
@@ -386,7 +385,7 @@ def parse(self, stream, context=6, jobs=None):
             (tuple): two lists containing ``BuildError`` and
                 ``BuildWarning`` objects.
         """
-        if isinstance(stream, string_types):
+        if isinstance(stream, str):
             with open(stream) as f:
                 return self.parse(f, context, jobs)
 

lib/spack/external/py2/functools32/LICENSE

@@ -1,289 +0,0 @@
-A. HISTORY OF THE SOFTWARE
-==========================
-
-Python was created in the early 1990s by Guido van Rossum at Stichting
-Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
-as a successor of a language called ABC.  Guido remains Python's
-principal author, although it includes many contributions from others.
-
-In 1995, Guido continued his work on Python at the Corporation for
-National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
-in Reston, Virginia where he released several versions of the
-software.
-
-In May 2000, Guido and the Python core development team moved to
-BeOpen.com to form the BeOpen PythonLabs team.  In October of the same
-year, the PythonLabs team moved to Digital Creations (now Zope
-Corporation, see http://www.zope.com).  In 2001, the Python Software
-Foundation (PSF, see http://www.python.org/psf/) was formed, a
-non-profit organization created specifically to own Python-related
-Intellectual Property.  Zope Corporation is a sponsoring member of
-the PSF.
-
-All Python releases are Open Source (see http://www.opensource.org for
-the Open Source Definition).  Historically, most, but not all, Python
-releases have also been GPL-compatible; the table below summarizes
-the various releases.
-
-    Release         Derived     Year        Owner       GPL-
-                    from                                compatible? (1)
-
-    0.9.0 thru 1.2              1991-1995   CWI         yes
-    1.3 thru 1.5.2  1.2         1995-1999   CNRI        yes
-    1.6             1.5.2       2000        CNRI        no
-    2.0             1.6         2000        BeOpen.com  no
-    1.6.1           1.6         2001        CNRI        yes (2)
-    2.1             2.0+1.6.1   2001        PSF         no
-    2.0.1           2.0+1.6.1   2001        PSF         yes
-    2.1.1           2.1+2.0.1   2001        PSF         yes
-    2.2             2.1.1       2001        PSF         yes
-    2.1.2           2.1.1       2002        PSF         yes
-    2.1.3           2.1.2       2002        PSF         yes
-    2.2.1           2.2         2002        PSF         yes
-    2.2.2           2.2.1       2002        PSF         yes
-    2.2.3           2.2.2       2003        PSF         yes
-    2.3             2.2.2       2002-2003   PSF         yes
-    2.3.1           2.3         2002-2003   PSF         yes
-    2.3.2           2.3.1       2002-2003   PSF         yes
-    2.3.3           2.3.2       2002-2003   PSF         yes
-    2.3.4           2.3.3       2004        PSF         yes
-    2.3.5           2.3.4       2005        PSF         yes
-    2.4             2.3         2004        PSF         yes
-    2.4.1           2.4         2005        PSF         yes
-    2.4.2           2.4.1       2005        PSF         yes
-    2.4.3           2.4.2       2006        PSF         yes
-    2.4.4           2.4.3       2006        PSF         yes
-    2.5             2.4         2006        PSF         yes
-    2.5.1           2.5         2007        PSF         yes
-    2.5.2           2.5.1       2008        PSF         yes
-    2.5.3           2.5.2       2008        PSF         yes
-    2.6             2.5         2008        PSF         yes
-    2.6.1           2.6         2008        PSF         yes
-    2.6.2           2.6.1       2009        PSF         yes
-    2.6.3           2.6.2       2009        PSF         yes
-    2.6.4           2.6.3       2009        PSF         yes
-    2.6.5           2.6.4       2010        PSF         yes
-    3.0             2.6         2008        PSF         yes
-    3.0.1           3.0         2009        PSF         yes
-    3.1             3.0.1       2009        PSF         yes
-    3.1.1           3.1         2009        PSF         yes
-    3.1.2           3.1.1       2010        PSF         yes
-    3.1.3           3.1.2       2010        PSF         yes
-    3.1.4           3.1.3       2011        PSF         yes
-    3.2             3.1         2011        PSF         yes
-    3.2.1           3.2         2011        PSF         yes
-    3.2.2           3.2.1       2011        PSF         yes
-    3.2.3           3.2.2       2012        PSF         yes
-
-Footnotes:
-
-(1) GPL-compatible doesn't mean that we're distributing Python under
-    the GPL.  All Python licenses, unlike the GPL, let you distribute
-    a modified version without making your changes open source.  The
-    GPL-compatible licenses make it possible to combine Python with
-    other software that is released under the GPL; the others don't.
-
-(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
-    because its license has a choice of law clause.  According to
-    CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
-    is "not incompatible" with the GPL.
-
-Thanks to the many outside volunteers who have worked under Guido's
-direction to make these releases possible.
-
-
-B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
-===============================================================
-
-PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
---------------------------------------------
-
-1. This LICENSE AGREEMENT is between the Python Software Foundation
-("PSF"), and the Individual or Organization ("Licensee") accessing and
-otherwise using this software ("Python") in source or binary form and
-its associated documentation.
-
-2. Subject to the terms and conditions of this License Agreement, PSF hereby
-grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
-analyze, test, perform and/or display publicly, prepare derivative works,
-distribute, and otherwise use Python alone or in any derivative version,
-provided, however, that PSF's License Agreement and PSF's notice of copyright,
-i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-2011, 2012 Python Software Foundation; All Rights Reserved" are retained in Python
-alone or in any derivative version prepared by Licensee.
-
-3. In the event Licensee prepares a derivative work that is based on
-or incorporates Python or any part thereof, and wants to make
-the derivative work available to others as provided herein, then
-Licensee hereby agrees to include in any such work a brief summary of
-the changes made to Python.
-
-4. PSF is making Python available to Licensee on an "AS IS"
-basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
-DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
-INFRINGE ANY THIRD PARTY RIGHTS.
-
-5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
-OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-
-6. This License Agreement will automatically terminate upon a material
-breach of its terms and conditions.
-
-7. Nothing in this License Agreement shall be deemed to create any
-relationship of agency, partnership, or joint venture between PSF and
-Licensee.  This License Agreement does not grant permission to use PSF
-trademarks or trade name in a trademark sense to endorse or promote
-products or services of Licensee, or any third party.
-
-8. By copying, installing or otherwise using Python, Licensee
-agrees to be bound by the terms and conditions of this License
-Agreement.
-
-
-BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
--------------------------------------------
-
-BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
-
-1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
-office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
-Individual or Organization ("Licensee") accessing and otherwise using
-this software in source or binary form and its associated
-documentation ("the Software").
-
-2. Subject to the terms and conditions of this BeOpen Python License
-Agreement, BeOpen hereby grants Licensee a non-exclusive,
-royalty-free, world-wide license to reproduce, analyze, test, perform
-and/or display publicly, prepare derivative works, distribute, and
-otherwise use the Software alone or in any derivative version,
-provided, however, that the BeOpen Python License is retained in the
-Software, alone or in any derivative version prepared by Licensee.
-
-3. BeOpen is making the Software available to Licensee on an "AS IS"
-basis.  BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
-DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
-INFRINGE ANY THIRD PARTY RIGHTS.
-
-4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
-SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
-AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
-DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-
-5. This License Agreement will automatically terminate upon a material
-breach of its terms and conditions.
-
-6. This License Agreement shall be governed by and interpreted in all
-respects by the law of the State of California, excluding conflict of
-law provisions.  Nothing in this License Agreement shall be deemed to
-create any relationship of agency, partnership, or joint venture
-between BeOpen and Licensee.  This License Agreement does not grant
-permission to use BeOpen trademarks or trade names in a trademark
-sense to endorse or promote products or services of Licensee, or any
-third party.  As an exception, the "BeOpen Python" logos available at
-http://www.pythonlabs.com/logos.html may be used according to the
-permissions granted on that web page.
-
-7. By copying, installing or otherwise using the software, Licensee
-agrees to be bound by the terms and conditions of this License
-Agreement.
-
-
-CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
----------------------------------------
-
-1. This LICENSE AGREEMENT is between the Corporation for National
-Research Initiatives, having an office at 1895 Preston White Drive,
-Reston, VA 20191 ("CNRI"), and the Individual or Organization
-("Licensee") accessing and otherwise using Python 1.6.1 software in
-source or binary form and its associated documentation.
-
-2. Subject to the terms and conditions of this License Agreement, CNRI
-hereby grants Licensee a nonexclusive, royalty-free, world-wide
-license to reproduce, analyze, test, perform and/or display publicly,
-prepare derivative works, distribute, and otherwise use Python 1.6.1
-alone or in any derivative version, provided, however, that CNRI's
-License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
-1995-2001 Corporation for National Research Initiatives; All Rights
-Reserved" are retained in Python 1.6.1 alone or in any derivative
-version prepared by Licensee.  Alternately, in lieu of CNRI's License
-Agreement, Licensee may substitute the following text (omitting the
-quotes): "Python 1.6.1 is made available subject to the terms and
-conditions in CNRI's License Agreement.  This Agreement together with
-Python 1.6.1 may be located on the Internet using the following
-unique, persistent identifier (known as a handle): 1895.22/1013.  This
-Agreement may also be obtained from a proxy server on the Internet
-using the following URL: http://hdl.handle.net/1895.22/1013".
-
-3. In the event Licensee prepares a derivative work that is based on
-or incorporates Python 1.6.1 or any part thereof, and wants to make
-the derivative work available to others as provided herein, then
-Licensee hereby agrees to include in any such work a brief summary of
-the changes made to Python 1.6.1.
-
-4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
-basis.  CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
-DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
-INFRINGE ANY THIRD PARTY RIGHTS.
-
-5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
-OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-
-6. This License Agreement will automatically terminate upon a material
-breach of its terms and conditions.
-
-7. This License Agreement shall be governed by the federal
-intellectual property law of the United States, including without
-limitation the federal copyright law, and, to the extent such
-U.S. federal law does not apply, by the law of the Commonwealth of
-Virginia, excluding Virginia's conflict of law provisions.
-Notwithstanding the foregoing, with regard to derivative works based
-on Python 1.6.1 that incorporate non-separable material that was
-previously distributed under the GNU General Public License (GPL), the
-law of the Commonwealth of Virginia shall govern this License
-Agreement only as to issues arising under or with respect to
-Paragraphs 4, 5, and 7 of this License Agreement.  Nothing in this
-License Agreement shall be deemed to create any relationship of
-agency, partnership, or joint venture between CNRI and Licensee.  This
-License Agreement does not grant permission to use CNRI trademarks or
-trade name in a trademark sense to endorse or promote products or
-services of Licensee, or any third party.
-
-8. By clicking on the "ACCEPT" button where indicated, or by copying,
-installing or otherwise using Python 1.6.1, Licensee agrees to be
-bound by the terms and conditions of this License Agreement.
-
-        ACCEPT
-
-
-CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
---------------------------------------------------
-
-Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
-The Netherlands.  All rights reserved.
-
-Permission to use, copy, modify, and distribute this software and its
-documentation for any purpose and without fee is hereby granted,
-provided that the above copyright notice appear in all copies and that
-both that copyright notice and this permission notice appear in
-supporting documentation, and that the name of Stichting Mathematisch
-Centrum or CWI not be used in advertising or publicity pertaining to
-distribution of the software without specific, written prior
-permission.
-
-STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
-THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
-FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
-FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
-ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
-OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

lib/spack/external/py2/functools32/__init__.py

@@ -1 +0,0 @@
-from .functools32 import *

lib/spack/external/py2/functools32/_dummy_thread32.py

@@ -1,158 +0,0 @@
-"""Drop-in replacement for the thread module.
-
-Meant to be used as a brain-dead substitute so that threaded code does
-not need to be rewritten for when the thread module is not present.
-
-Suggested usage is::
-
-    try:
-        try:
-            import _thread  # Python >= 3
-        except:
-            import thread as _thread  # Python < 3
-    except ImportError:
-        import _dummy_thread as _thread
-
-"""
-# Exports only things specified by thread documentation;
-# skipping obsolete synonyms allocate(), start_new(), exit_thread().
-__all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock',
-           'interrupt_main', 'LockType']
-
-# A dummy value
-TIMEOUT_MAX = 2**31
-
-# NOTE: this module can be imported early in the extension building process,
-# and so top level imports of other modules should be avoided.  Instead, all
-# imports are done when needed on a function-by-function basis.  Since threads
-# are disabled, the import lock should not be an issue anyway (??).
-
-class error(Exception):
-    """Dummy implementation of _thread.error."""
-
-    def __init__(self, *args):
-        self.args = args
-
-def start_new_thread(function, args, kwargs={}):
-    """Dummy implementation of _thread.start_new_thread().
-
-    Compatibility is maintained by making sure that ``args`` is a
-    tuple and ``kwargs`` is a dictionary.  If an exception is raised
-    and it is SystemExit (which can be done by _thread.exit()) it is
-    caught and nothing is done; all other exceptions are printed out
-    by using traceback.print_exc().
-
-    If the executed function calls interrupt_main the KeyboardInterrupt will be
-    raised when the function returns.
-
-    """
-    if type(args) != type(tuple()):
-        raise TypeError("2nd arg must be a tuple")
-    if type(kwargs) != type(dict()):
-        raise TypeError("3rd arg must be a dict")
-    global _main
-    _main = False
-    try:
-        function(*args, **kwargs)
-    except SystemExit:
-        pass
-    except:
-        import traceback
-        traceback.print_exc()
-    _main = True
-    global _interrupt
-    if _interrupt:
-        _interrupt = False
-        raise KeyboardInterrupt
-
-def exit():
-    """Dummy implementation of _thread.exit()."""
-    raise SystemExit
-
-def get_ident():
-    """Dummy implementation of _thread.get_ident().
-
-    Since this module should only be used when _threadmodule is not
-    available, it is safe to assume that the current process is the
-    only thread.  Thus a constant can be safely returned.
-    """
-    return -1
-
-def allocate_lock():
-    """Dummy implementation of _thread.allocate_lock()."""
-    return LockType()
-
-def stack_size(size=None):
-    """Dummy implementation of _thread.stack_size()."""
-    if size is not None:
-        raise error("setting thread stack size not supported")
-    return 0
-
-class LockType(object):
-    """Class implementing dummy implementation of _thread.LockType.
-
-    Compatibility is maintained by maintaining self.locked_status
-    which is a boolean that stores the state of the lock.  Pickling of
-    the lock, though, should not be done since if the _thread module is
-    then used with an unpickled ``lock()`` from here problems could
-    occur from this class not having atomic methods.
-
-    """
-
-    def __init__(self):
-        self.locked_status = False
-
-    def acquire(self, waitflag=None, timeout=-1):
-        """Dummy implementation of acquire().
-
-        For blocking calls, self.locked_status is automatically set to
-        True and returned appropriately based on value of
-        ``waitflag``.  If it is non-blocking, then the value is
-        actually checked and not set if it is already acquired.  This
-        is all done so that threading.Condition's assert statements
-        aren't triggered and throw a little fit.
-
-        """
-        if waitflag is None or waitflag:
-            self.locked_status = True
-            return True
-        else:
-            if not self.locked_status:
-                self.locked_status = True
-                return True
-            else:
-                if timeout > 0:
-                    import time
-                    time.sleep(timeout)
-                return False
-
-    __enter__ = acquire
-
-    def __exit__(self, typ, val, tb):
-        self.release()
-
-    def release(self):
-        """Release the dummy lock."""
-        # XXX Perhaps shouldn't actually bother to test?  Could lead
-        #     to problems for complex, threaded code.
-        if not self.locked_status:
-            raise error
-        self.locked_status = False
-        return True
-
-    def locked(self):
-        return self.locked_status
-
-# Used to signal that interrupt_main was called in a "thread"
-_interrupt = False
-# True when not executing in a "thread"
-_main = True
-
-def interrupt_main():
-    """Set _interrupt flag to True to have start_new_thread raise
-    KeyboardInterrupt upon exiting."""
-    if _main:
-        raise KeyboardInterrupt
-    else:
-        global _interrupt
-        _interrupt = True

lib/spack/external/py2/functools32/functools32.py

@@ -1,423 +0,0 @@
-"""functools.py - Tools for working with functions and callable objects
-"""
-# Python module wrapper for _functools C module
-# to allow utilities written in Python to be added
-# to the functools module.
-# Written by Nick Coghlan <ncoghlan at gmail.com>
-# and Raymond Hettinger <python at rcn.com>
-#   Copyright (C) 2006-2010 Python Software Foundation.
-# See C source code for _functools credits/copyright
-
-__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
-           'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial']
-
-from _functools import partial, reduce
-from collections import MutableMapping, namedtuple
-from .reprlib32 import recursive_repr as _recursive_repr
-from weakref import proxy as _proxy
-import sys as _sys
-try:
-    from thread import allocate_lock as Lock
-except ImportError:
-    from ._dummy_thread32 import allocate_lock as Lock
-
-################################################################################
-### OrderedDict
-################################################################################
-
-class _Link(object):
-    __slots__ = 'prev', 'next', 'key', '__weakref__'
-
-class OrderedDict(dict):
-    'Dictionary that remembers insertion order'
-    # An inherited dict maps keys to values.
-    # The inherited dict provides __getitem__, __len__, __contains__, and get.
-    # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as regular dictionaries.
-
-    # The internal self.__map dict maps keys to links in a doubly linked list.
-    # The circular doubly linked list starts and ends with a sentinel element.
-    # The sentinel element never gets deleted (this simplifies the algorithm).
-    # The sentinel is in self.__hardroot with a weakref proxy in self.__root.
-    # The prev links are weakref proxies (to prevent circular references).
-    # Individual links are kept alive by the hard reference in self.__map.
-    # Those hard references disappear when a key is deleted from an OrderedDict.
-
-    def __init__(self, *args, **kwds):
-        '''Initialize an ordered dictionary.  The signature is the same as
-        regular dictionaries, but keyword arguments are not recommended because
-        their insertion order is arbitrary.
-
-        '''
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__root
-        except AttributeError:
-            self.__hardroot = _Link()
-            self.__root = root = _proxy(self.__hardroot)
-            root.prev = root.next = root
-            self.__map = {}
-        self.__update(*args, **kwds)
-
-    def __setitem__(self, key, value,
-                    dict_setitem=dict.__setitem__, proxy=_proxy, Link=_Link):
-        'od.__setitem__(i, y) <==> od[i]=y'
-        # Setting a new item creates a new link at the end of the linked list,
-        # and the inherited dictionary is updated with the new key/value pair.
-        if key not in self:
-            self.__map[key] = link = Link()
-            root = self.__root
-            last = root.prev
-            link.prev, link.next, link.key = last, root, key
-            last.next = link
-            root.prev = proxy(link)
-        dict_setitem(self, key, value)
-
-    def __delitem__(self, key, dict_delitem=dict.__delitem__):
-        'od.__delitem__(y) <==> del od[y]'
-        # Deleting an existing item uses self.__map to find the link which gets
-        # removed by updating the links in the predecessor and successor nodes.
-        dict_delitem(self, key)
-        link = self.__map.pop(key)
-        link_prev = link.prev
-        link_next = link.next
-        link_prev.next = link_next
-        link_next.prev = link_prev
-
-    def __iter__(self):
-        'od.__iter__() <==> iter(od)'
-        # Traverse the linked list in order.
-        root = self.__root
-        curr = root.next
-        while curr is not root:
-            yield curr.key
-            curr = curr.next
-
-    def __reversed__(self):
-        'od.__reversed__() <==> reversed(od)'
-        # Traverse the linked list in reverse order.
-        root = self.__root
-        curr = root.prev
-        while curr is not root:
-            yield curr.key
-            curr = curr.prev
-
-    def clear(self):
-        'od.clear() -> None.  Remove all items from od.'
-        root = self.__root
-        root.prev = root.next = root
-        self.__map.clear()
-        dict.clear(self)
-
-    def popitem(self, last=True):
-        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
-        Pairs are returned in LIFO order if last is true or FIFO order if false.
-
-        '''
-        if not self:
-            raise KeyError('dictionary is empty')
-        root = self.__root
-        if last:
-            link = root.prev
-            link_prev = link.prev
-            link_prev.next = root
-            root.prev = link_prev
-        else:
-            link = root.next
-            link_next = link.next
-            root.next = link_next
-            link_next.prev = root
-        key = link.key
-        del self.__map[key]
-        value = dict.pop(self, key)
-        return key, value
-
-    def move_to_end(self, key, last=True):
-        '''Move an existing element to the end (or beginning if last==False).
-
-        Raises KeyError if the element does not exist.
-        When last=True, acts like a fast version of self[key]=self.pop(key).
-
-        '''
-        link = self.__map[key]
-        link_prev = link.prev
-        link_next = link.next
-        link_prev.next = link_next
-        link_next.prev = link_prev
-        root = self.__root
-        if last:
-            last = root.prev
-            link.prev = last
-            link.next = root
-            last.next = root.prev = link
-        else:
-            first = root.next
-            link.prev = root
-            link.next = first
-            root.next = first.prev = link
-
-    def __sizeof__(self):
-        sizeof = _sys.getsizeof
-        n = len(self) + 1                       # number of links including root
-        size = sizeof(self.__dict__)            # instance dictionary
-        size += sizeof(self.__map) * 2          # internal dict and inherited dict
-        size += sizeof(self.__hardroot) * n     # link objects
-        size += sizeof(self.__root) * n         # proxy objects
-        return size
-
-    update = __update = MutableMapping.update
-    keys = MutableMapping.keys
-    values = MutableMapping.values
-    items = MutableMapping.items
-    __ne__ = MutableMapping.__ne__
-
-    __marker = object()
-
-    def pop(self, key, default=__marker):
-        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
-        value.  If key is not found, d is returned if given, otherwise KeyError
-        is raised.
-
-        '''
-        if key in self:
-            result = self[key]
-            del self[key]
-            return result
-        if default is self.__marker:
-            raise KeyError(key)
-        return default
-
-    def setdefault(self, key, default=None):
-        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
-        if key in self:
-            return self[key]
-        self[key] = default
-        return default
-
-    @_recursive_repr()
-    def __repr__(self):
-        'od.__repr__() <==> repr(od)'
-        if not self:
-            return '%s()' % (self.__class__.__name__,)
-        return '%s(%r)' % (self.__class__.__name__, list(self.items()))
-
-    def __reduce__(self):
-        'Return state information for pickling'
-        items = [[k, self[k]] for k in self]
-        inst_dict = vars(self).copy()
-        for k in vars(OrderedDict()):
-            inst_dict.pop(k, None)
-        if inst_dict:
-            return (self.__class__, (items,), inst_dict)
-        return self.__class__, (items,)
-
-    def copy(self):
-        'od.copy() -> a shallow copy of od'
-        return self.__class__(self)
-
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
-        If not specified, the value defaults to None.
-
-        '''
-        self = cls()
-        for key in iterable:
-            self[key] = value
-        return self
-
-    def __eq__(self, other):
-        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
-        while comparison to a regular mapping is order-insensitive.
-
-        '''
-        if isinstance(other, OrderedDict):
-            return len(self)==len(other) and \
-                   all(p==q for p, q in zip(self.items(), other.items()))
-        return dict.__eq__(self, other)
-
-# update_wrapper() and wraps() are tools to help write
-# wrapper functions that can handle naive introspection
-
-WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__doc__')
-WRAPPER_UPDATES = ('__dict__',)
-def update_wrapper(wrapper,
-                   wrapped,
-                   assigned = WRAPPER_ASSIGNMENTS,
-                   updated = WRAPPER_UPDATES):
-    """Update a wrapper function to look like the wrapped function
-
-       wrapper is the function to be updated
-       wrapped is the original function
-       assigned is a tuple naming the attributes assigned directly
-       from the wrapped function to the wrapper function (defaults to
-       functools.WRAPPER_ASSIGNMENTS)
-       updated is a tuple naming the attributes of the wrapper that
-       are updated with the corresponding attribute from the wrapped
-       function (defaults to functools.WRAPPER_UPDATES)
-    """
-    wrapper.__wrapped__ = wrapped
-    for attr in assigned:
-        try:
-            value = getattr(wrapped, attr)
-        except AttributeError:
-            pass
-        else:
-            setattr(wrapper, attr, value)
-    for attr in updated:
-        getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
-    # Return the wrapper so this can be used as a decorator via partial()
-    return wrapper
-
-def wraps(wrapped,
-          assigned = WRAPPER_ASSIGNMENTS,
-          updated = WRAPPER_UPDATES):
-    """Decorator factory to apply update_wrapper() to a wrapper function
-
-       Returns a decorator that invokes update_wrapper() with the decorated
-       function as the wrapper argument and the arguments to wraps() as the
-       remaining arguments. Default arguments are as for update_wrapper().
-       This is a convenience function to simplify applying partial() to
-       update_wrapper().
-    """
-    return partial(update_wrapper, wrapped=wrapped,
-                   assigned=assigned, updated=updated)
-
-def total_ordering(cls):
-    """Class decorator that fills in missing ordering methods"""
-    convert = {
-        '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)),
-                   ('__le__', lambda self, other: self < other or self == other),
-                   ('__ge__', lambda self, other: not self < other)],
-        '__le__': [('__ge__', lambda self, other: not self <= other or self == other),
-                   ('__lt__', lambda self, other: self <= other and not self == other),
-                   ('__gt__', lambda self, other: not self <= other)],
-        '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)),
-                   ('__ge__', lambda self, other: self > other or self == other),
-                   ('__le__', lambda self, other: not self > other)],
-        '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other),
-                   ('__gt__', lambda self, other: self >= other and not self == other),
-                   ('__lt__', lambda self, other: not self >= other)]
-    }
-    roots = set(dir(cls)) & set(convert)
-    if not roots:
-        raise ValueError('must define at least one ordering operation: < > <= >=')
-    root = max(roots)       # prefer __lt__ to __le__ to __gt__ to __ge__
-    for opname, opfunc in convert[root]:
-        if opname not in roots:
-            opfunc.__name__ = opname
-            opfunc.__doc__ = getattr(int, opname).__doc__
-            setattr(cls, opname, opfunc)
-    return cls
-
-def cmp_to_key(mycmp):
-    """Convert a cmp= function into a key= function"""
-    class K(object):
-        __slots__ = ['obj']
-        def __init__(self, obj):
-            self.obj = obj
-        def __lt__(self, other):
-            return mycmp(self.obj, other.obj) < 0
-        def __gt__(self, other):
-            return mycmp(self.obj, other.obj) > 0
-        def __eq__(self, other):
-            return mycmp(self.obj, other.obj) == 0
-        def __le__(self, other):
-            return mycmp(self.obj, other.obj) <= 0
-        def __ge__(self, other):
-            return mycmp(self.obj, other.obj) >= 0
-        def __ne__(self, other):
-            return mycmp(self.obj, other.obj) != 0
-        __hash__ = None
-    return K
-
-_CacheInfo = namedtuple("CacheInfo", "hits misses maxsize currsize")
-
-def lru_cache(maxsize=100):
-    """Least-recently-used cache decorator.
-
-    If *maxsize* is set to None, the LRU features are disabled and the cache
-    can grow without bound.
-
-    Arguments to the cached function must be hashable.
-
-    View the cache statistics named tuple (hits, misses, maxsize, currsize) with
-    f.cache_info().  Clear the cache and statistics with f.cache_clear().
-    Access the underlying function with f.__wrapped__.
-
-    See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
-
-    """
-    # Users should only access the lru_cache through its public API:
-    #       cache_info, cache_clear, and f.__wrapped__
-    # The internals of the lru_cache are encapsulated for thread safety and
-    # to allow the implementation to change (including a possible C version).
-
-    def decorating_function(user_function,
-                tuple=tuple, sorted=sorted, len=len, KeyError=KeyError):
-
-        hits, misses = [0], [0]
-        kwd_mark = (object(),)          # separates positional and keyword args
-        lock = Lock()                   # needed because OrderedDict isn't threadsafe
-
-        if maxsize is None:
-            cache = dict()              # simple cache without ordering or size limit
-
-            @wraps(user_function)
-            def wrapper(*args, **kwds):
-                key = args
-                if kwds:
-                    key += kwd_mark + tuple(sorted(kwds.items()))
-                try:
-                    result = cache[key]
-                    hits[0] += 1
-                    return result
-                except KeyError:
-                    pass
-                result = user_function(*args, **kwds)
-                cache[key] = result
-                misses[0] += 1
-                return result
-        else:
-            cache = OrderedDict()           # ordered least recent to most recent
-            cache_popitem = cache.popitem
-            cache_renew = cache.move_to_end
-
-            @wraps(user_function)
-            def wrapper(*args, **kwds):
-                key = args
-                if kwds:
-                    key += kwd_mark + tuple(sorted(kwds.items()))
-                with lock:
-                    try:
-                        result = cache[key]
-                        cache_renew(key)    # record recent use of this key
-                        hits[0] += 1
-                        return result
-                    except KeyError:
-                        pass
-                result = user_function(*args, **kwds)
-                with lock:
-                    cache[key] = result     # record recent use of this key
-                    misses[0] += 1
-                    if len(cache) > maxsize:
-                        cache_popitem(0)    # purge least recently used cache entry
-                return result
-
-        def cache_info():
-            """Report cache statistics"""
-            with lock:
-                return _CacheInfo(hits[0], misses[0], maxsize, len(cache))
-
-        def cache_clear():
-            """Clear the cache and cache statistics"""
-            with lock:
-                cache.clear()
-                hits[0] = misses[0] = 0
-
-        wrapper.cache_info = cache_info
-        wrapper.cache_clear = cache_clear
-        return wrapper
-
-    return decorating_function

lib/spack/external/py2/functools32/reprlib32.py

@@ -1,157 +0,0 @@
-"""Redo the builtin repr() (representation) but with limits on most sizes."""
-
-__all__ = ["Repr", "repr", "recursive_repr"]
-
-import __builtin__ as builtins
-from itertools import islice
-try:
-    from thread import get_ident
-except ImportError:
-    from _dummy_thread32 import get_ident
-
-def recursive_repr(fillvalue='...'):
-    'Decorator to make a repr function return fillvalue for a recursive call'
-
-    def decorating_function(user_function):
-        repr_running = set()
-
-        def wrapper(self):
-            key = id(self), get_ident()
-            if key in repr_running:
-                return fillvalue
-            repr_running.add(key)
-            try:
-                result = user_function(self)
-            finally:
-                repr_running.discard(key)
-            return result
-
-        # Can't use functools.wraps() here because of bootstrap issues
-        wrapper.__module__ = getattr(user_function, '__module__')
-        wrapper.__doc__ = getattr(user_function, '__doc__')
-        wrapper.__name__ = getattr(user_function, '__name__')
-        wrapper.__annotations__ = getattr(user_function, '__annotations__', {})
-        return wrapper
-
-    return decorating_function
-
-class Repr:
-
-    def __init__(self):
-        self.maxlevel = 6
-        self.maxtuple = 6
-        self.maxlist = 6
-        self.maxarray = 5
-        self.maxdict = 4
-        self.maxset = 6
-        self.maxfrozenset = 6
-        self.maxdeque = 6
-        self.maxstring = 30
-        self.maxlong = 40
-        self.maxother = 30
-
-    def repr(self, x):
-        return self.repr1(x, self.maxlevel)
-
-    def repr1(self, x, level):
-        typename = type(x).__name__
-        if ' ' in typename:
-            parts = typename.split()
-            typename = '_'.join(parts)
-        if hasattr(self, 'repr_' + typename):
-            return getattr(self, 'repr_' + typename)(x, level)
-        else:
-            return self.repr_instance(x, level)
-
-    def _repr_iterable(self, x, level, left, right, maxiter, trail=''):
-        n = len(x)
-        if level <= 0 and n:
-            s = '...'
-        else:
-            newlevel = level - 1
-            repr1 = self.repr1
-            pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)]
-            if n > maxiter:  pieces.append('...')
-            s = ', '.join(pieces)
-            if n == 1 and trail:  right = trail + right
-        return '%s%s%s' % (left, s, right)
-
-    def repr_tuple(self, x, level):
-        return self._repr_iterable(x, level, '(', ')', self.maxtuple, ',')
-
-    def repr_list(self, x, level):
-        return self._repr_iterable(x, level, '[', ']', self.maxlist)
-
-    def repr_array(self, x, level):
-        header = "array('%s', [" % x.typecode
-        return self._repr_iterable(x, level, header, '])', self.maxarray)
-
-    def repr_set(self, x, level):
-        x = _possibly_sorted(x)
-        return self._repr_iterable(x, level, 'set([', '])', self.maxset)
-
-    def repr_frozenset(self, x, level):
-        x = _possibly_sorted(x)
-        return self._repr_iterable(x, level, 'frozenset([', '])',
-                                   self.maxfrozenset)
-
-    def repr_deque(self, x, level):
-        return self._repr_iterable(x, level, 'deque([', '])', self.maxdeque)
-
-    def repr_dict(self, x, level):
-        n = len(x)
-        if n == 0: return '{}'
-        if level <= 0: return '{...}'
-        newlevel = level - 1
-        repr1 = self.repr1
-        pieces = []
-        for key in islice(_possibly_sorted(x), self.maxdict):
-            keyrepr = repr1(key, newlevel)
-            valrepr = repr1(x[key], newlevel)
-            pieces.append('%s: %s' % (keyrepr, valrepr))
-        if n > self.maxdict: pieces.append('...')
-        s = ', '.join(pieces)
-        return '{%s}' % (s,)
-
-    def repr_str(self, x, level):
-        s = builtins.repr(x[:self.maxstring])
-        if len(s) > self.maxstring:
-            i = max(0, (self.maxstring-3)//2)
-            j = max(0, self.maxstring-3-i)
-            s = builtins.repr(x[:i] + x[len(x)-j:])
-            s = s[:i] + '...' + s[len(s)-j:]
-        return s
-
-    def repr_int(self, x, level):
-        s = builtins.repr(x) # XXX Hope this isn't too slow...
-        if len(s) > self.maxlong:
-            i = max(0, (self.maxlong-3)//2)
-            j = max(0, self.maxlong-3-i)
-            s = s[:i] + '...' + s[len(s)-j:]
-        return s
-
-    def repr_instance(self, x, level):
-        try:
-            s = builtins.repr(x)
-            # Bugs in x.__repr__() can cause arbitrary
-            # exceptions -- then make up something
-        except Exception:
-            return '<%s instance at %x>' % (x.__class__.__name__, id(x))
-        if len(s) > self.maxother:
-            i = max(0, (self.maxother-3)//2)
-            j = max(0, self.maxother-3-i)
-            s = s[:i] + '...' + s[len(s)-j:]
-        return s
-
-
-def _possibly_sorted(x):
-    # Since not all sequences of items can be sorted and comparison
-    # functions may raise arbitrary exceptions, return an unsorted
-    # sequence in that case.
-    try:
-        return sorted(x)
-    except Exception:
-        return list(x)
-
-aRepr = Repr()
-repr = aRepr.repr

lib/spack/external/py2/typing.py

@@ -1,103 +0,0 @@
-# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-"""
-This is a fake set of symbols to allow spack to import typing in python
-versions where we do not support type checking (<3)
-"""
-from collections import defaultdict
-
-# (1) Unparameterized types.
-Annotated = object
-Any = object
-AnyStr = object
-ByteString = object
-Counter = object
-Final = object
-Hashable = object
-NoReturn = object
-Sized = object
-SupportsAbs = object
-SupportsBytes = object
-SupportsComplex = object
-SupportsFloat = object
-SupportsIndex = object
-SupportsInt = object
-SupportsRound = object
-
-# (2) Parameterized types.
-AbstractSet = defaultdict(lambda: object)
-AsyncContextManager = defaultdict(lambda: object)
-AsyncGenerator = defaultdict(lambda: object)
-AsyncIterable = defaultdict(lambda: object)
-AsyncIterator = defaultdict(lambda: object)
-Awaitable = defaultdict(lambda: object)
-Callable = defaultdict(lambda: object)
-ChainMap = defaultdict(lambda: object)
-ClassVar = defaultdict(lambda: object)
-Collection = defaultdict(lambda: object)
-Container = defaultdict(lambda: object)
-ContextManager = defaultdict(lambda: object)
-Coroutine = defaultdict(lambda: object)
-DefaultDict = defaultdict(lambda: object)
-Deque = defaultdict(lambda: object)
-Dict = defaultdict(lambda: object)
-ForwardRef = defaultdict(lambda: object)
-FrozenSet = defaultdict(lambda: object)
-Generator = defaultdict(lambda: object)
-Generic = defaultdict(lambda: object)
-ItemsView = defaultdict(lambda: object)
-Iterable = defaultdict(lambda: object)
-Iterator = defaultdict(lambda: object)
-KeysView = defaultdict(lambda: object)
-List = defaultdict(lambda: object)
-Literal = defaultdict(lambda: object)
-Mapping = defaultdict(lambda: object)
-MappingView = defaultdict(lambda: object)
-MutableMapping = defaultdict(lambda: object)
-MutableSequence = defaultdict(lambda: object)
-MutableSet = defaultdict(lambda: object)
-NamedTuple = defaultdict(lambda: object)
-Optional = defaultdict(lambda: object)
-OrderedDict = defaultdict(lambda: object)
-Reversible = defaultdict(lambda: object)
-Sequence = defaultdict(lambda: object)
-Set = defaultdict(lambda: object)
-Tuple = defaultdict(lambda: object)
-Type = defaultdict(lambda: object)
-TypedDict = defaultdict(lambda: object)
-Union = defaultdict(lambda: object)
-ValuesView = defaultdict(lambda: object)
-
-# (3) Type variable declarations.
-TypeVar = lambda *args, **kwargs: None
-
-# (4) Functions.
-cast = lambda _type, x: x
-get_args = None
-get_origin = None
-get_type_hints = None
-no_type_check = None
-no_type_check_decorator = None
-
-## typing_extensions
-# We get a ModuleNotFoundError when attempting to import anything from typing_extensions
-# if we separate this into a separate typing_extensions.py file for some reason.
-
-# (1) Unparameterized types.
-IntVar = object
-Literal = object
-NewType = object
-Text = object
-
-# (2) Parameterized types.
-Protocol = defaultdict(lambda: object)
-
-# (3) Macro for avoiding evaluation except during type checking.
-TYPE_CHECKING = False
-
-# (4) Decorators.
-final = lambda x: x
-overload = lambda x: x
-runtime_checkable = lambda x: x

lib/spack/llnl/util/argparsewriter.py

@@ -7,11 +7,10 @@
 
 import argparse
 import errno
+import io
 import re
 import sys
 
-from six import StringIO
-
 
 class Command(object):
     """Parsed representation of a command from argparse.
@@ -181,7 +180,7 @@ def __init__(self, prog, out=None, aliases=False, rst_levels=_rst_levels):
         self.rst_levels = rst_levels
 
     def format(self, cmd):
-        string = StringIO()
+        string = io.StringIO()
         string.write(self.begin_command(cmd.prog))
 
         if cmd.description:

lib/spack/llnl/util/compat.py

@@ -1,39 +0,0 @@
-# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-# isort: off
-
-import sys
-
-if sys.version_info < (3,):
-    from itertools import ifilter as filter
-    from itertools import imap as map
-    from itertools import izip as zip
-    from itertools import izip_longest as zip_longest  # novm
-    from urllib import urlencode as urlencode
-    from urllib import urlopen as urlopen
-else:
-    filter = filter
-    map = map
-    zip = zip
-    from itertools import zip_longest as zip_longest  # novm # noqa: F401
-    from urllib.parse import urlencode as urlencode  # novm # noqa: F401
-    from urllib.request import urlopen as urlopen  # novm # noqa: F401
-
-if sys.version_info >= (3, 3):
-    from collections.abc import Hashable as Hashable  # novm
-    from collections.abc import Iterable as Iterable  # novm
-    from collections.abc import Mapping as Mapping  # novm
-    from collections.abc import MutableMapping as MutableMapping  # novm
-    from collections.abc import MutableSequence as MutableSequence  # novm
-    from collections.abc import MutableSet as MutableSet  # novm
-    from collections.abc import Sequence as Sequence  # novm
-else:
-    from collections import Hashable as Hashable  # noqa: F401
-    from collections import Iterable as Iterable  # noqa: F401
-    from collections import Mapping as Mapping  # noqa: F401
-    from collections import MutableMapping as MutableMapping  # noqa: F401
-    from collections import MutableSequence as MutableSequence  # noqa: F401
-    from collections import MutableSet as MutableSet  # noqa: F401
-    from collections import Sequence as Sequence  # noqa: F401

lib/spack/llnl/util/filesystem.py

@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import collections
+import collections.abc
 import errno
 import glob
 import hashlib
@@ -17,10 +18,7 @@
 from contextlib import contextmanager
 from sys import platform as _platform
 
-import six
-
 from llnl.util import tty
-from llnl.util.compat import Sequence
 from llnl.util.lang import dedupe, memoized
 from llnl.util.symlink import islink, symlink
 
@@ -290,9 +288,7 @@ def groupid_to_group(x):
         shutil.copy(filename, tmp_filename)
 
         try:
-            extra_kwargs = {}
-            if sys.version_info > (3, 0):
-                extra_kwargs = {"errors": "surrogateescape"}
+            extra_kwargs = {"errors": "surrogateescape"}
 
             # Open as a text file and filter until the end of the file is
             # reached or we found a marker in the line if it was specified
@@ -522,7 +518,7 @@ def chgrp(path, group, follow_symlinks=True):
     if is_windows:
         raise OSError("Function 'chgrp' is not supported on Windows")
 
-    if isinstance(group, six.string_types):
+    if isinstance(group, str):
         gid = grp.getgrnam(group).gr_gid
     else:
         gid = group
@@ -1019,7 +1015,7 @@ def open_if_filename(str_or_file, mode="r"):
 
     If it's a file object, just yields the file object.
     """
-    if isinstance(str_or_file, six.string_types):
+    if isinstance(str_or_file, str):
         with open(str_or_file, mode) as f:
             yield f
     else:
@@ -1309,46 +1305,34 @@ def visit_directory_tree(root, visitor, rel_path="", depth=0):
         depth (str): current depth from the root
     """
     dir = os.path.join(root, rel_path)
-
-    if sys.version_info >= (3, 5, 0):
-        dir_entries = sorted(os.scandir(dir), key=lambda d: d.name)  # novermin
-    else:
-        dir_entries = os.listdir(dir)
-        dir_entries.sort()
+    dir_entries = sorted(os.scandir(dir), key=lambda d: d.name)
 
     for f in dir_entries:
-        if sys.version_info >= (3, 5, 0):
-            rel_child = os.path.join(rel_path, f.name)
-            islink = f.is_symlink()
-            # On Windows, symlinks to directories are distinct from
-            # symlinks to files, and it is possible to create a
-            # broken symlink to a directory (e.g. using os.symlink
-            # without `target_is_directory=True`), invoking `isdir`
-            # on a symlink on Windows that is broken in this manner
-            # will result in an error. In this case we can work around
-            # the issue by reading the target and resolving the
-            # directory ourselves
-            try:
-                isdir = f.is_dir()
-            except OSError as e:
-                if is_windows and hasattr(e, "winerror") and e.winerror == 5 and islink:
-                    # if path is a symlink, determine destination and
-                    # evaluate file vs directory
-                    link_target = resolve_link_target_relative_to_the_link(f)
-                    # link_target might be relative but
-                    # resolve_link_target_relative_to_the_link
-                    # will ensure that if so, that it is relative
-                    # to the CWD and therefore
-                    # makes sense
-                    isdir = os.path.isdir(link_target)
-                else:
-                    raise e
-
-        else:
-            rel_child = os.path.join(rel_path, f)
-            lexists, islink, isdir = lexists_islink_isdir(os.path.join(dir, f))
-            if not lexists:
-                continue
+        rel_child = os.path.join(rel_path, f.name)
+        islink = f.is_symlink()
+        # On Windows, symlinks to directories are distinct from
+        # symlinks to files, and it is possible to create a
+        # broken symlink to a directory (e.g. using os.symlink
+        # without `target_is_directory=True`), invoking `isdir`
+        # on a symlink on Windows that is broken in this manner
+        # will result in an error. In this case we can work around
+        # the issue by reading the target and resolving the
+        # directory ourselves
+        try:
+            isdir = f.is_dir()
+        except OSError as e:
+            if is_windows and hasattr(e, "winerror") and e.winerror == 5 and islink:
+                # if path is a symlink, determine destination and
+                # evaluate file vs directory
+                link_target = resolve_link_target_relative_to_the_link(f)
+                # link_target might be relative but
+                # resolve_link_target_relative_to_the_link
+                # will ensure that if so, that it is relative
+                # to the CWD and therefore
+                # makes sense
+                isdir = os.path.isdir(link_target)
+            else:
+                raise e
 
         if not isdir and not islink:
             # handle non-symlink files
@@ -1609,14 +1593,14 @@ def find(root, files, recursive=True):
 
     Parameters:
         root (str): The root directory to start searching from
-        files (str or Sequence): Library name(s) to search for
+        files (str or collections.abc.Sequence): Library name(s) to search for
         recursive (bool): if False search only root folder,
             if True descends top-down from the root. Defaults to True.
 
     Returns:
         list: The files that have been found
     """
-    if isinstance(files, six.string_types):
+    if isinstance(files, str):
         files = [files]
 
     if recursive:
@@ -1673,14 +1657,14 @@ def _find_non_recursive(root, search_files):
 # Utilities for libraries and headers
 
 
-class FileList(Sequence):
+class FileList(collections.abc.Sequence):
     """Sequence of absolute paths to files.
 
     Provides a few convenience methods to manipulate file paths.
     """
 
     def __init__(self, files):
-        if isinstance(files, six.string_types):
+        if isinstance(files, str):
             files = [files]
 
         self.files = list(dedupe(files))
@@ -1776,7 +1760,7 @@ def directories(self):
     def directories(self, value):
         value = value or []
         # Accept a single directory as input
-        if isinstance(value, six.string_types):
+        if isinstance(value, str):
             value = [value]
 
         self._directories = [path_to_os_path(os.path.normpath(x))[0] for x in value]
@@ -1912,9 +1896,9 @@ def find_headers(headers, root, recursive=False):
     Returns:
         HeaderList: The headers that have been found
     """
-    if isinstance(headers, six.string_types):
+    if isinstance(headers, str):
         headers = [headers]
-    elif not isinstance(headers, Sequence):
+    elif not isinstance(headers, collections.abc.Sequence):
         message = "{0} expects a string or sequence of strings as the "
         message += "first argument [got {1} instead]"
         message = message.format(find_headers.__name__, type(headers))
@@ -2078,9 +2062,9 @@ def find_system_libraries(libraries, shared=True):
     Returns:
         LibraryList: The libraries that have been found
     """
-    if isinstance(libraries, six.string_types):
+    if isinstance(libraries, str):
         libraries = [libraries]
-    elif not isinstance(libraries, Sequence):
+    elif not isinstance(libraries, collections.abc.Sequence):
         message = "{0} expects a string or sequence of strings as the "
         message += "first argument [got {1} instead]"
         message = message.format(find_system_libraries.__name__, type(libraries))
@@ -2135,9 +2119,9 @@ def find_libraries(libraries, root, shared=True, recursive=False, runtime=True):
     Returns:
         LibraryList: The libraries that have been found
     """
-    if isinstance(libraries, six.string_types):
+    if isinstance(libraries, str):
         libraries = [libraries]
-    elif not isinstance(libraries, Sequence):
+    elif not isinstance(libraries, collections.abc.Sequence):
         message = "{0} expects a string or sequence of strings as the "
         message += "first argument [got {1} instead]"
         message = message.format(find_libraries.__name__, type(libraries))

lib/spack/llnl/util/lang.py

@@ -5,9 +5,11 @@
 
 from __future__ import division
 
+import collections.abc
 import contextlib
 import functools
 import inspect
+import itertools
 import os
 import re
 import sys
@@ -15,11 +17,6 @@
 from datetime import datetime, timedelta
 from typing import Any, Callable, Iterable, List, Tuple
 
-import six
-from six import string_types
-
-from llnl.util.compat import MutableMapping, MutableSequence, zip_longest
-
 # Ignore emacs backups when listing modules
 ignore_modules = [r"^\.#", "~$"]
 
@@ -200,14 +197,9 @@ def _memoized_function(*args, **kwargs):
             return ret
         except TypeError as e:
             # TypeError is raised when indexing into a dict if the key is unhashable.
-            raise six.raise_from(
-                UnhashableArguments(
-                    "args + kwargs '{}' was not hashable for function '{}'".format(
-                        key, func.__name__
-                    ),
-                ),
-                e,
-            )
+            raise UnhashableArguments(
+                "args + kwargs '{}' was not hashable for function '{}'".format(key, func.__name__),
+            ) from e
 
     return _memoized_function
 
@@ -312,7 +304,7 @@ def lazy_eq(lseq, rseq):
     # zip_longest is implemented in native code, so use it for speed.
     # use zip_longest instead of zip because it allows us to tell
     # which iterator was longer.
-    for left, right in zip_longest(liter, riter, fillvalue=done):
+    for left, right in itertools.zip_longest(liter, riter, fillvalue=done):
         if (left is done) or (right is done):
             return False
 
@@ -332,7 +324,7 @@ def lazy_lt(lseq, rseq):
     liter = lseq()
     riter = rseq()
 
-    for left, right in zip_longest(liter, riter, fillvalue=done):
+    for left, right in itertools.zip_longest(liter, riter, fillvalue=done):
         if (left is done) or (right is done):
             return left is done  # left was shorter than right
 
@@ -482,7 +474,7 @@ def add_func_to_class(name, func):
 
 
 @lazy_lexicographic_ordering
-class HashableMap(MutableMapping):
+class HashableMap(collections.abc.MutableMapping):
     """This is a hashable, comparable dictionary.  Hash is performed on
     a tuple of the values in the dictionary."""
 
@@ -574,7 +566,7 @@ def match_predicate(*args):
 
     def match(string):
         for arg in args:
-            if isinstance(arg, string_types):
+            if isinstance(arg, str):
                 if re.search(arg, string):
                     return True
             elif isinstance(arg, list) or isinstance(arg, tuple):
@@ -887,32 +879,28 @@ def load_module_from_file(module_name, module_path):
         ImportError: when the module can't be loaded
         FileNotFoundError: when module_path doesn't exist
     """
+    import importlib.util
+
     if module_name in sys.modules:
         return sys.modules[module_name]
 
     # This recipe is adapted from https://stackoverflow.com/a/67692/771663
-    if sys.version_info[0] == 3 and sys.version_info[1] >= 5:
-        import importlib.util
 
-        spec = importlib.util.spec_from_file_location(module_name, module_path)  # novm
-        module = importlib.util.module_from_spec(spec)  # novm
-        # The module object needs to exist in sys.modules before the
-        # loader executes the module code.
-        #
-        # See https://docs.python.org/3/reference/import.html#loading
-        sys.modules[spec.name] = module
+    spec = importlib.util.spec_from_file_location(module_name, module_path)  # novm
+    module = importlib.util.module_from_spec(spec)  # novm
+    # The module object needs to exist in sys.modules before the
+    # loader executes the module code.
+    #
+    # See https://docs.python.org/3/reference/import.html#loading
+    sys.modules[spec.name] = module
+    try:
+        spec.loader.exec_module(module)
+    except BaseException:
         try:
-            spec.loader.exec_module(module)
-        except BaseException:
-            try:
-                del sys.modules[spec.name]
-            except KeyError:
-                pass
-            raise
-    elif sys.version_info[0] == 2:
-        import imp
-
-        module = imp.load_source(module_name, module_path)
+            del sys.modules[spec.name]
+        except KeyError:
+            pass
+        raise
     return module
 
 
@@ -1030,7 +1018,7 @@ def ensure_last(lst, *elements):
         lst.append(lst.pop(lst.index(elt)))
 
 
-class TypedMutableSequence(MutableSequence):
+class TypedMutableSequence(collections.abc.MutableSequence):
     """Base class that behaves like a list, just with a different type.
 
     Client code can inherit from this base class:

lib/spack/llnl/util/lock.py

@@ -9,7 +9,6 @@
 import sys
 import time
 from datetime import datetime
-from typing import Dict, Tuple  # novm
 
 import llnl.util.tty as tty
 from llnl.util.lang import pretty_seconds
@@ -81,7 +80,7 @@ class OpenFileTracker(object):
 
     def __init__(self):
         """Create a new ``OpenFileTracker``."""
-        self._descriptors = {}  # type: Dict[Tuple[int, int], OpenFile]
+        self._descriptors = {}
 
     def get_fh(self, path):
         """Get a filehandle for a lockfile.
@@ -103,7 +102,7 @@ def get_fh(self, path):
         try:
             # see whether we've seen this inode/pid before
             stat = os.stat(path)
-            key = (stat.st_ino, pid)
+            key = (stat.st_dev, stat.st_ino, pid)
             open_file = self._descriptors.get(key)
 
         except OSError as e:
@@ -129,32 +128,32 @@ def get_fh(self, path):
 
             # if we just created the file, we'll need to get its inode here
             if not stat:
-                inode = os.fstat(fd).st_ino
-                key = (inode, pid)
+                stat = os.fstat(fd)
+                key = (stat.st_dev, stat.st_ino, pid)
 
             self._descriptors[key] = open_file
 
         open_file.refs += 1
         return open_file.fh
 
-    def release_fh(self, path):
-        """Release a filehandle, only closing it if there are no more references."""
-        try:
-            inode = os.stat(path).st_ino
-        except OSError as e:
-            if e.errno != errno.ENOENT:  # only handle file not found
-                raise
-            inode = None  # this will not be in self._descriptors
-
-        key = (inode, os.getpid())
+    def release_by_stat(self, stat):
+        key = (stat.st_dev, stat.st_ino, os.getpid())
         open_file = self._descriptors.get(key)
-        assert open_file, "Attempted to close non-existing lock path: %s" % path
+        assert open_file, "Attempted to close non-existing inode: %s" % stat.st_inode
 
         open_file.refs -= 1
         if not open_file.refs:
             del self._descriptors[key]
             open_file.fh.close()
 
+    def release_by_fh(self, fh):
+        self.release_by_stat(os.fstat(fh.fileno()))
+
+    def purge(self):
+        for key in list(self._descriptors.keys()):
+            self._descriptors[key].fh.close()
+            del self._descriptors[key]
+
 
 #: Open file descriptors for locks in this process. Used to prevent one process
 #: from opening the sam file many times for different byte range locks
@@ -432,8 +431,7 @@ def _unlock(self):
 
         """
         fcntl.lockf(self._file, fcntl.LOCK_UN, self._length, self._start, os.SEEK_SET)
-
-        file_tracker.release_fh(self.path)
+        file_tracker.release_by_fh(self._file)
         self._file = None
         self._reads = 0
         self._writes = 0

lib/spack/llnl/util/symlink.py

@@ -24,7 +24,7 @@ def symlink(real_path, link_path):
     On Windows, use junctions if os.symlink fails.
     """
     if not is_windows or _win32_can_symlink():
-        os.symlink(real_path, link_path)
+        os.symlink(real_path, link_path, target_is_directory=os.path.isdir(real_path))
     else:
         try:
             # Try to use junctions

lib/spack/llnl/util/tty/__init__.py

@@ -6,6 +6,7 @@
 from __future__ import unicode_literals
 
 import contextlib
+import io
 import os
 import struct
 import sys
@@ -14,10 +15,6 @@
 from datetime import datetime
 from sys import platform as _platform
 
-import six
-from six import StringIO
-from six.moves import input
-
 if _platform != "win32":
     import fcntl
     import termios
@@ -183,7 +180,7 @@ def msg(message, *args, **kwargs):
     else:
         cwrite("@*b{%s==>} %s%s" % (st_text, get_timestamp(), cescape(_output_filter(message))))
     for arg in args:
-        print(indent + _output_filter(six.text_type(arg)))
+        print(indent + _output_filter(str(arg)))
 
 
 def info(message, *args, **kwargs):
@@ -201,13 +198,13 @@ def info(message, *args, **kwargs):
         st_text = process_stacktrace(st_countback)
     cprint(
         "@%s{%s==>} %s%s"
-        % (format, st_text, get_timestamp(), cescape(_output_filter(six.text_type(message)))),
+        % (format, st_text, get_timestamp(), cescape(_output_filter(str(message)))),
         stream=stream,
     )
     for arg in args:
         if wrap:
             lines = textwrap.wrap(
-                _output_filter(six.text_type(arg)),
+                _output_filter(str(arg)),
                 initial_indent=indent,
                 subsequent_indent=indent,
                 break_long_words=break_long_words,
@@ -215,7 +212,7 @@ def info(message, *args, **kwargs):
             for line in lines:
                 stream.write(line + "\n")
         else:
-            stream.write(indent + _output_filter(six.text_type(arg)) + "\n")
+            stream.write(indent + _output_filter(str(arg)) + "\n")
 
 
 def verbose(message, *args, **kwargs):
@@ -238,7 +235,7 @@ def error(message, *args, **kwargs):
 
     kwargs.setdefault("format", "*r")
     kwargs.setdefault("stream", sys.stderr)
-    info("Error: " + six.text_type(message), *args, **kwargs)
+    info("Error: " + str(message), *args, **kwargs)
 
 
 def warn(message, *args, **kwargs):
@@ -247,7 +244,7 @@ def warn(message, *args, **kwargs):
 
     kwargs.setdefault("format", "*Y")
     kwargs.setdefault("stream", sys.stderr)
-    info("Warning: " + six.text_type(message), *args, **kwargs)
+    info("Warning: " + str(message), *args, **kwargs)
 
 
 def die(message, *args, **kwargs):
@@ -271,7 +268,7 @@ def get_number(prompt, **kwargs):
     while number is None:
         msg(prompt, newline=False)
         ans = input()
-        if ans == six.text_type(abort):
+        if ans == str(abort):
             return None
 
         if ans:
@@ -336,11 +333,11 @@ def hline(label=None, **kwargs):
         cols -= 2
     cols = min(max_width, cols)
 
-    label = six.text_type(label)
+    label = str(label)
     prefix = char * 2 + " "
     suffix = " " + (cols - len(prefix) - clen(label)) * char
 
-    out = StringIO()
+    out = io.StringIO()
     out.write(prefix)
     out.write(label)
     out.write(suffix)
@@ -372,10 +369,5 @@ def ioctl_gwinsz(fd):
 
         return int(rc[0]), int(rc[1])
     else:
-        if sys.version_info[0] < 3:
-            raise RuntimeError(
-                "Terminal size not obtainable on Windows with a\
-Python version older than 3"
-            )
         rc = (os.environ.get("LINES", 25), os.environ.get("COLUMNS", 80))
         return int(rc[0]), int(rc[1])

lib/spack/llnl/util/tty/colify.py

@@ -8,11 +8,10 @@
 """
 from __future__ import division, unicode_literals
 
+import io
 import os
 import sys
 
-from six import StringIO, text_type
-
 from llnl.util.tty import terminal_size
 from llnl.util.tty.color import cextra, clen
 
@@ -134,7 +133,7 @@ def colify(elts, **options):
         )
 
     # elts needs to be an array of strings so we can count the elements
-    elts = [text_type(elt) for elt in elts]
+    elts = [str(elt) for elt in elts]
     if not elts:
         return (0, ())
 
@@ -232,7 +231,7 @@ def transpose():
 def colified(elts, **options):
     """Invokes the ``colify()`` function but returns the result as a string
     instead of writing it to an output string."""
-    sio = StringIO()
+    sio = io.StringIO()
     options["output"] = sio
     colify(elts, **options)
     return sio.getvalue()

lib/spack/llnl/util/tty/color.py

@@ -65,8 +65,6 @@
 import sys
 from contextlib import contextmanager
 
-import six
-
 
 class ColorParseError(Exception):
     """Raised when a color format fails to parse."""
@@ -259,7 +257,7 @@ def cescape(string):
     Returns:
         (str): the string with color codes escaped
     """
-    string = six.text_type(string)
+    string = str(string)
     string = string.replace("@", "@@")
     string = string.replace("}", "}}")
     return string

lib/spack/llnl/util/tty/log.py

@@ -24,8 +24,6 @@
 from types import ModuleType  # novm
 from typing import Optional  # novm
 
-from six import StringIO, string_types
-
 import llnl.util.tty as tty
 
 termios = None  # type: Optional[ModuleType]
@@ -241,8 +239,7 @@ def __exit__(self, exc_type, exception, traceback):
         """If termios was available, restore old settings."""
         if self.old_cfg:
             self._restore_default_terminal_settings()
-            if sys.version_info >= (3,):
-                atexit.unregister(self._restore_default_terminal_settings)
+            atexit.unregister(self._restore_default_terminal_settings)
 
         # restore SIGSTP and SIGCONT handlers
         if self.old_handlers:
@@ -309,7 +306,7 @@ def __init__(self, file_like):
 
         self.file_like = file_like
 
-        if isinstance(file_like, string_types):
+        if isinstance(file_like, str):
             self.open = True
         elif _file_descriptors_work(file_like):
             self.open = False
@@ -323,12 +320,9 @@ def __init__(self, file_like):
     def unwrap(self):
         if self.open:
             if self.file_like:
-                if sys.version_info < (3,):
-                    self.file = open(self.file_like, "w")
-                else:
-                    self.file = open(self.file_like, "w", encoding="utf-8")  # novm
+                self.file = open(self.file_like, "w", encoding="utf-8")
             else:
-                self.file = StringIO()
+                self.file = io.StringIO()
             return self.file
         else:
             # We were handed an already-open file object. In this case we also
@@ -699,13 +693,10 @@ def __init__(self, sys_attr):
         self.sys_attr = sys_attr
         self.saved_stream = None
         if sys.platform.startswith("win32"):
-            if sys.version_info < (3, 5):
-                libc = ctypes.CDLL(ctypes.util.find_library("c"))
+            if hasattr(sys, "gettotalrefcount"):  # debug build
+                libc = ctypes.CDLL("ucrtbased")
             else:
-                if hasattr(sys, "gettotalrefcount"):  # debug build
-                    libc = ctypes.CDLL("ucrtbased")
-                else:
-                    libc = ctypes.CDLL("api-ms-win-crt-stdio-l1-1-0")
+                libc = ctypes.CDLL("api-ms-win-crt-stdio-l1-1-0")
 
             kernel32 = ctypes.WinDLL("kernel32")
 
@@ -794,7 +785,7 @@ def __enter__(self):
             raise RuntimeError("file argument must be set by __init__ ")
 
         # Open both write and reading on logfile
-        if type(self.logfile) == StringIO:
+        if type(self.logfile) == io.StringIO:
             self._ioflag = True
             # cannot have two streams on tempfile, so we must make our own
             sys.stdout = self.logfile
@@ -927,13 +918,10 @@ def _writer_daemon(
     if sys.version_info < (3, 8) or sys.platform != "darwin":
         os.close(write_fd)
 
-    # Use line buffering (3rd param = 1) since Python 3 has a bug
+    # 1. Use line buffering (3rd param = 1) since Python 3 has a bug
     # that prevents unbuffered text I/O.
-    if sys.version_info < (3,):
-        in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1)
-    else:
-        # Python 3.x before 3.7 does not open with UTF-8 encoding by default
-        in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1, encoding="utf-8")
+    # 2. Python 3.x before 3.7 does not open with UTF-8 encoding by default
+    in_pipe = os.fdopen(read_multiprocess_fd.fd, "r", 1, encoding="utf-8")
 
     if stdin_multiprocess_fd:
         stdin = os.fdopen(stdin_multiprocess_fd.fd)
@@ -1023,7 +1011,7 @@ def _writer_daemon(
 
     finally:
         # send written data back to parent if we used a StringIO
-        if isinstance(log_file, StringIO):
+        if isinstance(log_file, io.StringIO):
             control_pipe.send(log_file.getvalue())
         log_file_wrapper.close()
         close_connection_and_file(read_multiprocess_fd, in_pipe)

lib/spack/spack/__init__.py

@@ -3,11 +3,20 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-#: (major, minor, micro, dev release) tuple
-spack_version_info = (0, 19, 0, "dev0")
-
 #: PEP440 canonical <major>.<minor>.<micro>.<devN> string
-spack_version = ".".join(str(s) for s in spack_version_info)
+__version__ = "0.20.0.dev0"
+spack_version = __version__
+
+
+def __try_int(v):
+    try:
+        return int(v)
+    except ValueError:
+        return v
+
+
+#: (major, minor, micro, dev release) tuple
+spack_version_info = tuple([__try_int(v) for v in __version__.split(".")])
+
 
 __all__ = ["spack_version_info", "spack_version"]
-__version__ = spack_version

lib/spack/spack/audit.py

@@ -37,15 +37,14 @@ def _search_duplicate_compilers(error_cls):
 """
 import ast
 import collections
+import collections.abc
 import inspect
 import itertools
 import pickle
 import re
-
-from six.moves.urllib.request import urlopen
+from urllib.request import urlopen
 
 import llnl.util.lang
-from llnl.util.compat import Sequence
 
 import spack.config
 import spack.patch
@@ -81,7 +80,7 @@ def __hash__(self):
         return hash(value)
 
 
-class AuditClass(Sequence):
+class AuditClass(collections.abc.Sequence):
     def __init__(self, group, tag, description, kwargs):
         """Return an object that acts as a decorator to register functions
         associated with a specific class of sanity checks.
@@ -288,7 +287,7 @@ def _check_build_test_callbacks(pkgs, error_cls):
     errors = []
     for pkg_name in pkgs:
         pkg_cls = spack.repo.path.get_pkg_class(pkg_name)
-        test_callbacks = pkg_cls.build_time_test_callbacks
+        test_callbacks = getattr(pkg_cls, "build_time_test_callbacks", None)
 
         if test_callbacks and "test" in test_callbacks:
             msg = '{0} package contains "test" method in ' "build_time_test_callbacks"

lib/spack/spack/binary_distribution.py

@@ -7,6 +7,7 @@
 import collections
 import hashlib
 import json
+import multiprocessing.pool
 import os
 import shutil
 import sys
@@ -16,9 +17,9 @@
 import traceback
 import warnings
 from contextlib import closing
+from urllib.error import HTTPError, URLError
 
 import ruamel.yaml as yaml
-from six.moves.urllib.error import HTTPError, URLError
 
 import llnl.util.filesystem as fsys
 import llnl.util.lang
@@ -45,6 +46,7 @@
 from spack.relocate import utf8_paths_to_single_binary_regex
 from spack.spec import Spec
 from spack.stage import Stage
+from spack.util.executable import which
 
 _build_cache_relative_path = "build_cache"
 _build_cache_keys_relative_path = "_pgp"
@@ -72,6 +74,10 @@ def __init__(self, errors):
         super(FetchCacheError, self).__init__(self.message)
 
 
+class ListMirrorSpecsError(spack.error.SpackError):
+    """Raised when unable to retrieve list of specs from the mirror"""
+
+
 class BinaryCacheIndex(object):
     """
     The BinaryCacheIndex tracks what specs are available on (usually remote)
@@ -881,37 +887,52 @@ def sign_specfile(key, force, specfile_path):
     spack.util.gpg.sign(key, specfile_path, signed_specfile_path, clearsign=True)
 
 
-def _fetch_spec_from_mirror(spec_url):
-    s = None
-    tty.debug("fetching {0}".format(spec_url))
-    _, _, spec_file = web_util.read_from_url(spec_url)
-    spec_file_contents = codecs.getreader("utf-8")(spec_file).read()
-    # Need full spec.json name or this gets confused with index.json.
-    if spec_url.endswith(".json.sig"):
-        specfile_json = Spec.extract_json_from_clearsig(spec_file_contents)
-        s = Spec.from_dict(specfile_json)
-    elif spec_url.endswith(".json"):
-        s = Spec.from_json(spec_file_contents)
-    elif spec_url.endswith(".yaml"):
-        s = Spec.from_yaml(spec_file_contents)
-    return s
+def _read_specs_and_push_index(file_list, read_method, cache_prefix, db, temp_dir, concurrency):
+    """Read all the specs listed in the provided list, using thread given thread parallelism,
+        generate the index, and push it to the mirror.
 
+    Args:
+        file_list (list(str)): List of urls or file paths pointing at spec files to read
+        read_method: A function taking a single argument, either a url or a file path,
+            and which reads the spec file at that location, and returns the spec.
+        cache_prefix (str): prefix of the build cache on s3 where index should be pushed.
+        db: A spack database used for adding specs and then writing the index.
+        temp_dir (str): Location to write index.json and hash for pushing
+        concurrency (int): Number of parallel processes to use when fetching
 
-def _read_specs_and_push_index(file_list, cache_prefix, db, db_root_dir):
-    for file_path in file_list:
-        try:
-            s = _fetch_spec_from_mirror(url_util.join(cache_prefix, file_path))
-        except (URLError, web_util.SpackWebError) as url_err:
-            tty.error("Error reading specfile: {0}".format(file_path))
-            tty.error(url_err)
+    Return:
+        None
+    """
 
-        if s:
-            db.add(s, None)
-            db.mark(s, "in_buildcache", True)
+    def _fetch_spec_from_mirror(spec_url):
+        spec_file_contents = read_method(spec_url)
+
+        if spec_file_contents:
+            # Need full spec.json name or this gets confused with index.json.
+            if spec_url.endswith(".json.sig"):
+                specfile_json = Spec.extract_json_from_clearsig(spec_file_contents)
+                return Spec.from_dict(specfile_json)
+            if spec_url.endswith(".json"):
+                return Spec.from_json(spec_file_contents)
+            if spec_url.endswith(".yaml"):
+                return Spec.from_yaml(spec_file_contents)
+
+    tp = multiprocessing.pool.ThreadPool(processes=concurrency)
+    try:
+        fetched_specs = tp.map(
+            llnl.util.lang.star(_fetch_spec_from_mirror), [(f,) for f in file_list]
+        )
+    finally:
+        tp.terminate()
+        tp.join()
+
+    for fetched_spec in fetched_specs:
+        db.add(fetched_spec, None)
+        db.mark(fetched_spec, "in_buildcache", True)
 
     # Now generate the index, compute its hash, and push the two files to
     # the mirror.
-    index_json_path = os.path.join(db_root_dir, "index.json")
+    index_json_path = os.path.join(temp_dir, "index.json")
     with open(index_json_path, "w") as f:
         db._write_to_file(f)
 
@@ -921,7 +942,7 @@ def _read_specs_and_push_index(file_list, cache_prefix, db, db_root_dir):
         index_hash = compute_hash(index_string)
 
     # Write the hash out to a local file
-    index_hash_path = os.path.join(db_root_dir, "index.json.hash")
+    index_hash_path = os.path.join(temp_dir, "index.json.hash")
     with open(index_hash_path, "w") as f:
         f.write(index_hash)
 
@@ -942,31 +963,142 @@ def _read_specs_and_push_index(file_list, cache_prefix, db, db_root_dir):
     )
 
 
-def generate_package_index(cache_prefix):
-    """Create the build cache index page.
+def _specs_from_cache_aws_cli(cache_prefix):
+    """Use aws cli to sync all the specs into a local temporary directory.
 
-    Creates (or replaces) the "index.json" page at the location given in
-    cache_prefix.  This page contains a link for each binary package (.yaml or
-    .json) under cache_prefix.
+    Args:
+        cache_prefix (str): prefix of the build cache on s3
+
+    Return:
+        List of the local file paths and a function that can read each one from the file system.
     """
+    read_fn = None
+    file_list = None
+    aws = which("aws")
+
+    def file_read_method(file_path):
+        with open(file_path) as fd:
+            return fd.read()
+
+    tmpspecsdir = tempfile.mkdtemp()
+    sync_command_args = [
+        "s3",
+        "sync",
+        "--exclude",
+        "*",
+        "--include",
+        "*.spec.json.sig",
+        "--include",
+        "*.spec.json",
+        "--include",
+        "*.spec.yaml",
+        cache_prefix,
+        tmpspecsdir,
+    ]
+
+    try:
+        tty.debug(
+            "Using aws s3 sync to download specs from {0} to {1}".format(cache_prefix, tmpspecsdir)
+        )
+        aws(*sync_command_args, output=os.devnull, error=os.devnull)
+        file_list = fsys.find(tmpspecsdir, ["*.spec.json.sig", "*.spec.json", "*.spec.yaml"])
+        read_fn = file_read_method
+    except Exception:
+        tty.warn("Failed to use aws s3 sync to retrieve specs, falling back to parallel fetch")
+        shutil.rmtree(tmpspecsdir)
+
+    return file_list, read_fn
+
+
+def _specs_from_cache_fallback(cache_prefix):
+    """Use spack.util.web module to get a list of all the specs at the remote url.
+
+    Args:
+        cache_prefix (str): Base url of mirror (location of spec files)
+
+    Return:
+        The list of complete spec file urls and a function that can read each one from its
+            remote location (also using the spack.util.web module).
+    """
+    read_fn = None
+    file_list = None
+
+    def url_read_method(url):
+        contents = None
+        try:
+            _, _, spec_file = web_util.read_from_url(url)
+            contents = codecs.getreader("utf-8")(spec_file).read()
+        except (URLError, web_util.SpackWebError) as url_err:
+            tty.error("Error reading specfile: {0}".format(url))
+            tty.error(url_err)
+        return contents
+
     try:
         file_list = [
-            entry
+            url_util.join(cache_prefix, entry)
             for entry in web_util.list_url(cache_prefix)
             if entry.endswith(".yaml")
             or entry.endswith("spec.json")
             or entry.endswith("spec.json.sig")
         ]
+        read_fn = url_read_method
     except KeyError as inst:
         msg = "No packages at {0}: {1}".format(cache_prefix, inst)
         tty.warn(msg)
-        return
     except Exception as err:
         # If we got some kind of S3 (access denied or other connection
         # error), the first non boto-specific class in the exception
         # hierarchy is Exception.  Just print a warning and return
         msg = "Encountered problem listing packages at {0}: {1}".format(cache_prefix, err)
         tty.warn(msg)
+
+    return file_list, read_fn
+
+
+def _spec_files_from_cache(cache_prefix):
+    """Get a list of all the spec files in the mirror and a function to
+    read them.
+
+    Args:
+        cache_prefix (str): Base url of mirror (location of spec files)
+
+    Return:
+        A tuple where the first item is a list of absolute file paths or
+        urls pointing to the specs that should be read from the mirror,
+        and the second item is a function taking a url or file path and
+        returning the spec read from that location.
+    """
+    callbacks = []
+    if cache_prefix.startswith("s3"):
+        callbacks.append(_specs_from_cache_aws_cli)
+
+    callbacks.append(_specs_from_cache_fallback)
+
+    for specs_from_cache_fn in callbacks:
+        file_list, read_fn = specs_from_cache_fn(cache_prefix)
+        if file_list:
+            return file_list, read_fn
+
+    raise ListMirrorSpecsError("Failed to get list of specs from {0}".format(cache_prefix))
+
+
+def generate_package_index(cache_prefix, concurrency=32):
+    """Create or replace the build cache index on the given mirror.  The
+    buildcache index contains an entry for each binary package under the
+    cache_prefix.
+
+    Args:
+        cache_prefix(str): Base url of binary mirror.
+        concurrency: (int): The desired threading concurrency to use when
+            fetching the spec files from the mirror.
+
+    Return:
+        None
+    """
+    try:
+        file_list, read_fn = _spec_files_from_cache(cache_prefix)
+    except ListMirrorSpecsError as err:
+        tty.error("Unabled to generate package index, {0}".format(err))
         return
 
     if any(x.endswith(".yaml") for x in file_list):
@@ -989,7 +1121,7 @@ def generate_package_index(cache_prefix):
     )
 
     try:
-        _read_specs_and_push_index(file_list, cache_prefix, db, db_root_dir)
+        _read_specs_and_push_index(file_list, read_fn, cache_prefix, db, db_root_dir, concurrency)
     except Exception as err:
         msg = "Encountered problem pushing package index to {0}: {1}".format(cache_prefix, err)
         tty.warn(msg)
@@ -1502,7 +1634,7 @@ def make_package_relative(workdir, spec, allow_root):
     if "elf" in platform.binary_formats:
         relocate.make_elf_binaries_relative(cur_path_names, orig_path_names, old_layout_root)
 
-    relocate.raise_if_not_relocatable(cur_path_names, allow_root)
+    allow_root or relocate.ensure_binaries_are_relocatable(cur_path_names)
     orig_path_names = list()
     cur_path_names = list()
     for linkname in buildinfo.get("relocate_links", []):
@@ -1520,7 +1652,7 @@ def check_package_relocatable(workdir, spec, allow_root):
     cur_path_names = list()
     for filename in buildinfo["relocate_binaries"]:
         cur_path_names.append(os.path.join(workdir, filename))
-    relocate.raise_if_not_relocatable(cur_path_names, allow_root)
+    allow_root or relocate.ensure_binaries_are_relocatable(cur_path_names)
 
 
 def dedupe_hardlinks_if_necessary(root, buildinfo):

lib/spack/spack/bootstrap.py

@@ -17,8 +17,6 @@
 import sysconfig
 import uuid
 
-import six
-
 import archspec.cpu
 
 import llnl.util.filesystem as fs
@@ -78,7 +76,7 @@ def _try_import_from_store(module, query_spec, query_info=None):
             command found and the concrete spec providing it
     """
     # If it is a string assume it's one of the root specs by this module
-    if isinstance(query_spec, six.string_types):
+    if isinstance(query_spec, str):
         # We have to run as part of this python interpreter
         query_spec += " ^" + spec_for_current_python()
 
@@ -92,13 +90,6 @@ def _try_import_from_store(module, query_spec, query_info=None):
         ]  # type: list[str]
         path_before = list(sys.path)
 
-        # Python 3.8+ on Windows does not search dependent DLLs in PATH,
-        # so we need to manually add it using os.add_dll_directory
-        # https://docs.python.org/3/whatsnew/3.8.html#bpo-36085-whatsnew
-        if sys.version_info[:2] >= (3, 8) and sys.platform == "win32":
-            if os.path.isdir(candidate_spec.prefix.bin):
-                os.add_dll_directory(candidate_spec.prefix.bin)  # novermin
-
         # NOTE: try module_paths first and last, last allows an existing version in path
         # to be picked up and used, possibly depending on something in the store, first
         # allows the bootstrap version to work when an incompatible version is in
@@ -476,21 +467,14 @@ def source_is_enabled_or_raise(conf):
 
 def spec_for_current_python():
     """For bootstrapping purposes we are just interested in the Python
-    minor version (all patches are ABI compatible with the same minor)
-    and on whether ucs4 support has been enabled for Python 2.7
+    minor version (all patches are ABI compatible with the same minor).
 
     See:
       https://www.python.org/dev/peps/pep-0513/
       https://stackoverflow.com/a/35801395/771663
     """
     version_str = ".".join(str(x) for x in sys.version_info[:2])
-    variant_str = ""
-    if sys.version_info[0] == 2 and sys.version_info[1] == 7:
-        unicode_size = sysconfig.get_config_var("Py_UNICODE_SIZE")
-        variant_str = "+ucs4" if unicode_size == 4 else "~ucs4"
-
-    spec_fmt = "python@{0} {1}"
-    return spec_fmt.format(version_str, variant_str)
+    return "python@{0}".format(version_str)
 
 
 @contextlib.contextmanager
@@ -873,9 +857,7 @@ def ensure_mypy_in_path_or_raise():
 
 
 def black_root_spec():
-    # black v21 is the last version to support Python 2.7.
-    # Upgrade when we no longer support Python 2.7
-    return _root_spec("py-black@:21")
+    return _root_spec("py-black")
 
 
 def ensure_black_in_path_or_raise():
@@ -932,7 +914,7 @@ def _missing(name, purpose, system_only=True):
 
 def _required_system_executable(exes, msg):
     """Search for an executable is the system path only."""
-    if isinstance(exes, six.string_types):
+    if isinstance(exes, str):
         exes = (exes,)
     if spack.util.executable.which_string(*exes):
         return True, None
@@ -950,7 +932,7 @@ def _required_python_module(module, query_spec, msg):
 
 def _required_executable(exes, query_spec, msg):
     """Search for an executable in the system path or in the bootstrap store."""
-    if isinstance(exes, six.string_types):
+    if isinstance(exes, str):
         exes = (exes,)
     if spack.util.executable.which_string(*exes) or _executables_in_store(exes, query_spec):
         return True, None

lib/spack/spack/build_environment.py

@@ -33,6 +33,7 @@
 calls you can make from within the install() function.
 """
 import inspect
+import io
 import multiprocessing
 import os
 import re
@@ -41,8 +42,6 @@
 import traceback
 import types
 
-from six import StringIO
-
 import llnl.util.tty as tty
 from llnl.util.filesystem import install, install_tree, mkdirp
 from llnl.util.lang import dedupe
@@ -285,6 +284,23 @@ def clean_environment():
     return env
 
 
+def _add_werror_handling(keep_werror, env):
+    keep_flags = set()
+    # set of pairs
+    replace_flags = []  # type: List[Tuple[str,str]]
+    if keep_werror == "all":
+        keep_flags.add("-Werror*")
+    else:
+        if keep_werror == "specific":
+            keep_flags.add("-Werror-*")
+            keep_flags.add("-Werror=*")
+        # This extra case is to handle -Werror-implicit-function-declaration
+        replace_flags.append(("-Werror-", "-Wno-error="))
+        replace_flags.append(("-Werror", "-Wno-error"))
+    env.set("SPACK_COMPILER_FLAGS_KEEP", "|".join(keep_flags))
+    env.set("SPACK_COMPILER_FLAGS_REPLACE", " ".join(["|".join(item) for item in replace_flags]))
+
+
 def set_compiler_environment_variables(pkg, env):
     assert pkg.spec.concrete
     compiler = pkg.compiler
@@ -331,6 +347,13 @@ def set_compiler_environment_variables(pkg, env):
         env.set("SPACK_DTAGS_TO_STRIP", compiler.disable_new_dtags)
         env.set("SPACK_DTAGS_TO_ADD", compiler.enable_new_dtags)
 
+    if pkg.keep_werror is not None:
+        keep_werror = pkg.keep_werror
+    else:
+        keep_werror = spack.config.get("config:flags:keep_werror")
+
+    _add_werror_handling(keep_werror, env)
+
     # Set the target parameters that the compiler will add
     # Don't set on cray platform because the targeting module handles this
     if spec.satisfies("platform=cray"):
@@ -353,10 +376,8 @@ def set_compiler_environment_variables(pkg, env):
         if isinstance(pkg.flag_handler, types.FunctionType):
             handler = pkg.flag_handler
         else:
-            if sys.version_info >= (3, 0):
-                handler = pkg.flag_handler.__func__
-            else:
-                handler = pkg.flag_handler.im_func
+            handler = pkg.flag_handler.__func__
+
         injf, envf, bsf = handler(pkg, flag, spec.compiler_flags[flag][:])
         inject_flags[flag] = injf or []
         env_flags[flag] = envf or []
@@ -542,14 +563,18 @@ def determine_number_of_jobs(
     return min(max_cpus, config_default)
 
 
-def _set_variables_for_single_module(pkg, module):
-    """Helper function to set module variables for single module."""
+def set_module_variables_for_package(pkg):
+    """Populate the Python module of a package with some useful global names.
+    This makes things easier for package writers.
+    """
     # Put a marker on this module so that it won't execute the body of this
     # function again, since it is not needed
     marker = "_set_run_already_called"
-    if getattr(module, marker, False):
+    if getattr(pkg.module, marker, False):
         return
 
+    module = ModuleChangePropagator(pkg)
+
     jobs = determine_number_of_jobs(parallel=pkg.parallel)
 
     m = module
@@ -569,6 +594,7 @@ def _set_variables_for_single_module(pkg, module):
 
     if sys.platform == "win32":
         m.nmake = Executable("nmake")
+        m.msbuild = Executable("msbuild")
     # Standard CMake arguments
     m.std_cmake_args = spack.build_systems.cmake.CMakeBuilder.std_args(pkg)
     m.std_meson_args = spack.build_systems.meson.MesonBuilder.std_args(pkg)
@@ -616,20 +642,7 @@ def static_to_shared_library(static_lib, shared_lib=None, **kwargs):
     # Put a marker on this module so that it won't execute the body of this
     # function again, since it is not needed
     setattr(m, marker, True)
-
-
-def set_module_variables_for_package(pkg):
-    """Populate the module scope of install() with some useful functions.
-    This makes things easier for package writers.
-    """
-    # If a user makes their own package repo, e.g.
-    # spack.pkg.mystuff.libelf.Libelf, and they inherit from an existing class
-    # like spack.pkg.original.libelf.Libelf, then set the module variables
-    # for both classes so the parent class can still use them if it gets
-    # called. parent_class_modules includes pkg.module.
-    modules = parent_class_modules(pkg.__class__)
-    for mod in modules:
-        _set_variables_for_single_module(pkg, mod)
+    module.propagate_changes_to_mro()
 
 
 def _static_to_shared_library(arch, compiler, static_lib, shared_lib=None, **kwargs):
@@ -739,25 +752,6 @@ def get_rpaths(pkg):
     return list(dedupe(filter_system_paths(rpaths)))
 
 
-def parent_class_modules(cls):
-    """
-    Get list of superclass modules that descend from spack.package_base.PackageBase
-
-    Includes cls.__module__
-    """
-    if not issubclass(cls, spack.package_base.PackageBase) or issubclass(
-        spack.package_base.PackageBase, cls
-    ):
-        return []
-    result = []
-    module = sys.modules.get(cls.__module__)
-    if module:
-        result = [module]
-    for c in cls.__bases__:
-        result.extend(parent_class_modules(c))
-    return result
-
-
 def load_external_modules(pkg):
     """Traverse a package's spec DAG and load any external modules.
 
@@ -978,22 +972,9 @@ def add_modifications_for_dep(dep):
             if set_package_py_globals:
                 set_module_variables_for_package(dpkg)
 
-            # Allow dependencies to modify the module
-            # Get list of modules that may need updating
-            modules = []
-            for cls in inspect.getmro(type(spec.package)):
-                module = cls.module
-                if module == spack.package_base:
-                    break
-                modules.append(module)
-
-            # Execute changes as if on a single module
-            # copy dict to ensure prior changes are available
-            changes = spack.util.pattern.Bunch()
-            dpkg.setup_dependent_package(changes, spec)
-
-            for module in modules:
-                module.__dict__.update(changes.__dict__)
+            current_module = ModuleChangePropagator(spec.package)
+            dpkg.setup_dependent_package(current_module, spec)
+            current_module.propagate_changes_to_mro()
 
             if context == "build":
                 builder = spack.builder.create(dpkg)
@@ -1271,6 +1252,8 @@ def make_stack(tb, stack=None):
             obj = frame.f_locals["self"]
             if isinstance(obj, spack.package_base.PackageBase):
                 break
+    else:
+        return None
 
     # We found obj, the Package implementation we care about.
     # Point out the location in the install method where we failed.
@@ -1352,7 +1335,7 @@ def __init__(self, msg, module, classname, traceback_string, log_name, log_type,
 
     @property
     def long_message(self):
-        out = StringIO()
+        out = io.StringIO()
         out.write(self._long_message if self._long_message else "")
 
         have_log = self.log_name and os.path.exists(self.log_name)
@@ -1437,3 +1420,51 @@ def write_log_summary(out, log_type, log, last=None):
         # If no errors are found but warnings are, display warnings
         out.write("\n%s found in %s log:\n" % (plural(nwar, "warning"), log_type))
         out.write(make_log_context(warnings))
+
+
+class ModuleChangePropagator:
+    """Wrapper class to accept changes to a package.py Python module, and propagate them in the
+    MRO of the package.
+
+    It is mainly used as a substitute of the ``package.py`` module, when calling the
+    "setup_dependent_package" function during build environment setup.
+    """
+
+    _PROTECTED_NAMES = ("package", "current_module", "modules_in_mro", "_set_attributes")
+
+    def __init__(self, package):
+        self._set_self_attributes("package", package)
+        self._set_self_attributes("current_module", package.module)
+
+        #: Modules for the classes in the MRO up to PackageBase
+        modules_in_mro = []
+        for cls in inspect.getmro(type(package)):
+            module = cls.module
+
+            if module == self.current_module:
+                continue
+
+            if module == spack.package_base:
+                break
+
+            modules_in_mro.append(module)
+        self._set_self_attributes("modules_in_mro", modules_in_mro)
+        self._set_self_attributes("_set_attributes", {})
+
+    def _set_self_attributes(self, key, value):
+        super().__setattr__(key, value)
+
+    def __getattr__(self, item):
+        return getattr(self.current_module, item)
+
+    def __setattr__(self, key, value):
+        if key in ModuleChangePropagator._PROTECTED_NAMES:
+            msg = f'Cannot set attribute "{key}" in ModuleMonkeyPatcher'
+            return AttributeError(msg)
+
+        setattr(self.current_module, key, value)
+        self._set_attributes[key] = value
+
+    def propagate_changes_to_mro(self):
+        for module_in_mro in self.modules_in_mro:
+            module_in_mro.__dict__.update(self._set_attributes)

lib/spack/spack/build_systems/_checks.py

@@ -4,8 +4,6 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
 
-import six
-
 import llnl.util.lang
 
 import spack.builder
@@ -26,7 +24,7 @@ def sanity_check_prefix(builder):
     pkg = builder.pkg
 
     def check_paths(path_list, filetype, predicate):
-        if isinstance(path_list, six.string_types):
+        if isinstance(path_list, str):
             path_list = [path_list]
 
         for path in path_list:
@@ -89,11 +87,11 @@ def ensure_build_dependencies_or_raise(spec, dependencies, error_msg):
     )
 
     for dep in missing_deps:
-        msg += "    depends_on('{0}', type='build', when='@{1} {2}')\n".format(
+        msg += '    depends_on("{0}", type="build", when="@{1} {2}")\n'.format(
             dep, spec.version, "build_system=autotools"
         )
 
-    msg += "\nUpdate the version (when='@{0}') as needed.".format(spec.version)
+    msg += '\nUpdate the version (when="@{0}") as needed.'.format(spec.version)
     raise RuntimeError(msg)
 
 

lib/spack/spack/build_systems/cached_cmake.py

@@ -205,13 +205,7 @@ def initconfig_hardware_entries(self):
             entries.append(cmake_cache_path("CUDA_TOOLKIT_ROOT_DIR", cudatoolkitdir))
             cudacompiler = "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc"
             entries.append(cmake_cache_path("CMAKE_CUDA_COMPILER", cudacompiler))
-
-            if spec.satisfies("^mpi"):
-                entries.append(cmake_cache_path("CMAKE_CUDA_HOST_COMPILER", "${MPI_CXX_COMPILER}"))
-            else:
-                entries.append(
-                    cmake_cache_path("CMAKE_CUDA_HOST_COMPILER", "${CMAKE_CXX_COMPILER}")
-                )
+            entries.append(cmake_cache_path("CMAKE_CUDA_HOST_COMPILER", "${CMAKE_CXX_COMPILER}"))
 
         return entries
 

lib/spack/spack/build_systems/cmake.py

@@ -2,6 +2,7 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import collections.abc
 import inspect
 import os
 import platform
@@ -9,10 +10,7 @@
 import sys
 from typing import List, Tuple
 
-import six
-
 import llnl.util.filesystem as fs
-from llnl.util.compat import Sequence
 
 import spack.build_environment
 import spack.builder
@@ -302,7 +300,7 @@ def define(cmake_var, value):
             value = "ON" if value else "OFF"
         else:
             kind = "STRING"
-            if isinstance(value, Sequence) and not isinstance(value, six.string_types):
+            if isinstance(value, collections.abc.Sequence) and not isinstance(value, str):
                 value = ";".join(str(v) for v in value)
             else:
                 value = str(value)

lib/spack/spack/build_systems/meson.py

@@ -142,15 +142,17 @@ def std_args(pkg):
             default_library = "shared"
 
         args = [
-            "--prefix={0}".format(pkg.prefix),
+            "-Dprefix={0}".format(pkg.prefix),
             # If we do not specify libdir explicitly, Meson chooses something
             # like lib/x86_64-linux-gnu, which causes problems when trying to
             # find libraries and pkg-config files.
             # See https://github.com/mesonbuild/meson/issues/2197
-            "--libdir={0}".format(pkg.prefix.lib),
+            "-Dlibdir={0}".format(pkg.prefix.lib),
             "-Dbuildtype={0}".format(build_type),
             "-Dstrip={0}".format(strip),
             "-Ddefault_library={0}".format(default_library),
+            # Do not automatically download and install dependencies
+            "-Dwrap_mode=nodownload",
         ]
 
         return args

lib/spack/spack/build_systems/python.py

@@ -15,6 +15,7 @@
 import spack.builder
 import spack.multimethod
 import spack.package_base
+import spack.spec
 from spack.directives import build_system, depends_on, extends
 from spack.error import NoHeadersError, NoLibrariesError, SpecError
 from spack.version import Version
@@ -218,6 +219,27 @@ def list_url(cls):
             name = cls.pypi.split("/")[0]
             return "https://pypi.org/simple/" + name + "/"
 
+    def update_external_dependencies(self):
+        """
+        Ensure all external python packages have a python dependency
+
+        If another package in the DAG depends on python, we use that
+        python for the dependency of the external. If not, we assume
+        that the external PythonPackage is installed into the same
+        directory as the python it depends on.
+        """
+        # TODO: Include this in the solve, rather than instantiating post-concretization
+        if "python" not in self.spec:
+            if "python" in self.spec.root:
+                python = self.spec.root["python"]
+            else:
+                python = spack.spec.Spec("python")
+                repo = spack.repo.path.repo_for_pkg(python)
+                python.namespace = repo.namespace
+                python._mark_concrete()
+                python.external_path = self.prefix
+            self.spec.add_dependency_edge(python, ("build", "link", "run"))
+
     @property
     def headers(self):
         """Discover header files in platlib."""

lib/spack/spack/build_systems/rocm.py

@@ -96,18 +96,33 @@ class ROCmPackage(PackageBase):
         "gfx803",
         "gfx900",
         "gfx900:xnack-",
+        "gfx902",
+        "gfx904",
         "gfx906",
-        "gfx908",
-        "gfx90a",
         "gfx906:xnack-",
+        "gfx908",
         "gfx908:xnack-",
+        "gfx909",
+        "gfx90a",
         "gfx90a:xnack-",
         "gfx90a:xnack+",
+        "gfx90c",
+        "gfx940",
         "gfx1010",
         "gfx1011",
         "gfx1012",
+        "gfx1013",
         "gfx1030",
         "gfx1031",
+        "gfx1032",
+        "gfx1033",
+        "gfx1034",
+        "gfx1035",
+        "gfx1036",
+        "gfx1100",
+        "gfx1101",
+        "gfx1102",
+        "gfx1103",
     )
 
     variant("rocm", default=False, description="Enable ROCm support")
@@ -144,6 +159,29 @@ def hip_flags(amdgpu_target):
     # depends_on('hip@:6.0', when='amdgpu_target=gfx701')
     # to indicate minimum version for each architecture.
 
+    # Add compiler minimum versions based on the first release where the
+    # processor is included in llvm/lib/Support/TargetParser.cpp
+    depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx900:xnack-")
+    depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx906:xnack-")
+    depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx908:xnack-")
+    depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx90c")
+    depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx90a")
+    depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx90a:xnack-")
+    depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx90a:xnack+")
+    depends_on("llvm-amdgpu@5.2.0:", when="amdgpu_target=gfx940")
+    depends_on("llvm-amdgpu@4.5.0:", when="amdgpu_target=gfx1013")
+    depends_on("llvm-amdgpu@3.8.0:", when="amdgpu_target=gfx1030")
+    depends_on("llvm-amdgpu@3.9.0:", when="amdgpu_target=gfx1031")
+    depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx1032")
+    depends_on("llvm-amdgpu@4.1.0:", when="amdgpu_target=gfx1033")
+    depends_on("llvm-amdgpu@4.3.0:", when="amdgpu_target=gfx1034")
+    depends_on("llvm-amdgpu@4.5.0:", when="amdgpu_target=gfx1035")
+    depends_on("llvm-amdgpu@5.2.0:", when="amdgpu_target=gfx1036")
+    depends_on("llvm-amdgpu@5.3.0:", when="amdgpu_target=gfx1100")
+    depends_on("llvm-amdgpu@5.3.0:", when="amdgpu_target=gfx1101")
+    depends_on("llvm-amdgpu@5.3.0:", when="amdgpu_target=gfx1102")
+    depends_on("llvm-amdgpu@5.3.0:", when="amdgpu_target=gfx1103")
+
     # Compiler conflicts
 
     # TODO: add conflicts statements along the lines of

lib/spack/spack/build_systems/scons.py

@@ -21,8 +21,6 @@ class SConsPackage(spack.package_base.PackageBase):
     #: build-system class we are using
     build_system_class = "SConsPackage"
 
-    #: Callback names for build-time test
-    build_time_test_callbacks = ["build_test"]
     #: Legacy buildsystem attribute used to deserialize and install old specs
     legacy_buildsystem = "scons"
 
@@ -48,18 +46,24 @@ class SConsBuilder(BaseBuilder):
     phases = ("build", "install")
 
     #: Names associated with package methods in the old build-system format
-    legacy_methods = ("build_args", "install_args", "build_test")
+    legacy_methods = ("install_args", "build_test")
+
+    #: Same as legacy_methods, but the signature is different
+    legacy_long_methods = ("build_args",)
 
     #: Names associated with package attributes in the old build-system format
-    legacy_attributes = ()
+    legacy_attributes = ("build_time_test_callbacks",)
 
-    def build_args(self):
+    #: Callback names for build-time test
+    build_time_test_callbacks = ["build_test"]
+
+    def build_args(self, spec, prefix):
         """Arguments to pass to build."""
         return []
 
     def build(self, pkg, spec, prefix):
         """Build the package."""
-        args = self.build_args()
+        args = self.build_args(spec, prefix)
         inspect.getmodule(self.pkg).scons(*args)
 
     def install_args(self):

lib/spack/spack/build_systems/sip.py

@@ -157,7 +157,7 @@ def configure(self, pkg, spec, prefix):
             ]
         )
 
-        self.python(configure, *args)
+        self.pkg.python(configure, *args)
 
     def configure_args(self):
         """Arguments to pass to configure."""

lib/spack/spack/build_systems/waf.py

@@ -72,9 +72,9 @@ class WafBuilder(BaseBuilder):
 
     #: Names associated with package attributes in the old build-system format
     legacy_attributes = (
-        "build_time_test_callbacks",
         "build_time_test_callbacks",
         "build_directory",
+        "install_time_test_callbacks",
     )
 
     # Callback names for build-time test

lib/spack/spack/builder.py

@@ -3,15 +3,12 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import collections
+import collections.abc
 import copy
 import functools
 import inspect
 from typing import List, Optional, Tuple
 
-import six
-
-import llnl.util.compat
-
 import spack.build_environment
 
 #: Builder classes, as registered by the "builder" decorator
@@ -168,7 +165,7 @@ def __forward(self):
             property(forward_property_to_getattr(attribute_name)),
         )
 
-    class Adapter(six.with_metaclass(_PackageAdapterMeta, base_cls)):
+    class Adapter(base_cls, metaclass=_PackageAdapterMeta):
         def __init__(self, pkg):
             # Deal with custom phases in packages here
             if hasattr(pkg, "phases"):
@@ -280,7 +277,7 @@ def _decorator(fn):
         return _decorator
 
 
-class BuilderMeta(PhaseCallbacksMeta, type(llnl.util.compat.Sequence)):  # type: ignore
+class BuilderMeta(PhaseCallbacksMeta, type(collections.abc.Sequence)):  # type: ignore
     pass
 
 
@@ -457,7 +454,7 @@ def copy(self):
         return copy.deepcopy(self)
 
 
-class Builder(six.with_metaclass(BuilderMeta, llnl.util.compat.Sequence)):
+class Builder(collections.abc.Sequence, metaclass=BuilderMeta):
     """A builder is a class that, given a package object (i.e. associated with
     concrete spec), knows how to install it.
 

lib/spack/spack/ci.py

@@ -16,11 +16,9 @@
 import tempfile
 import time
 import zipfile
-
-from six import iteritems, string_types
-from six.moves.urllib.error import HTTPError, URLError
-from six.moves.urllib.parse import urlencode
-from six.moves.urllib.request import HTTPHandler, Request, build_opener
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode
+from urllib.request import HTTPHandler, Request, build_opener
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
@@ -215,7 +213,7 @@ def stage_spec_jobs(specs, check_index_only=False, mirrors_to_check=None):
     def _remove_satisfied_deps(deps, satisfied_list):
         new_deps = {}
 
-        for key, value in iteritems(deps):
+        for key, value in deps.items():
             new_value = set([v for v in value if v not in satisfied_list])
             if new_value:
                 new_deps[key] = new_value
@@ -1167,7 +1165,14 @@ def generate_gitlab_ci_yaml(
         "after_script",
     ]
 
-    service_job_retries = {"max": 2, "when": ["runner_system_failure", "stuck_or_timeout_failure"]}
+    service_job_retries = {
+        "max": 2,
+        "when": [
+            "runner_system_failure",
+            "stuck_or_timeout_failure",
+            "script_failure",
+        ],
+    }
 
     if job_id > 0:
         if temp_storage_url_prefix:
@@ -1762,9 +1767,9 @@ def reproduce_ci_job(url, work_dir):
     download_and_extract_artifacts(url, work_dir)
 
     lock_file = fs.find(work_dir, "spack.lock")[0]
-    concrete_env_dir = os.path.dirname(lock_file)
+    repro_lock_dir = os.path.dirname(lock_file)
 
-    tty.debug("Concrete environment directory: {0}".format(concrete_env_dir))
+    tty.debug("Found lock file in: {0}".format(repro_lock_dir))
 
     yaml_files = fs.find(work_dir, ["*.yaml", "*.yml"])
 
@@ -1787,6 +1792,20 @@ def reproduce_ci_job(url, work_dir):
     if pipeline_yaml:
         tty.debug("\n{0} is likely your pipeline file".format(yf))
 
+    relative_concrete_env_dir = pipeline_yaml["variables"]["SPACK_CONCRETE_ENV_DIR"]
+    tty.debug("Relative environment path used by cloud job: {0}".format(relative_concrete_env_dir))
+
+    # Using the relative concrete environment path found in the generated
+    # pipeline variable above, copy the spack environment files so they'll
+    # be found in the same location as when the job ran in the cloud.
+    concrete_env_dir = os.path.join(work_dir, relative_concrete_env_dir)
+    os.makedirs(concrete_env_dir, exist_ok=True)
+    copy_lock_path = os.path.join(concrete_env_dir, "spack.lock")
+    orig_yaml_path = os.path.join(repro_lock_dir, "spack.yaml")
+    copy_yaml_path = os.path.join(concrete_env_dir, "spack.yaml")
+    shutil.copyfile(lock_file, copy_lock_path)
+    shutil.copyfile(orig_yaml_path, copy_yaml_path)
+
     # Find the install script in the unzipped artifacts and make it executable
     install_script = fs.find(work_dir, "install.sh")[0]
     st = os.stat(install_script)
@@ -1842,6 +1861,7 @@ def reproduce_ci_job(url, work_dir):
         if repro_details:
             mount_as_dir = repro_details["ci_project_dir"]
             mounted_repro_dir = os.path.join(mount_as_dir, rel_repro_dir)
+            mounted_env_dir = os.path.join(mount_as_dir, relative_concrete_env_dir)
 
         # We will also try to clone spack from your local checkout and
         # reproduce the state present during the CI build, and put that into
@@ -1925,7 +1945,7 @@ def reproduce_ci_job(url, work_dir):
     inst_list.append("        $ source {0}/share/spack/setup-env.sh\n".format(spack_root))
     inst_list.append(
         "        $ spack env activate --without-view {0}\n\n".format(
-            mounted_repro_dir if job_image else repro_dir
+            mounted_env_dir if job_image else repro_dir
         )
     )
     inst_list.append("    - Run the install script\n\n")
@@ -1953,7 +1973,7 @@ def process_command(name, commands, repro_dir):
     """
     tty.debug("spack {0} arguments: {1}".format(name, commands))
 
-    if len(commands) == 0 or isinstance(commands[0], string_types):
+    if len(commands) == 0 or isinstance(commands[0], str):
         commands = [commands]
 
     # Create a string [command 1] && [command 2] && ... && [command n] with commands

lib/spack/spack/ci_needs_workaround.py

@@ -2,12 +2,11 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-from llnl.util.compat import Mapping
+import collections.abc
 
 get_job_name = lambda needs_entry: (
     needs_entry.get("job")
-    if (isinstance(needs_entry, Mapping) and needs_entry.get("artifacts", True))
+    if (isinstance(needs_entry, collections.abc.Mapping) and needs_entry.get("artifacts", True))
     else needs_entry
     if isinstance(needs_entry, str)
     else None
@@ -15,7 +14,7 @@
 
 
 def convert_job(job_entry):
-    if not isinstance(job_entry, Mapping):
+    if not isinstance(job_entry, collections.abc.Mapping):
         return job_entry
 
     needs = job_entry.get("needs")

lib/spack/spack/ci_optimization.py

@@ -2,23 +2,21 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
+import collections
+import collections.abc
 import copy
 import hashlib
-from collections import defaultdict
-
-from llnl.util.compat import Mapping, Sequence
 
 import spack.util.spack_yaml as syaml
 
 
 def sort_yaml_obj(obj):
-    if isinstance(obj, Mapping):
+    if isinstance(obj, collections.abc.Mapping):
         return syaml.syaml_dict(
             (k, sort_yaml_obj(v)) for k, v in sorted(obj.items(), key=(lambda item: str(item[0])))
         )
 
-    if isinstance(obj, Sequence) and not isinstance(obj, str):
+    if isinstance(obj, collections.abc.Sequence) and not isinstance(obj, str):
         return syaml.syaml_list(sort_yaml_obj(x) for x in obj)
 
     return obj
@@ -38,15 +36,15 @@ def matches(obj, proto):
 
     Precondition: proto must not have any reference cycles
     """
-    if isinstance(obj, Mapping):
-        if not isinstance(proto, Mapping):
+    if isinstance(obj, collections.abc.Mapping):
+        if not isinstance(proto, collections.abc.Mapping):
             return False
 
         return all((key in obj and matches(obj[key], val)) for key, val in proto.items())
 
-    if isinstance(obj, Sequence) and not isinstance(obj, str):
+    if isinstance(obj, collections.abc.Sequence) and not isinstance(obj, str):
 
-        if not (isinstance(proto, Sequence) and not isinstance(proto, str)):
+        if not (isinstance(proto, collections.abc.Sequence) and not isinstance(proto, str)):
             return False
 
         if len(obj) != len(proto):
@@ -76,7 +74,9 @@ def subkeys(obj, proto):
 
     Otherwise, obj is returned.
     """
-    if not (isinstance(obj, Mapping) and isinstance(proto, Mapping)):
+    if not (
+        isinstance(obj, collections.abc.Mapping) and isinstance(proto, collections.abc.Mapping)
+    ):
         return obj
 
     new_obj = {}
@@ -88,7 +88,7 @@ def subkeys(obj, proto):
         if matches(value, proto[key]) and matches(proto[key], value):
             continue
 
-        if isinstance(value, Mapping):
+        if isinstance(value, collections.abc.Mapping):
             new_obj[key] = subkeys(value, proto[key])
             continue
 
@@ -116,7 +116,7 @@ def add_extends(yaml, key):
     has_key = "extends" in yaml
     extends = yaml.get("extends")
 
-    if has_key and not isinstance(extends, (str, Sequence)):
+    if has_key and not isinstance(extends, (str, collections.abc.Sequence)):
         return
 
     if extends is None:
@@ -261,7 +261,7 @@ def build_histogram(iterator, key):
     The list is sorted in descending order by count, yielding the most
     frequently occuring hashes first.
     """
-    buckets = defaultdict(int)
+    buckets = collections.defaultdict(int)
     values = {}
 
     num_objects = 0

lib/spack/spack/cmd/__init__.py

@@ -14,7 +14,6 @@
 from typing import List, Tuple
 
 import ruamel.yaml as yaml
-import six
 from ruamel.yaml.error import MarkedYAMLError
 
 import llnl.util.tty as tty
@@ -30,6 +29,7 @@
 import spack.paths
 import spack.spec
 import spack.store
+import spack.traverse as traverse
 import spack.user_environment as uenv
 import spack.util.spack_json as sjson
 import spack.util.string
@@ -216,7 +216,7 @@ def parse_specs(args, **kwargs):
     tests = kwargs.get("tests", False)
 
     sargs = args
-    if not isinstance(args, six.string_types):
+    if not isinstance(args, str):
         sargs = " ".join(args)
     unquoted_flags = _UnquotedFlags.extract(sargs)
 
@@ -464,11 +464,12 @@ def format_list(specs):
         # create the final, formatted versions of all specs
         formatted = []
         for spec in specs:
-            formatted.append((fmt(spec), spec))
             if deps:
-                for depth, dep in spec.traverse(root=False, depth=True):
-                    formatted.append((fmt(dep, depth), dep))
+                for depth, dep in traverse.traverse_tree([spec], depth_first=False):
+                    formatted.append((fmt(dep.spec, depth), dep.spec))
                 formatted.append(("", None))  # mark newlines
+            else:
+                formatted.append((fmt(spec), spec))
 
         # unless any of these are set, we can just colify and be done.
         if not any((deps, paths)):

lib/spack/spack/cmd/activate.py

@@ -1,53 +0,0 @@
-# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-import llnl.util.tty as tty
-
-import spack.cmd
-import spack.cmd.common.arguments as arguments
-import spack.environment as ev
-from spack.filesystem_view import YamlFilesystemView
-
-description = "activate a package extension"
-section = "extensions"
-level = "long"
-
-
-def setup_parser(subparser):
-    subparser.add_argument(
-        "-f", "--force", action="store_true", help="activate without first activating dependencies"
-    )
-    subparser.add_argument("-v", "--view", metavar="VIEW", type=str, help="the view to operate on")
-    arguments.add_common_arguments(subparser, ["installed_spec"])
-
-
-def activate(parser, args):
-
-    tty.warn(
-        "spack activate is deprecated in favor of " "environments and will be removed in v0.19.0"
-    )
-
-    specs = spack.cmd.parse_specs(args.spec)
-    if len(specs) != 1:
-        tty.die("activate requires one spec.  %d given." % len(specs))
-
-    spec = spack.cmd.disambiguate_spec(specs[0], ev.active_environment())
-    if not spec.package.is_extension:
-        tty.die("%s is not an extension." % spec.name)
-
-    if args.view:
-        target = args.view
-    else:
-        target = spec.package.extendee_spec.prefix
-
-    view = YamlFilesystemView(target, spack.store.layout)
-
-    if spec.package.is_activated(view):
-        tty.msg("Package %s is already activated." % specs[0].short_spec)
-        return
-
-    # TODO: refactor FilesystemView.add_extension and use that here (so there
-    # aren't two ways of activating extensions)
-    spec.package.do_activate(view, with_dependencies=not args.force)

lib/spack/spack/cmd/bootstrap.py

@@ -8,7 +8,6 @@
 import platform
 import shutil
 import tempfile
-import warnings
 
 import llnl.util.filesystem
 import llnl.util.tty
@@ -52,6 +51,7 @@
 
 CLINGO_JSON = "$spack/share/spack/bootstrap/github-actions-v0.4/clingo.json"
 GNUPG_JSON = "$spack/share/spack/bootstrap/github-actions-v0.4/gnupg.json"
+PATCHELF_JSON = "$spack/share/spack/bootstrap/github-actions-v0.4/patchelf.json"
 
 # Metadata for a generated source mirror
 SOURCE_METADATA = {
@@ -111,18 +111,10 @@ def setup_parser(subparser):
     list = sp.add_parser("list", help="list all the sources of software to bootstrap Spack")
     _add_scope_option(list)
 
-    trust = sp.add_parser("trust", help="(DEPRECATED) trust a bootstrapping source")
-    _add_scope_option(trust)
-    trust.add_argument("name", help="name of the source to be trusted")
-
-    untrust = sp.add_parser("untrust", help="(DEPRECATED) untrust a bootstrapping source")
-    _add_scope_option(untrust)
-    untrust.add_argument("name", help="name of the source to be untrusted")
-
     add = sp.add_parser("add", help="add a new source for bootstrapping")
     _add_scope_option(add)
     add.add_argument(
-        "--trust", action="store_true", help="trust the source immediately upon addition"
+        "--trust", action="store_true", help="enable the source immediately upon addition"
     )
     add.add_argument("name", help="name of the new source of software")
     add.add_argument("metadata_dir", help="directory where to find metadata files")
@@ -155,9 +147,9 @@ def _enable_or_disable(args):
         return
 
     if value is True:
-        _trust(args)
+        _enable_source(args)
     else:
-        _untrust(args)
+        _disable_source(args)
 
 
 def _reset(args):
@@ -253,8 +245,14 @@ def sort_fn(x):
         _print_method(s, trusted.get(s["name"], None))
 
 
-def _write_trust_state(args, value):
-    name = args.name
+def _write_bootstrapping_source_status(name, enabled, scope=None):
+    """Write if a bootstrapping source is enable or disabled to config file.
+
+    Args:
+        name (str): name of the bootstrapping source.
+        enabled (bool): True if the source is enabled, False if it is disabled.
+        scope (None or str): configuration scope to modify. If none use the default scope.
+    """
     sources = spack.config.get("bootstrap:sources")
 
     matches = [s for s in sources if s["name"] == name]
@@ -276,30 +274,18 @@ def _write_trust_state(args, value):
 
     # Setting the scope explicitly is needed to not copy over to a new scope
     # the entire default configuration for bootstrap.yaml
-    scope = args.scope or spack.config.default_modify_scope("bootstrap")
-    spack.config.add("bootstrap:trusted:{0}:{1}".format(name, str(value)), scope=scope)
+    scope = scope or spack.config.default_modify_scope("bootstrap")
+    spack.config.add("bootstrap:trusted:{0}:{1}".format(name, str(enabled)), scope=scope)
 
 
-def _deprecate_command(deprecated_cmd, suggested_cmd):
-    msg = (
-        "the 'spack bootstrap {} ...' command is deprecated and will be "
-        "removed in v0.20, use 'spack bootstrap {} ...' instead"
-    )
-    warnings.warn(msg.format(deprecated_cmd, suggested_cmd))
-
-
-def _trust(args):
-    if args.subcommand == "trust":
-        _deprecate_command("trust", "enable")
-    _write_trust_state(args, value=True)
+def _enable_source(args):
+    _write_bootstrapping_source_status(args.name, enabled=True, scope=args.scope)
     msg = '"{0}" is now enabled for bootstrapping'
     llnl.util.tty.msg(msg.format(args.name))
 
 
-def _untrust(args):
-    if args.subcommand == "untrust":
-        _deprecate_command("untrust", "disable")
-    _write_trust_state(args, value=False)
+def _disable_source(args):
+    _write_bootstrapping_source_status(args.name, enabled=False, scope=args.scope)
     msg = '"{0}" is now disabled and will not be used for bootstrapping'
     llnl.util.tty.msg(msg.format(args.name))
 
@@ -363,7 +349,7 @@ def _add(args):
     msg = 'New bootstrapping source "{0}" added in the "{1}" configuration scope'
     llnl.util.tty.msg(msg.format(args.name, write_scope))
     if args.trust:
-        _trust(args)
+        _enable_source(args)
 
 
 def _remove(args):
@@ -443,6 +429,7 @@ def write_metadata(subdir, metadata):
         abs_directory, rel_directory = write_metadata(subdir="binaries", metadata=BINARY_METADATA)
         shutil.copy(spack.util.path.canonicalize_path(CLINGO_JSON), abs_directory)
         shutil.copy(spack.util.path.canonicalize_path(GNUPG_JSON), abs_directory)
+        shutil.copy(spack.util.path.canonicalize_path(PATCHELF_JSON), abs_directory)
         instructions += cmd.format("local-binaries", rel_directory)
     print(instructions)
 
@@ -463,8 +450,6 @@ def bootstrap(parser, args):
         "reset": _reset,
         "root": _root,
         "list": _list,
-        "trust": _trust,
-        "untrust": _untrust,
         "add": _add,
         "remove": _remove,
         "mirror": _mirror,

lib/spack/spack/cmd/buildcache.py

@@ -454,7 +454,7 @@ def check_fn(args):
 
     if not specs:
         tty.msg("No specs provided, exiting.")
-        sys.exit(0)
+        return
 
     for spec in specs:
         spec.concretize()
@@ -467,9 +467,10 @@ def check_fn(args):
 
     if not configured_mirrors:
         tty.msg("No mirrors provided, exiting.")
-        sys.exit(0)
+        return
 
-    sys.exit(bindist.check_specs_against_mirrors(configured_mirrors, specs, args.output_file))
+    if bindist.check_specs_against_mirrors(configured_mirrors, specs, args.output_file) == 1:
+        sys.exit(1)
 
 
 def download_fn(args):
@@ -479,11 +480,11 @@ def download_fn(args):
     least one of the required buildcache components."""
     if not args.spec and not args.spec_file:
         tty.msg("No specs provided, exiting.")
-        sys.exit(0)
+        return
 
     if not args.path:
         tty.msg("No download path provided, exiting")
-        sys.exit(0)
+        return
 
     spec = _concrete_spec_from_args(args)
     result = bindist.download_single_spec(spec, args.path)
@@ -532,8 +533,6 @@ def save_specfile_fn(args):
         root_spec_as_json, args.specfile_dir, args.specs.split(), spec_format
     )
 
-    sys.exit(0)
-
 
 def copy_buildcache_file(src_url, dest_url, local_path=None):
     """Copy from source url to destination url"""

lib/spack/spack/cmd/checksum.py

@@ -6,6 +6,7 @@
 from __future__ import print_function
 
 import argparse
+import sys
 
 import llnl.util.tty as tty
 
@@ -16,6 +17,7 @@
 import spack.stage
 import spack.util.crypto
 from spack.package_base import deprecated_version, preferred_version
+from spack.util.editor import editor
 from spack.util.naming import valid_fully_qualified_module_name
 from spack.version import VersionBase, ver
 
@@ -53,6 +55,13 @@ def setup_parser(subparser):
         default=False,
         help="checksum the preferred version only",
     )
+    subparser.add_argument(
+        "-a",
+        "--add-to-package",
+        action="store_true",
+        default=False,
+        help="add new versions to package",
+    )
     arguments.add_common_arguments(subparser, ["package"])
     subparser.add_argument(
         "versions", nargs=argparse.REMAINDER, help="versions to generate checksums for"
@@ -118,3 +127,46 @@ def checksum(parser, args):
     print()
     print(version_lines)
     print()
+
+    if args.add_to_package:
+        filename = spack.repo.path.filename_for_package_name(pkg.name)
+        # Make sure we also have a newline after the last version
+        versions = [v + "\n" for v in version_lines.splitlines()]
+        versions.append("\n")
+        # We need to insert the versions in reversed order
+        versions.reverse()
+        versions.append("    # FIXME: Added by `spack checksum`\n")
+        version_line = None
+
+        with open(filename, "r") as f:
+            lines = f.readlines()
+            for i in range(len(lines)):
+                # Black is drunk, so this is what it looks like for now
+                # See https://github.com/psf/black/issues/2156 for more information
+                if lines[i].startswith("    # FIXME: Added by `spack checksum`") or lines[
+                    i
+                ].startswith("    version("):
+                    version_line = i
+                    break
+
+        if version_line is not None:
+            for v in versions:
+                lines.insert(version_line, v)
+
+            with open(filename, "w") as f:
+                f.writelines(lines)
+
+            msg = "opening editor to verify"
+
+            if not sys.stdout.isatty():
+                msg = "please verify"
+
+            tty.info(
+                "Added {0} new versions to {1}, "
+                "{2}.".format(len(versions) - 2, args.package, msg)
+            )
+
+            if sys.stdout.isatty():
+                editor(filename)
+        else:
+            tty.warn("Could not add new versions to {0}.".format(args.package))

lib/spack/spack/cmd/ci.py

@@ -6,7 +6,6 @@
 import json
 import os
 import shutil
-import sys
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
@@ -498,7 +497,7 @@ def ci_rebuild(args):
             bindist.download_single_spec(job_spec, build_cache_dir, mirror_url=matching_mirror)
 
         # Now we are done and successful
-        sys.exit(0)
+        return 0
 
     # Before beginning the install, if this is a "rebuild everything" pipeline, we
     # only want to keep the mirror being used by the current pipeline as it's binary
@@ -531,7 +530,6 @@ def ci_rebuild(args):
     slash_hash = "/{}".format(job_spec.dag_hash())
     deps_install_args = install_args
     root_install_args = install_args + [
-        "--no-add",
         "--keep-stage",
         "--only=package",
         "--use-buildcache=package:never,dependencies:only",
@@ -567,8 +565,6 @@ def ci_rebuild(args):
             "-o",
             "Makefile",
             "--use-buildcache=package:never,dependencies:only",
-            "--make-target-prefix",
-            "ci",
             slash_hash,  # limit to spec we're building
         ],
         [
@@ -585,7 +581,7 @@ def ci_rebuild(args):
             "SPACK_COLOR=always",
             "SPACK_INSTALL_FLAGS={}".format(args_to_string(deps_install_args)),
             "-j$(nproc)",
-            "ci/.install-deps/{}".format(job_spec.dag_hash()),
+            "install-deps/{}".format(job_spec.format("{name}-{version}-{hash}")),
         ],
         spack_cmd + ["install"] + root_install_args,
     ]

lib/spack/spack/cmd/compiler.py

@@ -8,8 +8,6 @@
 import argparse
 import sys
 
-from six import iteritems
-
 import llnl.util.tty as tty
 from llnl.util.lang import index_by
 from llnl.util.tty.colify import colify
@@ -138,13 +136,13 @@ def compiler_info(args):
                 print("\t\t%s = %s" % (cpath, getattr(c, cpath, None)))
             if c.flags:
                 print("\tflags:")
-                for flag, flag_value in iteritems(c.flags):
+                for flag, flag_value in c.flags.items():
                     print("\t\t%s = %s" % (flag, flag_value))
             if len(c.environment) != 0:
                 if len(c.environment.get("set", {})) != 0:
                     print("\tenvironment:")
                     print("\t    set:")
-                    for key, value in iteritems(c.environment["set"]):
+                    for key, value in c.environment["set"].items():
                         print("\t        %s = %s" % (key, value))
             if c.extra_rpaths:
                 print("\tExtra rpaths:")

lib/spack/spack/cmd/deactivate.py

@@ -1,96 +0,0 @@
-# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-import llnl.util.tty as tty
-
-import spack.cmd
-import spack.cmd.common.arguments as arguments
-import spack.environment as ev
-import spack.graph
-import spack.store
-from spack.filesystem_view import YamlFilesystemView
-
-description = "deactivate a package extension"
-section = "extensions"
-level = "long"
-
-
-def setup_parser(subparser):
-    subparser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        help="run deactivation even if spec is NOT currently activated",
-    )
-    subparser.add_argument("-v", "--view", metavar="VIEW", type=str, help="the view to operate on")
-    subparser.add_argument(
-        "-a",
-        "--all",
-        action="store_true",
-        help="deactivate all extensions of an extendable package, or "
-        "deactivate an extension AND its dependencies",
-    )
-    arguments.add_common_arguments(subparser, ["installed_spec"])
-
-
-def deactivate(parser, args):
-
-    tty.warn(
-        "spack deactivate is deprecated in favor of " "environments and will be removed in v0.19.0"
-    )
-
-    specs = spack.cmd.parse_specs(args.spec)
-    if len(specs) != 1:
-        tty.die("deactivate requires one spec.  %d given." % len(specs))
-
-    env = ev.active_environment()
-    spec = spack.cmd.disambiguate_spec(specs[0], env)
-    pkg = spec.package
-
-    if args.view:
-        target = args.view
-    elif pkg.is_extension:
-        target = pkg.extendee_spec.prefix
-    elif pkg.extendable:
-        target = spec.prefix
-
-    view = YamlFilesystemView(target, spack.store.layout)
-
-    if args.all:
-        if pkg.extendable:
-            tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec)
-            ext_pkgs = spack.store.db.activated_extensions_for(spec, view.extensions_layout)
-
-            for ext_pkg in ext_pkgs:
-                ext_pkg.spec.normalize()
-                if ext_pkg.is_activated(view):
-                    ext_pkg.do_deactivate(view, force=True)
-
-        elif pkg.is_extension:
-            if not args.force and not spec.package.is_activated(view):
-                tty.die("%s is not activated." % pkg.spec.short_spec)
-
-            tty.msg("Deactivating %s and all dependencies." % pkg.spec.short_spec)
-
-            nodes_in_topological_order = spack.graph.topological_sort(spec)
-            for espec in reversed(nodes_in_topological_order):
-                epkg = espec.package
-                if epkg.extends(pkg.extendee_spec):
-                    if epkg.is_activated(view) or args.force:
-                        epkg.do_deactivate(view, force=args.force)
-
-        else:
-            tty.die("spack deactivate --all requires an extendable package " "or an extension.")
-
-    else:
-        if not pkg.is_extension:
-            tty.die(
-                "spack deactivate requires an extension.", "Did you mean 'spack deactivate --all'?"
-            )
-
-        if not args.force and not spec.package.is_activated(view):
-            tty.die("Package %s is not activated." % spec.short_spec)
-
-        spec.package.do_deactivate(view, force=args.force)

lib/spack/spack/cmd/diff.py

@@ -46,6 +46,14 @@ def setup_parser(subparser):
     )
 
 
+def shift(asp_function):
+    """Transforms ``attr("foo", "bar")`` into ``foo("bar")``."""
+    if not asp_function.args:
+        raise ValueError(f"Can't shift ASP function with no arguments: {str(asp_function)}")
+    first, *rest = asp_function.args
+    return asp.AspFunction(first, rest)
+
+
 def compare_specs(a, b, to_string=False, color=None):
     """
     Generate a comparison, including diffs (for each side) and an intersection.
@@ -71,22 +79,24 @@ def compare_specs(a, b, to_string=False, color=None):
     # get facts for specs, making sure to include build dependencies of concrete
     # specs and to descend into dependency hashes so we include all facts.
     a_facts = set(
-        t
-        for t in setup.spec_clauses(
+        shift(func)
+        for func in setup.spec_clauses(
             a,
             body=True,
             expand_hashes=True,
             concrete_build_deps=True,
         )
+        if func.name == "attr"
     )
     b_facts = set(
-        t
-        for t in setup.spec_clauses(
+        shift(func)
+        for func in setup.spec_clauses(
             b,
             body=True,
             expand_hashes=True,
             concrete_build_deps=True,
         )
+        if func.name == "attr"
     )
 
     # We want to present them to the user as simple key: values

lib/spack/spack/cmd/env.py

@@ -4,13 +4,12 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import argparse
+import io
 import os
 import shutil
 import sys
 import tempfile
 
-import six
-
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 from llnl.util.tty.colify import colify
@@ -674,13 +673,16 @@ def build_cache_flag(self, depth):
         return ""
 
     def accept(self, node):
-        dag_hash = node.edge.spec.dag_hash()
+        fmt = "{name}-{version}-{hash}"
+        tgt = node.edge.spec.format(fmt)
         spec_str = node.edge.spec.format(
             "{name}{@version}{%compiler}{variants}{arch=architecture}"
         )
         buildcache_flag = self.build_cache_flag(node.depth)
-        prereqs = " ".join([self.target(dep.spec.dag_hash()) for dep in self.neighbors(node)])
-        self.adjacency_list.append((dag_hash, spec_str, buildcache_flag, prereqs))
+        prereqs = " ".join([self.target(dep.spec.format(fmt)) for dep in self.neighbors(node)])
+        self.adjacency_list.append(
+            (tgt, prereqs, node.edge.spec.dag_hash(), spec_str, buildcache_flag)
+        )
 
         # We already accepted this
         return True
@@ -691,6 +693,8 @@ def env_depfile(args):
     spack.cmd.require_active_env(cmd_name="env depfile")
     env = ev.active_environment()
 
+    # Special make targets are useful when including a makefile in another, and you
+    # need to "namespace" the targets to avoid conflicts.
     if args.make_target_prefix is None:
         target_prefix = os.path.join(env.env_subdir_path, "makedeps")
     else:
@@ -707,10 +711,10 @@ def get_target(name):
             return os.path.join(target_prefix, name)
 
     def get_install_target(name):
-        return os.path.join(target_prefix, ".install", name)
+        return os.path.join(target_prefix, "install", name)
 
     def get_install_deps_target(name):
-        return os.path.join(target_prefix, ".install-deps", name)
+        return os.path.join(target_prefix, "install-deps", name)
 
     # What things do we build when running make? By default, we build the
     # root specs. If specific specs are provided as input, we build those.
@@ -729,15 +733,24 @@ def get_install_deps_target(name):
     )
 
     # Root specs without deps are the prereqs for the environment target
-    root_install_targets = [get_install_target(h.dag_hash()) for h in roots]
+    root_install_targets = [get_install_target(h.format("{name}-{version}-{hash}")) for h in roots]
 
-    # Cleanable targets...
-    cleanable_targets = [get_install_target(h) for h, _, _, _ in make_targets.adjacency_list]
-    cleanable_targets.extend(
-        [get_install_deps_target(h) for h, _, _, _ in make_targets.adjacency_list]
-    )
+    # All install and install-deps targets
+    all_install_related_targets = []
 
-    buf = six.StringIO()
+    # Convenience shortcuts: ensure that `make install/pkg-version-hash` triggers
+    # <absolute path to env>/.spack-env/makedeps/install/pkg-version-hash in case
+    # we don't have a custom make target prefix.
+    phony_convenience_targets = []
+
+    for tgt, _, _, _, _ in make_targets.adjacency_list:
+        all_install_related_targets.append(get_install_target(tgt))
+        all_install_related_targets.append(get_install_deps_target(tgt))
+        if args.make_target_prefix is None:
+            phony_convenience_targets.append(os.path.join("install", tgt))
+            phony_convenience_targets.append(os.path.join("install-deps", tgt))
+
+    buf = io.StringIO()
 
     template = spack.tengine.make_environment().get_template(os.path.join("depfile", "Makefile"))
 
@@ -746,15 +759,17 @@ def get_install_deps_target(name):
             "all_target": get_target("all"),
             "env_target": get_target("env"),
             "clean_target": get_target("clean"),
-            "cleanable_targets": " ".join(cleanable_targets),
+            "all_install_related_targets": " ".join(all_install_related_targets),
             "root_install_targets": " ".join(root_install_targets),
             "dirs_target": get_target("dirs"),
             "environment": env.path,
-            "install_target": get_target(".install"),
-            "install_deps_target": get_target(".install-deps"),
+            "install_target": get_target("install"),
+            "install_deps_target": get_target("install-deps"),
             "any_hash_target": get_target("%"),
             "jobserver_support": "+" if args.jobserver else "",
             "adjacency_list": make_targets.adjacency_list,
+            "phony_convenience_targets": " ".join(phony_convenience_targets),
+            "target_prefix": target_prefix,
         }
     )
 

lib/spack/spack/cmd/extensions.py

@@ -14,7 +14,6 @@
 import spack.environment as ev
 import spack.repo
 import spack.store
-from spack.filesystem_view import YamlFilesystemView
 
 description = "list extensions for package"
 section = "extensions"
@@ -38,10 +37,9 @@ def setup_parser(subparser):
         "--show",
         action="store",
         default="all",
-        choices=("packages", "installed", "activated", "all"),
+        choices=("packages", "installed", "all"),
         help="show only part of output",
     )
-    subparser.add_argument("-v", "--view", metavar="VIEW", type=str, help="the view to operate on")
 
     subparser.add_argument(
         "spec",
@@ -91,13 +89,6 @@ def extensions(parser, args):
             tty.msg("%d extensions:" % len(extensions))
             colify(ext.name for ext in extensions)
 
-    if args.view:
-        target = args.view
-    else:
-        target = spec.prefix
-
-    view = YamlFilesystemView(target, spack.store.layout)
-
     if args.show in ("installed", "all"):
         # List specs of installed extensions.
         installed = [s.spec for s in spack.store.db.installed_extensions_for(spec)]
@@ -109,14 +100,3 @@ def extensions(parser, args):
         else:
             tty.msg("%d installed:" % len(installed))
             cmd.display_specs(installed, args)
-
-    if args.show in ("activated", "all"):
-        # List specs of activated extensions.
-        activated = view.extensions_layout.extension_map(spec)
-        if args.show == "all":
-            print
-        if not activated:
-            tty.msg("None activated.")
-        else:
-            tty.msg("%d activated:" % len(activated))
-            cmd.display_specs(activated.values(), args)

lib/spack/spack/cmd/find.py

@@ -140,13 +140,6 @@ def setup_parser(subparser):
 
     subparser.add_argument("--start-date", help="earliest date of installation [YYYY-MM-DD]")
     subparser.add_argument("--end-date", help="latest date of installation [YYYY-MM-DD]")
-    subparser.add_argument(
-        "-b",
-        "--bootstrap",
-        action="store_true",
-        help="show software in the internal bootstrap store",
-    )
-
     arguments.add_common_arguments(subparser, ["constraint"])
 
 
@@ -251,23 +244,6 @@ def display_env(env, args, decorator, results):
 
 
 def find(parser, args):
-    if args.bootstrap:
-        tty.warn(
-            "`spack find --bootstrap` is deprecated and will be removed in v0.19.",
-            "Use `spack --bootstrap find` instead.",
-        )
-
-    if args.bootstrap:
-        bootstrap_store_path = spack.bootstrap.store_path()
-        with spack.bootstrap.ensure_bootstrap_configuration():
-            msg = 'Showing internal bootstrap store at "{0}"'
-            tty.msg(msg.format(bootstrap_store_path))
-            _find(parser, args)
-        return
-    _find(parser, args)
-
-
-def _find(parser, args):
     q_args = query_arguments(args)
     results = args.specs(**q_args)
 

lib/spack/spack/cmd/gc.py

@@ -43,4 +43,4 @@ def gc(parser, args):
     if not args.yes_to_all:
         spack.cmd.uninstall.confirm_removal(specs)
 
-    spack.cmd.uninstall.do_uninstall(None, specs, force=False)
+    spack.cmd.uninstall.do_uninstall(specs, force=False)

lib/spack/spack/cmd/info.py

@@ -7,8 +7,7 @@
 
 import inspect
 import textwrap
-
-from six.moves import zip_longest
+from itertools import zip_longest
 
 import llnl.util.tty as tty
 import llnl.util.tty.color as color
@@ -242,8 +241,8 @@ def print_tests(pkg):
     # So the presence of a callback in Spack does not necessarily correspond
     # to the actual presence of built-time tests for a package.
     for callbacks, phase in [
-        (pkg.build_time_test_callbacks, "Build"),
-        (pkg.install_time_test_callbacks, "Install"),
+        (getattr(pkg, "build_time_test_callbacks", None), "Build"),
+        (getattr(pkg, "install_time_test_callbacks", None), "Install"),
     ]:
         color.cprint("")
         color.cprint(section_title("Available {0} Phase Test Methods:".format(phase)))

lib/spack/spack/cmd/install.py

@@ -193,14 +193,22 @@ def setup_parser(subparser):
         default=False,
         help="(with environment) only install already concretized specs",
     )
-    subparser.add_argument(
-        "--no-add",
+
+    updateenv_group = subparser.add_mutually_exclusive_group()
+    updateenv_group.add_argument(
+        "--add",
         action="store_true",
         default=False,
-        help="""(with environment) partially install an environment, limiting
-to concrete specs in the environment matching the arguments.
-Non-roots remain installed implicitly.""",
+        help="""(with environment) add spec to the environment as a root.""",
     )
+    updateenv_group.add_argument(
+        "--no-add",
+        action="store_false",
+        dest="add",
+        help="""(with environment) do not add spec to the environment as a
+root (the default behavior).""",
+    )
+
     subparser.add_argument(
         "-f",
         "--file",
@@ -289,11 +297,12 @@ def install_specs_inside_environment(specs, install_kwargs, cli_args):
         # the matches.  Getting to this point means there were either
         # no matches or exactly one match.
 
-        if not m_spec and cli_args.no_add:
+        if not m_spec and not cli_args.add:
             msg = (
-                "You asked to install {0} without adding it (--no-add), but no such spec "
-                "exists in environment"
-            ).format(abstract.name)
+                "Cannot install '{0}' because it is not in the current environment."
+                " You can add it to the environment with 'spack add {0}', or as part"
+                " of the install command with 'spack install --add {0}'"
+            ).format(str(abstract))
             tty.die(msg)
 
         if not m_spec:
@@ -303,14 +312,16 @@ def install_specs_inside_environment(specs, install_kwargs, cli_args):
 
         tty.debug("exactly one match for {0} in env -> {1}".format(m_spec.name, m_spec.dag_hash()))
 
-        if m_spec in env.roots() or cli_args.no_add:
-            # either the single match is a root spec (and --no-add is
-            # the default for roots) or --no-add was stated explicitly
+        if m_spec in env.roots() or not cli_args.add:
+            # either the single match is a root spec (in which case
+            # the spec is not added to the env again), or the user did
+            # not specify --add (in which case it is assumed we are
+            # installing already-concretized specs in the env)
             tty.debug("just install {0}".format(m_spec.name))
             specs_to_install.append(m_spec)
         else:
             # the single match is not a root (i.e. it's a dependency),
-            # and --no-add was not specified, so we'll add it as a
+            # and --add was specified, so we'll add it as a
             # root before installing
             tty.debug("add {0} then install it".format(m_spec.name))
             specs_to_add.append((abstract, concrete))

lib/spack/spack/cmd/list.py

@@ -12,6 +12,7 @@
 import os
 import re
 import sys
+from html import escape
 
 import llnl.util.tty as tty
 from llnl.util.tty.colify import colify
@@ -21,11 +22,6 @@
 import spack.repo
 from spack.version import VersionList
 
-if sys.version_info > (3, 1):
-    from html import escape  # novm
-else:
-    from cgi import escape
-
 description = "list and search available packages"
 section = "basic"
 level = "short"

lib/spack/spack/cmd/location.py

@@ -9,6 +9,7 @@
 
 import llnl.util.tty as tty
 
+import spack.builder
 import spack.cmd
 import spack.cmd.common.arguments as arguments
 import spack.environment as ev
@@ -134,6 +135,7 @@ def location(parser, args):
     # Either concretize or filter from already concretized environment
     spec = spack.cmd.matching_spec_from_env(spec)
     pkg = spec.package
+    builder = spack.builder.create(pkg)
 
     if args.stage_dir:
         print(pkg.stage.path)
@@ -141,10 +143,10 @@ def location(parser, args):
 
     if args.build_dir:
         # Out of source builds have build_directory defined
-        if hasattr(pkg, "build_directory"):
+        if hasattr(builder, "build_directory"):
             # build_directory can be either absolute or relative to the stage path
             # in either case os.path.join makes it absolute
-            print(os.path.normpath(os.path.join(pkg.stage.path, pkg.build_directory)))
+            print(os.path.normpath(os.path.join(pkg.stage.path, builder.build_directory)))
             return
 
         # Otherwise assume in-source builds

lib/spack/spack/cmd/providers.py

@@ -3,10 +3,9 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import io
 import sys
 
-import six
-
 import llnl.util.tty.colify as colify
 
 import spack.cmd
@@ -29,7 +28,7 @@ def setup_parser(subparser):
 def providers(parser, args):
     valid_virtuals = sorted(spack.repo.path.provider_index.providers.keys())
 
-    buffer = six.StringIO()
+    buffer = io.StringIO()
     isatty = sys.stdout.isatty()
     if isatty:
         buffer.write("Virtual packages:\n")

lib/spack/spack/cmd/style.py

@@ -9,6 +9,7 @@
 import os
 import re
 import sys
+from itertools import zip_longest
 
 import llnl.util.tty as tty
 import llnl.util.tty.color as color
@@ -18,14 +19,6 @@
 import spack.paths
 from spack.util.executable import which
 
-if sys.version_info < (3, 0):
-    from itertools import izip_longest  # novm
-
-    zip_longest = izip_longest
-else:
-    from itertools import zip_longest  # novm
-
-
 description = "runs source code style checks on spack"
 section = "developer"
 level = "long"
@@ -267,7 +260,7 @@ def run_flake8(flake8_cmd, file_list, args):
             "--config=%s" % os.path.join(spack.paths.prefix, ".flake8"),
             *chunk,
             fail_on_error=False,
-            output=str
+            output=str,
         )
         returncode |= flake8_cmd.returncode
 
@@ -375,14 +368,6 @@ def run_black(black_cmd, file_list, args):
         packed_args = black_args + tuple(chunk)
         output = black_cmd(*packed_args, fail_on_error=False, output=str, error=str)
         returncode |= black_cmd.returncode
-
-        # ignore Python 2.7 deprecation error because we already know it's deprecated.
-        output = "\n".join(
-            line
-            for line in output.split("\n")
-            if "DEPRECATION: Python 2 support will be removed" not in line
-        )
-
         rewrite_and_print_output(output, args, pat, replacement)
 
     print_tool_result("black", returncode)
@@ -400,10 +385,6 @@ def validate_toolset(arg_value):
 
 
 def style(parser, args):
-    # ensure python version is new enough
-    if sys.version_info < (3, 6):
-        tty.die("spack style requires Python 3.6 or later.")
-
     # save initial working directory for relativizing paths later
     args.initial_working_dir = os.getcwd()
 

lib/spack/spack/cmd/tags.py

@@ -2,11 +2,9 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
+import io
 import sys
 
-import six
-
 import llnl.util.tty as tty
 import llnl.util.tty.colify as colify
 
@@ -20,7 +18,7 @@
 
 
 def report_tags(category, tags):
-    buffer = six.StringIO()
+    buffer = io.StringIO()
     isatty = sys.stdout.isatty()
 
     if isatty:
@@ -88,7 +86,7 @@ def tags(parser, args):
         return
 
     # Report packages associated with tags
-    buffer = six.StringIO()
+    buffer = io.StringIO()
     isatty = sys.stdout.isatty()
 
     tags = args.tag if args.tag else available_tags

lib/spack/spack/cmd/tutorial.py

@@ -11,6 +11,7 @@
 import llnl.util.tty as tty
 from llnl.util.filesystem import working_dir
 
+import spack
 import spack.cmd.common.arguments as arguments
 import spack.config
 import spack.paths
@@ -24,7 +25,7 @@
 
 
 # tutorial configuration parameters
-tutorial_branch = "releases/v0.18"
+tutorial_branch = "releases/v0.19"
 tutorial_mirror = "file:///mirror"
 tutorial_key = os.path.join(spack.paths.share_path, "keys", "tutorial.pub")
 

lib/spack/spack/cmd/uninstall.py

@@ -5,7 +5,6 @@
 
 from __future__ import print_function
 
-import itertools
 import sys
 
 from llnl.util import tty
@@ -18,6 +17,7 @@
 import spack.package_base
 import spack.repo
 import spack.store
+import spack.traverse as traverse
 from spack.database import InstallStatuses
 
 description = "remove installed packages"
@@ -61,6 +61,13 @@ def setup_parser(subparser):
         dest="force",
         help="remove regardless of whether other packages or environments " "depend on this one",
     )
+    subparser.add_argument(
+        "--remove",
+        action="store_true",
+        dest="remove",
+        help="if in an environment, then the spec should also be removed from "
+        "the environment description",
+    )
     arguments.add_common_arguments(
         subparser, ["recurse_dependents", "yes_to_all", "installed_specs"]
     )
@@ -134,31 +141,43 @@ def installed_dependents(specs, env):
         env (spack.environment.Environment or None): the active environment, or None
 
     Returns:
-        tuple: two mappings: one from specs to their dependent environments in the
-        active environment (or global scope if there is no environment), and one from
-        specs to their dependents in *inactive* environments (empty if there is no
-        environment
+        tuple: two mappings: one from specs to their dependent installs in the
+        active environment, and one from specs to dependent installs outside of
+        the active environment.
+
+        Every installed dependent spec is listed once.
+
+        If there is not current active environment, the first mapping will be
+        empty.
     """
     active_dpts = {}
-    inactive_dpts = {}
+    outside_dpts = {}
 
     env_hashes = set(env.all_hashes()) if env else set()
 
-    all_specs_in_db = spack.store.db.query()
+    # Ensure we stop traversal at input specs.
+    visited = set(s.dag_hash() for s in specs)
 
     for spec in specs:
-        installed = [x for x in all_specs_in_db if spec in x]
+        for dpt in traverse.traverse_nodes(
+            spec.dependents(deptype="all"),
+            direction="parents",
+            visited=visited,
+            deptype="all",
+            root=True,
+            key=lambda s: s.dag_hash(),
+        ):
+            hash = dpt.dag_hash()
+            # Ensure that all the specs we get are installed
+            record = spack.store.db.query_local_by_spec_hash(hash)
+            if record is None or not record.installed:
+                continue
+            if hash in env_hashes:
+                active_dpts.setdefault(spec, set()).add(dpt)
+            else:
+                outside_dpts.setdefault(spec, set()).add(dpt)
 
-        # separate installed dependents into dpts in this environment and
-        # dpts that are outside this environment
-        for dpt in installed:
-            if dpt not in specs:
-                if not env or dpt.dag_hash() in env_hashes:
-                    active_dpts.setdefault(spec, set()).add(dpt)
-                else:
-                    inactive_dpts.setdefault(spec, set()).add(dpt)
-
-    return active_dpts, inactive_dpts
+    return active_dpts, outside_dpts
 
 
 def dependent_environments(specs):
@@ -211,82 +230,83 @@ def _remove_from_env(spec, env):
         pass  # ignore non-root specs
 
 
-def do_uninstall(env, specs, force):
-    """Uninstalls all the specs in a list.
+def do_uninstall(specs, force=False):
+    # TODO: get rid of the call-sites that use this function,
+    # so that we don't have to do a dance of list -> set -> list -> set
+    hashes_to_remove = set(s.dag_hash() for s in specs)
 
-    Args:
-        env (spack.environment.Environment or None): active environment, or ``None``
-            if there is not one
-        specs (list): list of specs to be uninstalled
-        force (bool): force uninstallation (boolean)
-    """
-    packages = []
-    for item in specs:
-        try:
-            # should work if package is known to spack
-            packages.append(item.package)
-        except spack.repo.UnknownEntityError:
-            # The package.py file has gone away -- but still
-            # want to uninstall.
-            spack.package_base.PackageBase.uninstall_by_spec(item, force=True)
-
-    # A package is ready to be uninstalled when nothing else references it,
-    # unless we are requested to force uninstall it.
-    def is_ready(dag_hash):
-        if force:
-            return True
-
-        _, record = spack.store.db.query_by_spec_hash(dag_hash)
-        if not record.ref_count:
-            return True
-
-        # If this spec is only used as a build dependency, we can uninstall
-        return all(
-            dspec.deptypes == ("build",) or not dspec.parent.installed
-            for dspec in record.spec.edges_from_dependents()
-        )
-
-    while packages:
-        ready = [x for x in packages if is_ready(x.spec.dag_hash())]
-        if not ready:
-            msg = (
-                "unexpected error [cannot proceed uninstalling specs with"
-                " remaining link or run dependents {0}]"
-            )
-            msg = msg.format(", ".join(x.name for x in packages))
-            raise spack.error.SpackError(msg)
-
-        packages = [x for x in packages if x not in ready]
-        for item in ready:
-            item.do_uninstall(force=force)
+    for s in traverse.traverse_nodes(
+        specs,
+        order="topo",
+        direction="children",
+        root=True,
+        cover="nodes",
+        deptype="all",
+    ):
+        if s.dag_hash() in hashes_to_remove:
+            spack.package_base.PackageBase.uninstall_by_spec(s, force=force)
 
 
 def get_uninstall_list(args, specs, env):
-    # Gets the list of installed specs that match the ones give via cli
+    """Returns uninstall_list and remove_list: these may overlap (some things
+    may be both uninstalled and removed from the current environment).
+
+    It is assumed we are in an environment if --remove is specified (this
+    method raises an exception otherwise).
+
+    uninstall_list is topologically sorted: dependents come before
+    dependencies (so if a user uninstalls specs in the order provided,
+    the dependents will always be uninstalled first).
+    """
+    if args.remove and not env:
+        raise ValueError("Can only use --remove when in an environment")
+
+    # Gets the list of installed specs that match the ones given via cli
     # args.all takes care of the case where '-a' is given in the cli
-    uninstall_list = find_matching_specs(env, specs, args.all, args.force, args.origin)
+    base_uninstall_specs = set(find_matching_specs(env, specs, args.all, args.force))
 
-    # Takes care of '-R'
-    active_dpts, inactive_dpts = installed_dependents(uninstall_list, env)
+    active_dpts, outside_dpts = installed_dependents(base_uninstall_specs, env)
+    # It will be useful to track the unified set of specs with dependents, as
+    # well as to separately track specs in the current env with dependents
+    spec_to_dpts = {}
+    for spec, dpts in active_dpts.items():
+        spec_to_dpts[spec] = list(dpts)
+    for spec, dpts in outside_dpts.items():
+        if spec in spec_to_dpts:
+            spec_to_dpts[spec].extend(dpts)
+        else:
+            spec_to_dpts[spec] = list(dpts)
 
-    # if we are in the global scope, we complain if you try to remove a
-    # spec that's in an environment.  If we're in an environment, we'll
-    # just *remove* it from the environment, so we ignore this
-    # error when *in* an environment
-    spec_envs = dependent_environments(uninstall_list)
-    spec_envs = inactive_dependent_environments(spec_envs)
+    all_uninstall_specs = set(base_uninstall_specs)
+    if args.dependents:
+        for spec, lst in active_dpts.items():
+            all_uninstall_specs.update(lst)
+        for spec, lst in outside_dpts.items():
+            all_uninstall_specs.update(lst)
 
-    # Process spec_dependents and update uninstall_list
-    has_error = not args.force and (
-        (active_dpts and not args.dependents)  # dependents in the current env
-        or (not env and spec_envs)  # there are environments that need specs
+    # For each spec that we intend to uninstall, this tracks the set of
+    # environments outside the current active environment which depend on the
+    # spec. There may be environments not managed directly with Spack: such
+    # environments would not be included here.
+    spec_to_other_envs = inactive_dependent_environments(
+        dependent_environments(all_uninstall_specs)
+    )
+
+    has_error = not args.force and (
+        # There are dependents in the current env and we didn't ask to remove
+        # dependents
+        (spec_to_dpts and not args.dependents)
+        # An environment different than the current env (if any) depends on
+        # one or more of the specs to be uninstalled. There may also be
+        # packages in those envs which depend on the base set of packages
+        # to uninstall, but this covers that scenario.
+        or (not args.remove and spec_to_other_envs)
     )
 
-    # say why each problem spec is needed
     if has_error:
-        specs = set(active_dpts)
-        if not env:
-            specs.update(set(spec_envs))  # environments depend on this
+        # say why each problem spec is needed
+        specs = set(spec_to_dpts)
+        specs.update(set(spec_to_other_envs))  # environments depend on this
 
         for i, spec in enumerate(sorted(specs)):
             # space out blocks of reasons
@@ -296,65 +316,85 @@ def get_uninstall_list(args, specs, env):
             spec_format = "{name}{@version}{%compiler}{/hash:7}"
             tty.info("Will not uninstall %s" % spec.cformat(spec_format), format="*r")
 
-            dependents = active_dpts.get(spec)
-            if dependents:
+            dependents = spec_to_dpts.get(spec)
+            if dependents and not args.dependents:
                 print("The following packages depend on it:")
                 spack.cmd.display_specs(dependents, **display_args)
 
-            if not env:
-                envs = spec_envs.get(spec)
-                if envs:
-                    print("It is used by the following environments:")
-                    colify([e.name for e in envs], indent=4)
+            envs = spec_to_other_envs.get(spec)
+            if envs:
+                if env:
+                    env_context_qualifier = " other"
+                else:
+                    env_context_qualifier = ""
+                print("It is used by the following{0} environments:".format(env_context_qualifier))
+                colify([e.name for e in envs], indent=4)
 
         msgs = []
-        if active_dpts:
+        if spec_to_dpts and not args.dependents:
             msgs.append("use `spack uninstall --dependents` to remove dependents too")
-        if spec_envs:
+        if spec_to_other_envs:
             msgs.append("use `spack env remove` to remove from environments")
         print()
         tty.die("There are still dependents.", *msgs)
 
-    elif args.dependents:
-        for spec, lst in active_dpts.items():
-            uninstall_list.extend(lst)
-        uninstall_list = list(set(uninstall_list))
+    # If we are in an environment, this will track specs in this environment
+    # which should only be removed from the environment rather than uninstalled
+    remove_only = set()
+    if args.remove and not args.force:
+        remove_only.update(spec_to_other_envs)
+    if remove_only:
+        tty.info(
+            "The following specs will be removed but not uninstalled because"
+            " they are also used by another environment: {speclist}".format(
+                speclist=", ".join(x.name for x in remove_only)
+            )
+        )
 
-    # only force-remove (don't completely uninstall) specs that still
-    # have external dependent envs or pkgs
-    removes = set(inactive_dpts)
-    if env:
-        removes.update(spec_envs)
+    # Compute the set of specs that should be removed from the current env.
+    # This may overlap (some specs may be uninstalled and also removed from
+    # the current environment).
+    if args.remove:
+        remove_specs = set(base_uninstall_specs)
+        if args.dependents:
+            # Any spec matched from the cli, or dependent of, should be removed
+            # from the environment
+            for spec, lst in active_dpts.items():
+                remove_specs.update(lst)
+    else:
+        remove_specs = set()
 
-    # remove anything in removes from the uninstall list
-    uninstall_list = set(uninstall_list) - removes
+    all_uninstall_specs -= remove_only
+    # Inefficient topological sort: uninstall dependents before dependencies
+    all_uninstall_specs = sorted(
+        all_uninstall_specs, key=lambda x: sum(1 for i in x.traverse()), reverse=True
+    )
 
-    return uninstall_list, removes
+    return list(all_uninstall_specs), list(remove_specs)
 
 
 def uninstall_specs(args, specs):
     env = ev.active_environment()
 
     uninstall_list, remove_list = get_uninstall_list(args, specs, env)
-    anything_to_do = set(uninstall_list).union(set(remove_list))
 
-    if not anything_to_do:
+    if not uninstall_list:
         tty.warn("There are no package to uninstall.")
         return
 
     if not args.yes_to_all:
-        confirm_removal(anything_to_do)
+        confirm_removal(uninstall_list)
+
+    # Uninstall everything on the list
+    do_uninstall(uninstall_list, args.force)
 
     if env:
-        # Remove all the specs that are supposed to be uninstalled or just
-        # removed.
         with env.write_transaction():
-            for spec in itertools.chain(remove_list, uninstall_list):
+            for spec in remove_list:
                 _remove_from_env(spec, env)
             env.write()
 
-    # Uninstall everything on the list
-    do_uninstall(env, uninstall_list, args.force)
+        env.regenerate_views()
 
 
 def confirm_removal(specs):

lib/spack/spack/cmd/unit_test.py

@@ -7,6 +7,7 @@
 
 import argparse
 import collections
+import io
 import os.path
 import re
 import sys
@@ -16,8 +17,6 @@
 except ImportError:
     pytest = None  # type: ignore
 
-from six import StringIO
-
 import llnl.util.filesystem
 import llnl.util.tty.color as color
 from llnl.util.tty.colify import colify
@@ -126,7 +125,7 @@ def colorize(c, prefix):
 
     old_output = sys.stdout
     try:
-        sys.stdout = output = StringIO()
+        sys.stdout = output = io.StringIO()
         pytest.main(["--collect-only"] + extra_args)
     finally:
         sys.stdout = old_output

lib/spack/spack/cmd/url.py

@@ -5,10 +5,9 @@
 
 from __future__ import division, print_function
 
+import urllib.parse
 from collections import defaultdict
 
-import six.moves.urllib.parse as urllib_parse
-
 import llnl.util.tty.color as color
 from llnl.util import tty
 
@@ -323,7 +322,7 @@ def add(self, pkg_name, fetcher):
                     md5_hashes[pkg_name].append(fetcher.url)
 
                 # parse out the URL scheme (https/http/ftp/etc.)
-                urlinfo = urllib_parse.urlparse(fetcher.url)
+                urlinfo = urllib.parse.urlparse(fetcher.url)
                 self.schemes[urlinfo.scheme] += 1
 
                 if urlinfo.scheme == "http":

lib/spack/spack/compilers/__init__.py

@@ -12,8 +12,6 @@
 import os
 from typing import Dict  # novm
 
-import six
-
 import archspec.cpu
 
 import llnl.util.filesystem as fs
@@ -49,12 +47,26 @@
     "clang": "llvm+clang",
     "oneapi": "intel-oneapi-compilers",
     "rocmcc": "llvm-amdgpu",
+    "intel@2020:": "intel-oneapi-compilers-classic",
+}
+
+# TODO: generating this from the previous dict causes docs errors
+package_name_to_compiler_name = {
+    "llvm": "clang",
+    "intel-oneapi-compilers": "oneapi",
+    "llvm-amdgpu": "rocmcc",
+    "intel-oneapi-compilers-classic": "intel",
 }
 
 
 def pkg_spec_for_compiler(cspec):
     """Return the spec of the package that provides the compiler."""
-    spec_str = "%s@%s" % (_compiler_to_pkg.get(cspec.name, cspec.name), cspec.versions)
+    for spec, package in _compiler_to_pkg.items():
+        if cspec.satisfies(spec):
+            spec_str = "%s@%s" % (package, cspec.versions)
+            break
+    else:
+        spec_str = str(cspec)
     return spack.spec.Spec(spec_str)
 
 
@@ -413,7 +425,7 @@ def compiler_from_dict(items):
         environment,
         extra_rpaths,
         enable_implicit_rpaths=implicit_rpaths,
-        **compiler_flags
+        **compiler_flags,
     )
 
 
@@ -663,18 +675,18 @@ def _default(fn_args):
 
         try:
             version = callback(path)
-            if version and six.text_type(version).strip() and version != "unknown":
+            if version and str(version).strip() and version != "unknown":
                 value = fn_args._replace(id=compiler_id._replace(version=version))
                 return value, None
 
             error = "Couldn't get version for compiler {0}".format(path)
         except spack.util.executable.ProcessError as e:
-            error = "Couldn't get version for compiler {0}\n".format(path) + six.text_type(e)
+            error = "Couldn't get version for compiler {0}\n".format(path) + str(e)
         except Exception as e:
             # Catching "Exception" here is fine because it just
             # means something went wrong running a candidate executable.
             error = "Error while executing candidate compiler {0}" "\n{1}: {2}".format(
-                path, e.__class__.__name__, six.text_type(e)
+                path, e.__class__.__name__, str(e)
             )
         return None, error
 

lib/spack/spack/compilers/msvc.py

@@ -10,10 +10,13 @@
 from distutils.version import StrictVersion
 from typing import Dict, List, Set  # novm
 
+import spack.compiler
 import spack.operating_systems.windows_os
+import spack.platforms
 import spack.util.executable
 from spack.compiler import Compiler
 from spack.error import SpackError
+from spack.version import Version
 
 avail_fc_version = set()  # type: Set[str]
 fc_path = dict()  # type: Dict[str, str]
@@ -39,10 +42,10 @@ def get_valid_fortran_pth(comp_ver):
 
 class Msvc(Compiler):
     # Subclasses use possible names of C compiler
-    cc_names = ["cl.exe"]
+    cc_names = ["cl.exe"]  # type: List[str]
 
     # Subclasses use possible names of C++ compiler
-    cxx_names = ["cl.exe"]
+    cxx_names = ["cl.exe"]  # type: List[str]
 
     # Subclasses use possible names of Fortran 77 compiler
     f77_names = ["ifx.exe"]  # type: List[str]
@@ -91,45 +94,67 @@ def __init__(self, *args, **kwargs):
 
     @property
     def msvc_version(self):
-        ver = re.search(Msvc.version_regex, self.cc).group(1)
-        ver = "".join(ver.split(".")[:2])[:-1]
+        """This is the VCToolset version *NOT* the actual version of the cl compiler
+        For CL version, query `Msvc.cl_version`"""
+        return Version(re.search(Msvc.version_regex, self.cc).group(1))
+
+    @property
+    def short_msvc_version(self):
+        """
+        This is the shorthand VCToolset version of form
+        MSVC<short-ver> *NOT* the full version, for that see
+        Msvc.msvc_version
+        """
+        ver = self.msvc_version[:2].joined.string[:3]
         return "MSVC" + ver
 
+    @property
+    def cl_version(self):
+        """Cl toolset version"""
+        return spack.compiler.get_compiler_version_output(self.cc)
+
     def setup_custom_environment(self, pkg, env):
         """Set environment variables for MSVC using the
         Microsoft-provided script."""
-        if sys.version_info[:2] > (2, 6):
-            # Set the build environment variables for spack. Just using
-            # subprocess.call() doesn't work since that operates in its own
-            # environment which is destroyed (along with the adjusted variables)
-            # once the process terminates. So go the long way around: examine
-            # output, sort into dictionary, use that to make the build
-            # environment.
-            out = subprocess.check_output(  # novermin
-                'cmd /u /c "{}" {} && set'.format(self.setvarsfile, "amd64"),
-                stderr=subprocess.STDOUT,
-            )
-            if sys.version_info[0] >= 3:
-                out = out.decode("utf-16le", errors="replace")  # novermin
+        # Set the build environment variables for spack. Just using
+        # subprocess.call() doesn't work since that operates in its own
+        # environment which is destroyed (along with the adjusted variables)
+        # once the process terminates. So go the long way around: examine
+        # output, sort into dictionary, use that to make the build
+        # environment.
 
-            int_env = dict(
-                (key.lower(), value)
-                for key, _, value in (line.partition("=") for line in out.splitlines())
-                if key and value
-            )
+        # get current platform architecture and format for vcvars argument
+        arch = spack.platforms.real_host().default.lower()
+        arch = arch.replace("-", "_")
+        # vcvars can target specific sdk versions, force it to pick up concretized sdk
+        # version, if needed by spec
+        sdk_ver = "" if "win-sdk" not in pkg.spec else pkg.spec["win-sdk"].version.string + ".0"
+        # provide vcvars with msvc version selected by concretization,
+        # not whatever it happens to pick up on the system (highest available version)
+        out = subprocess.check_output(  # novermin
+            'cmd /u /c "{}" {} {} {} && set'.format(
+                self.setvarsfile, arch, sdk_ver, "-vcvars_ver=%s" % self.msvc_version
+            ),
+            stderr=subprocess.STDOUT,
+        )
+        if sys.version_info[0] >= 3:
+            out = out.decode("utf-16le", errors="replace")  # novermin
 
-            if "path" in int_env:
-                env.set_path("PATH", int_env["path"].split(";"))
-            env.set_path("INCLUDE", int_env.get("include", "").split(";"))
-            env.set_path("LIB", int_env.get("lib", "").split(";"))
+        int_env = dict(
+            (key.lower(), value)
+            for key, _, value in (line.partition("=") for line in out.splitlines())
+            if key and value
+        )
 
-            env.set("CC", self.cc)
-            env.set("CXX", self.cxx)
-            env.set("FC", self.fc)
-            env.set("F77", self.f77)
-        else:
-            # Should not this be an exception?
-            print("Cannot pull msvc compiler information in Python 2.6 or below")
+        if "path" in int_env:
+            env.set_path("PATH", int_env["path"].split(";"))
+        env.set_path("INCLUDE", int_env.get("include", "").split(";"))
+        env.set_path("LIB", int_env.get("lib", "").split(";"))
+
+        env.set("CC", self.cc)
+        env.set("CXX", self.cxx)
+        env.set("FC", self.fc)
+        env.set("F77", self.f77)
 
     @classmethod
     def fc_version(cls, fc):

lib/spack/spack/config.py

@@ -39,9 +39,7 @@
 from typing import List  # novm
 
 import ruamel.yaml as yaml
-import six
 from ruamel.yaml.error import MarkedYAMLError
-from six import iteritems
 
 import llnl.util.lang
 import llnl.util.tty as tty
@@ -358,7 +356,7 @@ def clear(self):
     def _process_dict_keyname_overrides(data):
         """Turn a trailing `:' in a key name into an override attribute."""
         result = {}
-        for sk, sv in iteritems(data):
+        for sk, sv in data.items():
             if sk.endswith(":"):
                 key = syaml.syaml_str(sk[:-1])
                 key.override = True
@@ -739,7 +737,7 @@ def override(path_or_scope, value=None):
 
 #: configuration scopes added on the command line
 #: set by ``spack.main.main()``.
-command_line_scopes = []  # type: List[str]
+command_line_scopes: List[str] = []
 
 
 def _add_platform_scope(cfg, scope_type, name, path):
@@ -973,7 +971,7 @@ def validate(data, schema, filename=None):
             line_number = e.instance.lc.line + 1
         else:
             line_number = None
-        raise six.raise_from(ConfigFormatError(e, data, filename, line_number), e)
+        raise ConfigFormatError(e, data, filename, line_number) from e
     # return the validated data so that we can access the raw data
     # mostly relevant for environments
     return test_data
@@ -991,7 +989,7 @@ def read_config_file(filename, schema=None):
 
     if not os.path.exists(filename):
         # Ignore nonexistent files.
-        tty.debug("Skipping nonexistent config path {0}".format(filename))
+        tty.debug("Skipping nonexistent config path {0}".format(filename), level=3)
         return None
 
     elif not os.path.isfile(filename):
@@ -1140,7 +1138,7 @@ def they_are(t):
         # come *before* dest in OrderdDicts
         dest_keys = [dk for dk in dest.keys() if dk not in source]
 
-        for sk, sv in iteritems(source):
+        for sk, sv in source.items():
             # always remove the dest items. Python dicts do not overwrite
             # keys on insert, so this ensures that source keys are copied
             # into dest along with mark provenance (i.e., file/line info).

lib/spack/spack/cray_manifest.py

@@ -4,11 +4,9 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import json
-import sys
 
 import jsonschema
 import jsonschema.exceptions
-import six
 
 import llnl.util.tty as tty
 
@@ -98,7 +96,7 @@ def spec_from_entry(entry):
                 continue
 
             # Value could be a list (of strings), boolean, or string
-            if isinstance(value, six.string_types):
+            if isinstance(value, str):
                 variant_strs.append("{0}={1}".format(name, value))
             else:
                 try:
@@ -163,21 +161,14 @@ def entries_to_specs(entries):
 
 
 def read(path, apply_updates):
-    if sys.version_info >= (3, 0):
-        decode_exception_type = json.decoder.JSONDecodeError
-    else:
-        decode_exception_type = ValueError
-
+    decode_exception_type = json.decoder.JSONDecodeError
     try:
         with open(path, "r") as json_file:
             json_data = json.load(json_file)
 
         jsonschema.validate(json_data, manifest_schema)
     except (jsonschema.exceptions.ValidationError, decode_exception_type) as e:
-        raise six.raise_from(
-            ManifestValidationError("error parsing manifest JSON:", str(e)),
-            e,
-        )
+        raise ManifestValidationError("error parsing manifest JSON:", str(e)) from e
 
     specs = entries_to_specs(json_data["specs"])
     tty.debug("{0}: {1} specs read from manifest".format(path, str(len(specs))))

lib/spack/spack/database.py

@@ -28,8 +28,6 @@
 import time
 from typing import Dict  # novm
 
-import six
-
 try:
     import uuid
 
@@ -53,7 +51,6 @@
     InconsistentInstallDirectoryError,
 )
 from spack.error import SpackError
-from spack.filesystem_view import YamlFilesystemView
 from spack.util.crypto import bit_length
 from spack.version import Version
 
@@ -726,6 +723,15 @@ def query_by_spec_hash(self, hash_key, data=None):
                 return True, db._data[hash_key]
         return False, None
 
+    def query_local_by_spec_hash(self, hash_key):
+        """Get a spec by hash in the local database
+
+        Return:
+            (InstallRecord or None): InstallRecord when installed
+                locally, otherwise None."""
+        with self.read_transaction():
+            return self._data.get(hash_key, None)
+
     def _assign_dependencies(self, hash_key, installs, data):
         # Add dependencies from other records in the install DB to
         # form a full spec.
@@ -771,10 +777,7 @@ def _read_from_file(self, filename):
             with open(filename, "r") as f:
                 fdata = sjson.load(f)
         except Exception as e:
-            raise six.raise_from(
-                CorruptDatabaseError("error parsing database:", str(e)),
-                e,
-            )
+            raise CorruptDatabaseError("error parsing database:", str(e)) from e
 
         if fdata is None:
             return
@@ -1379,23 +1382,6 @@ def installed_extensions_for(self, extendee_spec):
             if spec.package.extends(extendee_spec):
                 yield spec.package
 
-    @_autospec
-    def activated_extensions_for(self, extendee_spec, extensions_layout=None):
-        """
-        Return the specs of all packages that extend
-        the given spec
-        """
-        if extensions_layout is None:
-            view = YamlFilesystemView(extendee_spec.prefix, spack.store.layout)
-            extensions_layout = view.extensions_layout
-        for spec in self.query():
-            try:
-                extensions_layout.check_activated(extendee_spec, spec)
-                yield spec.package
-            except spack.directory_layout.NoSuchExtensionError:
-                continue
-            # TODO: conditional way to do this instead of catching exceptions
-
     def _get_by_hash_local(self, dag_hash, default=None, installed=any):
         # hash is a full hash and is in the data somewhere
         if dag_hash in self._data: