Compilers: make oneapi ifx version_regex tolerant to wrappers (#21742)

If a user creates a wrapper for the ifx binary called ifx_orig,
this causes the ifx --version command to produce:

$ ifx --version
ifx_orig (IFORT) 2021.1 Beta 20201113
Copyright (C) 1985-2020 Intel Corporation. All rights reserved.

The regex for ifx currently expects the output to begin with
"ifx (IFORT)..." so the wrapper would not be detected as ifx. This
PR removes the need for the static "ifx" string which allows wrappers
to be detected as ifx.

In general, the Intel compiler regexes do not include the invoked
executable name (i.e., ifort, icc, icx, etc.), so this is not
expected to cause any issues.

.flake8

@@ -27,6 +27,47 @@
 # - N813: camelcase imported as lowercase
 # - N814: camelcase imported as constant
 #
+# F4: pyflakes import checks, these are now checked by mypy more precisely
+# - F403: from module import *
+# - F405: undefined name or from *
+#
+# Black ignores, these are incompatible with black style and do not follow PEP-8
+# - E203: white space around slice operators can be required, ignore : warn
+# - W503: see above, already ignored for line-breaks
+#
 [flake8]
-ignore = E129,E221,E241,E272,E731,W503,W504,F999,N801,N813,N814
-max-line-length = 79
+ignore = E129,E221,E241,E272,E731,W503,W504,F999,N801,N813,N814,F403,F405
+max-line-length = 88
+
+# F4: Import
+# - F405: `name` may be undefined, or undefined from star imports: `module`
+#
+# F8: Name
+# - F821: undefined name `name`
+#
+per-file-ignores =
+  var/spack/repos/*/package.py:F405,F821
+
+# exclude things we usually do not want linting for.
+# These still get linted when passed explicitly, as when spack flake8 passes
+# them on the command line.
+exclude =
+  .git
+  etc/
+  opt/
+  share/
+  var/spack/cache/
+  var/spack/gpg*/
+  var/spack/junit-report/
+  var/spack/mock-configs/
+  lib/spack/external
+  __pycache__
+  var
+
+format = spack
+
+[flake8:local-plugins]
+report =
+  spack = flake8_formatter:SpackFormatter
+paths =
+  ./share/spack/qa/

.flake8_packages

@@ -1,24 +0,0 @@
-# -*- conf -*-
-# flake8 settings for Spack package files.
-#
-# This should include all the same exceptions that we use for core files.
-#
-# In Spack packages, we also allow the single `from spack import *`
-# wildcard import and dependencies can set globals for their
-# dependents. So we add exceptions for checks related to undefined names.
-#
-# Note that we also add *per-line* exemptions for certain patterns in the
-# `spack flake8` command.  This is where F403 for `from spack import *`
-# is added (because we *only* allow that wildcard).
-#
-# See .flake8 for regular exceptions.
-#
-# F4: Import
-# - F405: `name` may be undefined, or undefined from star imports: `module`
-#
-# F8: Name
-# - F821: undefined name `name`
-#
-[flake8]
-ignore = E129,E221,E241,E272,E731,W503,W504,F405,F821,F999,N801,N813,N814
-max-line-length = 79

.gitattributes

@@ -1 +1,3 @@
 *.py diff=python
+*.lp linguist-language=Prolog
+lib/spack/external/* linguist-vendored

.github/actions/add-maintainers-as-reviewers/entrypoint.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/linux_build_tests.yaml

@@ -37,7 +37,7 @@ jobs:
         - ruby-rake  # RubyPackage
     steps:
     - uses: actions/checkout@v2
-    - uses: actions/cache@v2
+    - uses: actions/cache@v2.1.4
       with:
         path: ~/.ccache
         key: ccache-build-${{ matrix.package }}

.github/workflows/linux_unit_tests.yaml

@@ -103,6 +103,7 @@ jobs:
     - uses: codecov/codecov-action@v1
       with:
         flags: shelltests,linux
+
   centos6:
     # Test for Python2.6 run on Centos 6
     runs-on: ubuntu-latest
@@ -117,6 +118,32 @@ jobs:
           git fetch origin ${{ github.ref }}:test-branch
           git checkout test-branch
           share/spack/qa/run-unit-tests
+
+  rhel8-platform-python:
+    runs-on: ubuntu-latest
+    container: registry.access.redhat.com/ubi8/ubi
+    steps:
+    - name: Install dependencies
+      run: |
+          dnf install -y \
+              bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
+              make patch tcl unzip which xz
+    - uses: actions/checkout@v2
+    - name: Setup repo and non-root user
+      run: |
+          git --version
+          git fetch --unshallow
+          . .github/workflows/setup_git.sh
+          useradd spack-test
+          chown -R spack-test .
+    - name: Run unit tests
+      env:
+          SPACK_PYTHON: /usr/libexec/platform-python
+      shell: runuser -u spack-test -- bash {0}
+      run: |
+          source share/spack/setup-env.sh
+          spack unit-test -k 'not svn and not hg' -x --verbose
+
   clingo:
     # Test for the clingo based solver
     runs-on: ubuntu-latest
@@ -138,3 +165,24 @@ jobs:
       - uses: codecov/codecov-action@v1
         with:
           flags: unittests,linux,clingo
+  clingo-cffi:
+    # Test for the clingo based solver (using clingo-cffi)
+    runs-on: ubuntu-latest
+    container: spack/github-actions:clingo-cffi
+    steps:
+      - name: Run unit tests
+        run: |
+          whoami && echo PWD=$PWD && echo HOME=$HOME && echo SPACK_TEST_SOLVER=$SPACK_TEST_SOLVER
+          python3 -c "import clingo; print(hasattr(clingo.Symbol, '_rep'), clingo.__version__)"
+          git clone https://github.com/spack/spack.git && cd spack
+          git fetch origin ${{ github.ref }}:test-branch
+          git checkout test-branch
+          . share/spack/setup-env.sh
+          spack compiler find
+          spack solve mpileaks%gcc
+          coverage run $(which spack) unit-test -v
+          coverage combine
+          coverage xml
+      - uses: codecov/codecov-action@v1
+        with:
+          flags: unittests,linux,clingo

.github/workflows/macos_unit_tests.yaml

@@ -26,7 +26,7 @@ jobs:
       run: |
           pip install --upgrade pip six setuptools
           pip install --upgrade codecov coverage
-          pip install --upgrade flake8 pep8-naming
+          pip install --upgrade flake8 pep8-naming mypy
     - name: Setup Homebrew packages
       run: |
         brew install dash fish gcc gnupg2 kcov

.github/workflows/style_and_docs.yaml

@@ -22,10 +22,10 @@ jobs:
         pip install --upgrade pip
         pip install --upgrade vermin
     - name: Minimum Version (Spack's Core)
-      run: vermin --backport argparse -t=2.6- -t=3.5- -v lib/spack/spack/ lib/spack/llnl/ bin/
+      run: vermin --backport argparse --backport typing -t=2.6- -t=3.5- -v lib/spack/spack/ lib/spack/llnl/ bin/
     - name: Minimum Version (Repositories)
-      run: vermin --backport argparse -t=2.6- -t=3.5- -v var/spack/repos
-  flake8:
+      run: vermin --backport argparse --backport typing -t=2.6- -t=3.5- -v var/spack/repos
+  style:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -36,15 +36,15 @@ jobs:
         python-version: 3.9
     - name: Install Python packages
       run: |
-        pip install --upgrade pip six setuptools flake8
+        pip install --upgrade pip six setuptools flake8 mypy>=0.800 black
     - name: Setup git configuration
       run: |
         # Need this for the git tests to succeed.
         git --version
         . .github/workflows/setup_git.sh
-    - name: Run flake8 tests
+    - name: Run style tests
       run: |
-          share/spack/qa/run-flake8-tests
+          share/spack/qa/run-style-tests
   documentation:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -1,40 +1,511 @@
-/db
+##########################
+# Spack-specific ignores #
+##########################
+
 /var/spack/stage
 /var/spack/cache
 /var/spack/environments
 /var/spack/repos/*/index.yaml
 /var/spack/repos/*/lock
-__pycache__/
-*.pyc
 /opt
-*~
-.DS_Store
-.idea
 # Ignore everything in /etc/spack except /etc/spack/defaults
 /etc/spack/*
 !/etc/spack/defaults
-/etc/spackconfig
-/share/spack/dotkit
 /share/spack/modules
 /share/spack/lmod
-/TAGS
-*.swp
-/htmlcov
-.coverage
-\#*
-.#*
-.cache
-lib/spack/spack/test/.cache
-/bin/spackc
+# Debug logs
+spack-db.*
 *.in.log
 *.out.log
-*.orig
 
-# Eclipse files
-.project
+###########################
+# Python-specific ignores #
+###########################
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+#lib/
+#lib64/
+parts/
+sdist/
+#var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+########################
+# Vim-specific ignores #
+########################
+
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+##########################
+# Emacs-specific ignores #
+##########################
+
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*
+
+# Org-mode
+.org-id-locations
+*_archive
+
+# flymake-mode
+*_flymake.*
+
+# eshell files
+/eshell/history
+/eshell/lastdir
+
+# elpa packages
+/elpa/
+
+# reftex files
+*.rel
+
+# AUCTeX auto folder
+/auto/
+
+# cask packages
+.cask/
+dist/
+
+# Flycheck
+flycheck_*.el
+
+# server auth directory
+/server/
+
+# projectiles files
+.projectile
+
+# directory configuration
+.dir-locals.el
+
+# network security
+/network-security.data
+
+############################
+# Eclipse-specific ignores #
+############################
+
+.metadata
+#bin/
+tmp/
+*.tmp
+*.bak
+*.swp
+*~.nib
+local.properties
+.settings/
+.loadpath
+.recommenders
+
+# External tool builders
+.externalToolBuilders/
+
+# Locally stored "Eclipse launch configurations"
+*.launch
+
+# PyDev specific (Python IDE for Eclipse)
+*.pydevproject
+
+# CDT-specific (C/C++ Development Tooling)
 .cproject
-.pydevproject
 
-# VSCode files
-.vscode
-.devcontainer
+# CDT- autotools
+.autotools
+
+# Java annotation processor (APT)
+.factorypath
+
+# PDT-specific (PHP Development Tools)
+.buildpath
+
+# sbteclipse plugin
+.target
+
+# Tern plugin
+.tern-project
+
+# TeXlipse plugin
+.texlipse
+
+# STS (Spring Tool Suite)
+.springBeans
+
+# Code Recommenders
+.recommenders/
+
+# Annotation Processing
+.apt_generated/
+.apt_generated_test/
+
+# Scala IDE specific (Scala & Java development for Eclipse)
+.cache-main
+.scala_dependencies
+.worksheet
+
+# Uncomment this line if you wish to ignore the project description file.
+# Typically, this file would be tracked if it contains build/dependency configurations:
+#.project
+
+##################################
+# Visual Studio-specific ignores #
+##################################
+
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/
+
+#################################
+# Sublime Text-specific ignores #
+#################################
+
+# Cache files for Sublime Text
+*.tmlanguage.cache
+*.tmPreferences.cache
+*.stTheme.cache
+
+# Workspace files are user-specific
+*.sublime-workspace
+
+# Project files should be checked into the repository, unless a significant
+# proportion of contributors will probably not be using Sublime Text
+# *.sublime-project
+
+# SFTP configuration file
+sftp-config.json
+sftp-config-alt*.json
+
+# Package control specific files
+Package Control.last-run
+Package Control.ca-list
+Package Control.ca-bundle
+Package Control.system-ca-bundle
+Package Control.cache/
+Package Control.ca-certs/
+Package Control.merged-ca-bundle
+Package Control.user-ca-bundle
+oscrypto-ca-bundle.crt
+bh_unicode_properties.cache
+
+# Sublime-github package stores a github token in this file
+# https://packagecontrol.io/packages/sublime-github
+GitHub.sublime-settings
+
+##############################
+# JetBrains-specific ignores #
+##############################
+
+# Ignore the entire folder since it may conatin more files than
+# just the ones listed below
+.idea/
+
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+##########################
+# macOS-specific ignores #
+##########################
+
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+##########################
+# Linux-specific ignores #
+##########################
+
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+############################
+# Windows-specific ignores #
+############################
+
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk

.mypy.ini

@@ -0,0 +1,35 @@
+[mypy]
+python_version = 3.7
+files=lib/spack/llnl/**/*.py,lib/spack/spack/**/*.py
+mypy_path=bin,lib/spack,lib/spack/external,var/spack/repos/builtin
+# This and a generated import file allows supporting packages
+namespace_packages=True
+# To avoid re-factoring all the externals, ignore errors and missing imports
+# globally, then turn back on in spack and spack submodules
+ignore_errors=True
+ignore_missing_imports=True
+
+[mypy-spack.*]
+ignore_errors=False
+ignore_missing_imports=False
+
+[mypy-packages.*]
+ignore_errors=False
+ignore_missing_imports=False
+
+[mypy-llnl.*]
+ignore_errors=False
+ignore_missing_imports=False
+
+[mypy-spack.test.packages]
+ignore_errors=True
+
+# ignore errors in fake import path for packages
+[mypy-spack.pkg.*]
+ignore_errors=True
+ignore_missing_imports=True
+
+# jinja has syntax in it that requires python3 and causes a parse error
+# skip importing it
+[mypy-jinja2]
+follow_imports=skip

bin/spack

@@ -1,7 +1,7 @@
 #!/bin/sh
 # -*- python -*-
 #
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -10,9 +10,12 @@
 # Following line is a shell no-op, and starts a multi-line Python comment.
 # See https://stackoverflow.com/a/47886254
 """:"
-# prefer python3, then python, then python2
-for cmd in python3 python python2; do
-   command -v > /dev/null $cmd && exec $cmd $0 "$@"
+# prefer SPACK_PYTHON environment variable, python3, python, then python2
+for cmd in "${SPACK_PYTHON:-}" python3 python python2; do
+    if command -v > /dev/null "$cmd"; then
+        export SPACK_PYTHON="$(command -v "$cmd")"
+        exec "${SPACK_PYTHON}" "$0" "$@"
+    fi
 done
 
 echo "==> Error: spack could not find a python interpreter!" >&2
@@ -42,6 +45,8 @@ sys.path.insert(0, spack_lib_path)
 # Add external libs
 spack_external_libs = os.path.join(spack_lib_path, "external")
 
+if sys.version_info[:2] <= (2, 7):
+    sys.path.insert(0, os.path.join(spack_external_libs, 'py2'))
 if sys.version_info[:2] == (2, 6):
     sys.path.insert(0, os.path.join(spack_external_libs, 'py26'))
 

bin/spack-python

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

etc/spack/defaults/config.yaml

@@ -107,7 +107,7 @@ config:
   # If set to true, Spack will attempt to build any compiler on the spec
   # that is not already available. If set to False, Spack will only use
   # compilers already configured in compilers.yaml
-  install_missing_compilers: False
+  install_missing_compilers: false
 
 
   # If set to true, Spack will always check checksums after downloading
@@ -115,6 +115,11 @@ config:
   checksum: true
 
 
+  # If set to true, Spack will fetch deprecated versions without warning.
+  # If false, Spack will raise an error when trying to install a deprecated version.
+  deprecated: false
+
+
   # If set to true, `spack install` and friends will NOT clean
   # potentially harmful variables from the build environment. Use wisely.
   dirty: false

etc/spack/defaults/darwin/packages.yaml

@@ -25,6 +25,8 @@ packages:
       - libelf
       unwind:
       - apple-libunwind
+      uuid:
+      - apple-libuuid
   apple-libunwind:
     buildable: false
     externals:
@@ -32,3 +34,10 @@ packages:
     # although the version number used here isn't critical
     - spec: apple-libunwind@35.3
       prefix: /usr
+  apple-libuuid:
+    buildable: false
+    externals:
+    # Apple bundles libuuid in libsystem_c version 1353.100.2,
+    # although the version number used here isn't critical
+    - spec: apple-libuuid@1353.100.2
+      prefix: /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk

etc/spack/defaults/modules.yaml

@@ -25,16 +25,6 @@ modules:
       - MANPATH
     share/aclocal:
       - ACLOCAL_PATH
-    lib:
-      - LIBRARY_PATH
-    lib64:
-      - LIBRARY_PATH
-    include:
-      - C_INCLUDE_PATH
-      - CPLUS_INCLUDE_PATH
-      # The INCLUDE env variable specifies paths to look for
-      # .mod file for Intel Fortran compilers
-      - INCLUDE
     lib/pkgconfig:
       - PKG_CONFIG_PATH
     lib64/pkgconfig:

etc/spack/defaults/packages.yaml

@@ -23,7 +23,7 @@ packages:
       daal: [intel-daal]
       elf: [elfutils]
       fftw-api: [fftw, amdfftw]
-      gl: [mesa+opengl, mesa18+opengl, opengl]
+      gl: [mesa+opengl, mesa18, opengl]
       glx: [mesa+glx, mesa18+glx, opengl]
       glu: [mesa-glu, openglu]
       golang: [gcc]
@@ -49,6 +49,7 @@ packages:
       unwind: [libunwind]
       yacc: [bison, byacc]
       flame: [libflame, amdlibflame]
+      uuid: [util-linux-uuid, libuuid]
     permissions:
       read: world
       write: user

lib/spack/docs/basic_usage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -133,8 +133,7 @@ dependencies first.  It then fetches the ``mpileaks`` tarball, expands
 it, verifies that it was downloaded without errors, builds it, and
 installs it in its own directory under ``$SPACK_ROOT/opt``. You'll see
 a number of messages from Spack, a lot of build output, and a message
-that the package is installed.  Add one or more debug options (``-d``)
-to get increasingly detailed output.
+that the package is installed.
 
 .. code-block:: console
 
@@ -151,8 +150,8 @@ to get increasingly detailed output.
 The last line, with the ``[+]``, indicates where the package is
 installed.
 
-Add the debug option -- ``spack  install -d mpileaks`` -- to get additional
-output.
+Add the Spack debug option (one or more times) -- ``spack -d install
+mpileaks`` -- to get additional (and even more verbose) output.
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Building a specific version

lib/spack/docs/binary_caches.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_settings.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -60,6 +60,7 @@ on these ideas for each distinct build system that Spack supports:
    build_systems/bundlepackage
    build_systems/cudapackage
    build_systems/intelpackage
+   build_systems/rocmpackage
    build_systems/custompackage
 
 For reference, the :py:mod:`Build System API docs <spack.build_systems>`

lib/spack/docs/build_systems/autotoolspackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/bundlepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/cmakepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/cudapackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -9,35 +9,120 @@
 CudaPackage
 -----------
 
-Different from other packages, ``CudaPackage`` does not represent a build
-system. Instead its goal is to simplify and unify usage of ``CUDA`` in other
-packages.
+Different from other packages, ``CudaPackage`` does not represent a build system.
+Instead its goal is to simplify and unify usage of ``CUDA`` in other packages by providing a ` mixin-class <https://en.wikipedia.org/wiki/Mixin>`__.
 
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Provided variants and dependencies
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+You can find source for the package at
+`<https://github.com/spack/spack/blob/develop/lib/spack/spack/build_systems/cuda.py>`__.
 
-``CudaPackage`` provides ``cuda`` variant (default to ``off``) to enable/disable
-``CUDA``, and ``cuda_arch`` variant to optionally specify the architecture.
-It also declares dependencies on the ``CUDA`` package ``depends_on('cuda@...')``
-based on the architecture as well as specifies conflicts for certain compiler versions.
+^^^^^^^^
+Variants
+^^^^^^^^
+
+This package provides the following variants:
+
+* **cuda**
+
+  This variant is used to enable/disable building with ``CUDA``. The default
+  is disabled (or ``False``).
+
+* **cuda_arch**
+
+  This variant supports the optional specification of the architecture.
+  Valid values are maintained in the ``cuda_arch_values`` property and
+  are the numeric character equivalent of the compute capability version
+  (e.g., '10' for version 1.0). Each provided value affects associated
+  ``CUDA`` dependencies and compiler conflicts.
+
+  GPUs and their compute capability versions are listed at
+  https://developer.nvidia.com/cuda-gpus .
+
+^^^^^^^^^
+Conflicts
+^^^^^^^^^
+
+Conflicts are used to prevent builds with known bugs or issues. While
+base ``CUDA`` conflicts have been included with this package, you may
+want to add more for your software.
+
+For example, if your package requires ``cuda_arch`` to be specified when
+``cuda`` is enabled, you can add the following conflict to your package
+to terminate such build attempts with a suitable message:
+
+.. code-block:: python
+
+    conflicts('cuda_arch=none', when='+cuda',
+              msg='CUDA architecture is required')
+
+Similarly, if your software does not support all versions of the property,
+you could add ``conflicts`` to your package for those versions.  For example,
+suppose your software does not work with CUDA compute capability versions
+prior to SM 5.0 (``50``).  You can add the following code to display a
+custom message should a user attempt such a build:
+
+.. code-block:: python
+
+    unsupported_cuda_archs = [
+        '10', '11', '12', '13',
+        '20', '21',
+        '30', '32', '35', '37'
+    ]
+    for value in unsupported_cuda_archs:
+        conflicts('cuda_arch={0}'.format(value), when='+cuda',
+                  msg='CUDA architecture {0} is not supported'.format(value))
+
+^^^^^^^
+Methods
+^^^^^^^
+
+This package provides one custom helper method, which is used to build
+standard CUDA compiler flags.
+
+**cuda_flags**
+
+    This built-in static method returns a list of command line flags 
+    for the chosen ``cuda_arch`` value(s).  The flags are intended to
+    be passed to the CUDA compiler driver (i.e., ``nvcc``).
+
+    This method must be explicitly called when you are creating the
+    arguments for your build in order to use the values.
 
 ^^^^^
 Usage
 ^^^^^
 
-In order to use it, just add another base class to your package, for example:
+This helper package can be added to your package by adding it as a base
+class of your package.  For example, you can add it to your
+:ref:`CMakePackage <cmakepackage>`-based package as follows:
 
 .. code-block:: python
+   :emphasize-lines: 1,7-16
 
-    class MyPackage(CMakePackage, CudaPackage):
+    class MyCudaPackage(CMakePackage, CudaPackage):
         ...
         def cmake_args(self):
             spec = self.spec
+            args = []
+            ...
             if '+cuda' in spec:
-                options.append('-DWITH_CUDA=ON')
-                cuda_arch = spec.variants['cuda_arch'].value
+                # Set up the cuda macros needed by the build
+                args.append('-DWITH_CUDA=ON')
+                cuda_arch_list = spec.variants['cuda_arch'].value
+                cuda_arch = cuda_arch_list[0]
                 if cuda_arch != 'none':
-                    options.append('-DCUDA_FLAGS=-arch=sm_{0}'.format(cuda_arch[0]))
+                    args.append('-DCUDA_FLAGS=-arch=sm_{0}'.format(cuda_arch))
             else:
-                options.append('-DWITH_CUDA=OFF')
+                # Ensure build with cuda is disabled
+                args.append('-DWITH_CUDA=OFF')
+            ...
+            return args
+
+assuming only the ``WITH_CUDA`` and ``CUDA_FLAGS`` flags are required.
+You will need to customize options as needed for your build.
+
+This example also illustrates how to check for the ``cuda`` variant using
+``self.spec`` and how to retrieve the ``cuda_arch`` variant's value, which
+is a list, using ``self.spec.variants['cuda_arch'].value``.
+
+With over 70 packages using ``CudaPackage`` as of January 2021 there are
+lots of examples to choose from to get more ideas for using this package.

lib/spack/docs/build_systems/custompackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/intelpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/makefilepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/mavenpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/mesonpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/octavepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/perlpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -120,8 +120,6 @@ so ``PerlPackage`` contains:
 
    extends('perl')
 
-   depends_on('perl', type=('build', 'run'))
-
 
 If your package requires a specific version of Perl, you should
 specify this.

lib/spack/docs/build_systems/pythonpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -23,20 +23,11 @@ can be overridden:
 * ``build_ext``
 * ``build_clib``
 * ``build_scripts``
-* ``clean``
 * ``install``
 * ``install_lib``
 * ``install_headers``
 * ``install_scripts``
 * ``install_data``
-* ``sdist``
-* ``register``
-* ``bdist``
-* ``bdist_dumb``
-* ``bdist_rpm``
-* ``bdist_wininst``
-* ``upload``
-* ``check``
 
 These are all standard ``setup.py`` commands and can be found by running:
 
@@ -55,7 +46,7 @@ If for whatever reason you need to run more phases, simply modify your
 
 .. code-block:: python
 
-   phases = ['build_ext', 'install', 'bdist']
+   phases = ['build_ext', 'install']
 
 
 Each phase provides a function ``<phase>`` that runs:
@@ -90,7 +81,7 @@ Instead of using the ``PythonPackage`` base class, you should extend
 the ``Package`` base class and implement the following custom installation
 procedure:
 
-.. code-block::
+.. code-block:: python
 
    def install(self, spec, prefix):
        pip = which('pip')
@@ -134,9 +125,9 @@ The zip file will not contain a ``setup.py``, but it will contain a
 ``METADATA`` file which contains all the information you need to
 write a ``package.py`` build recipe.
 
-^^^^^^^^^^^^^^^^^^^^^^^
-Finding Python packages
-^^^^^^^^^^^^^^^^^^^^^^^
+^^^^
+PyPI
+^^^^
 
 The vast majority of Python packages are hosted on PyPI - The Python
 Package Index. ``pip`` only supports packages hosted on PyPI, making
@@ -148,6 +139,26 @@ if a newer version is available. The download page is usually at::
 
    https://pypi.org/project/<package-name>
 
+
+Since PyPI is so common, the ``PythonPackage`` base class has a
+``pypi`` attribute that can be set. Once set, ``pypi`` will be used
+to define the ``homepage``, ``url``, and ``list_url``. For example,
+the following:
+
+.. code-block:: python
+
+   homepage = 'https://pypi.org/project/setuptools/'
+   url      = 'https://pypi.org/packages/source/s/setuptools/setuptools-49.2.0.zip'
+   list_url = 'https://pypi.org/simple/setuptools/'
+
+
+is equivalent to:
+
+.. code-block:: python
+
+   pypi = 'setuptools/setuptools-49.2.0.zip'
+
+
 ^^^^^^^^^^^
 Description
 ^^^^^^^^^^^
@@ -184,50 +195,11 @@ also get the homepage on the command-line by running:
 URL
 ^^^
 
-You may have noticed that Spack allows you to add multiple versions of
-the same package without adding multiple versions of the download URL.
-It does this by guessing what the version string in the URL is and
-replacing this with the requested version. Obviously, if Spack cannot
-guess the version correctly, or if non-version-related things change
-in the URL, Spack cannot substitute the version properly.
-
-Once upon a time, PyPI offered nice, simple download URLs like::
-
-   https://pypi.python.org/packages/source/n/numpy/numpy-1.13.1.zip
-
-
-As you can see, the version is 1.13.1. It probably isn't hard to guess
-what URL to use to download version 1.12.0, and Spack was perfectly
-capable of performing this calculation.
-
-However, PyPI switched to a new download URL format::
-
-   https://pypi.python.org/packages/c0/3a/40967d9f5675fbb097ffec170f59c2ba19fc96373e73ad47c2cae9a30aed/numpy-1.13.1.zip#md5=2c3c0f4edf720c3a7b525dacc825b9ae
-
-
-and more recently::
-
-   https://files.pythonhosted.org/packages/b0/2b/497c2bb7c660b2606d4a96e2035e92554429e139c6c71cdff67af66b58d2/numpy-1.14.3.zip
-
-
-As you can imagine, it is impossible for Spack to guess what URL to
-use to download version 1.12.0 given this URL. There is a solution,
-however. PyPI offers a new hidden interface for downloading
-Python packages that does not include a hash in the URL::
-
-   https://pypi.io/packages/source/n/numpy/numpy-1.13.1.zip
-
-
-This URL redirects to the https://files.pythonhosted.org URL. The general
-syntax for this https://pypi.io URL is::
-
-   https://pypi.io/packages/<type>/<first-letter-of-name>/<name>/<name>-<version>.<extension>
-
-
-Please use the https://pypi.io URL instead of the https://pypi.python.org
-URL. If both ``.tar.gz`` and ``.zip`` versions are available, ``.tar.gz``
-is preferred. If some releases offer both ``.tar.gz`` and ``.zip`` versions,
-but some only offer ``.zip`` versions, use ``.zip``.
+If ``pypi`` is set as mentioned above, ``url`` and ``list_url`` will
+be automatically set for you. If both ``.tar.gz`` and ``.zip`` versions
+are available, ``.tar.gz`` is preferred. If some releases offer both
+``.tar.gz`` and ``.zip`` versions, but some only offer ``.zip`` versions,
+use ``.zip``.
 
 Some Python packages are closed-source and do not ship ``.tar.gz`` or ``.zip``
 files on either PyPI or GitHub. If this is the case, you can still download
@@ -237,10 +209,9 @@ and can be downloaded from::
    https://pypi.io/packages/py3/a/azureml_sdk/azureml_sdk-1.11.0-py3-none-any.whl
 
 
-Note that instead of ``<type>`` being ``source``, it is now ``py3`` since this
-wheel will work for any generic version of Python 3. You may see Python-specific
-or OS-specific URLs. Note that when you add a ``.whl`` URL, you should add
-``expand=False`` to ensure that Spack doesn't try to extract the wheel:
+You may see Python-specific or OS-specific URLs. Note that when you add a
+``.whl`` URL, you should add ``expand=False`` to ensure that Spack doesn't
+try to extract the wheel:
 
 .. code-block:: python
 
@@ -255,7 +226,7 @@ Many packages are hosted on PyPI, but are developed on GitHub or another
 version control systems. The tarball can be downloaded from either
 location, but PyPI is preferred for the following reasons:
 
-#. PyPI contains the bare minimum of files to install the package.
+#. PyPI contains the bare minimum number of files needed to install the package.
 
    You may notice that the tarball you download from PyPI does not
    have the same checksum as the tarball you download from GitHub.
@@ -292,19 +263,6 @@ location, but PyPI is preferred for the following reasons:
    PyPI is nice because it makes it physically impossible to
    re-release the same version of a package with a different checksum.
 
-There are some reasons to prefer downloading from GitHub:
-
-#. The GitHub tarball may contain unit tests.
-
-   As previously mentioned, the PyPI tarball contains the bare minimum
-   of files to install the package. Unless explicitly specified by the
-   developers, it will not contain development files like unit tests.
-   If you desire to run the unit tests during installation, you should
-   use the GitHub tarball instead.
-
-If you really want to run these unit tests, no one will stop you from
-submitting a PR for a new package that downloads from GitHub.
-
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 Build system dependencies
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -569,7 +527,8 @@ If the package uses ``setuptools``, check for the following clues:
 
   These are packages that are required to run the unit tests for the
   package. These dependencies can be specified using the
-  ``type='test'`` dependency type.
+  ``type='test'`` dependency type. However, the PyPI tarballs rarely
+  contain unit tests, so there is usually no reason to add these.
 
 In the root directory of the package, you may notice a
 ``requirements.txt`` file. It may look like this file contains a list
@@ -625,7 +584,8 @@ add run-time dependencies if they aren't needed, so you need to
 determine whether or not setuptools is needed. Grep the installation
 directory for any files containing a reference to ``setuptools`` or
 ``pkg_resources``. Both modules come from ``py-setuptools``.
-``pkg_resources`` is particularly common in scripts in ``prefix/bin``.
+``pkg_resources`` is particularly common in scripts found in
+``prefix/bin``.
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Passing arguments to setup.py
@@ -699,49 +659,65 @@ a "package" is a directory containing files like:
    foo/baz.py
 
 
-whereas a "module" is a single Python file. Since ``find_packages``
-only returns packages, you'll have to determine the correct module
-names yourself. You can now add these packages and modules to the
-package like so:
+whereas a "module" is a single Python file.
+
+The ``PythonPackage`` base class automatically detects these module
+names for you. If, for whatever reason, the module names detected
+are wrong, you can provide the names yourself by overriding
+``import_modules`` like so:
 
 .. code-block:: python
 
    import_modules = ['six']
 
 
-When you run ``spack install --test=root py-six``, Spack will attempt
-to import the ``six`` module after installation.
+Sometimes the list of module names to import depends on how the
+package was built. For example, the ``py-pyyaml`` package has a
+``+libyaml`` variant that enables the build of a faster optimized
+version of the library. If the user chooses ``~libyaml``, only the
+``yaml`` library will be importable. If the user chooses ``+libyaml``,
+both the ``yaml`` and ``yaml.cyaml`` libraries will be available.
+This can be expressed like so:
 
-These tests most often catch missing dependencies and non-RPATHed
+.. code-block:: python
+
+   @property
+   def import_modules(self):
+       modules = ['yaml']
+
+       if '+libyaml' in self.spec:
+           modules.append('yaml.cyaml')
+
+       return modules
+
+
+These tests often catch missing dependencies and non-RPATHed
 libraries. Make sure not to add modules/packages containing the word
-"test", as these likely won't end up in installation directory.
+"test", as these likely won't end up in the installation directory,
+or may require test dependencies like pytest to be installed.
+
+These tests can be triggered by running ``spack install --test=root``
+or by running ``spack test run`` after the installation has finished.
 
 """"""""""
 Unit tests
 """"""""""
 
 The package you want to install may come with additional unit tests.
-By default, Spack runs:
-
-.. code-block:: console
-
-   $ python setup.py test
-
-
-if it detects that the ``setup.py`` file supports a ``test`` phase.
-You can add additional build-time or install-time tests by overriding
-``test`` or adding a custom install-time test function. For example,
-``py-numpy`` adds:
+You can add additional build-time or install-time tests by adding
+additional testing functions. For example, ``py-numpy`` adds:
 
 .. code-block:: python
 
-   install_time_test_callbacks = ['install_test', 'import_module_test']
-
+   @run_after('install')
+   @on_package_attributes(run_tests=True)
    def install_test(self):
-        with working_dir('..'):
-            python('-c', 'import numpy; numpy.test("full", verbose=2)')
+       with working_dir('spack-test', create=True):
+           python('-c', 'import numpy; numpy.test("full", verbose=2)')
 
 
+These tests can be triggered by running ``spack install --test=root``.
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Setup file in a sub-directory
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -781,7 +757,7 @@ PythonPackage vs. packages that use Python
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 There are many packages that make use of Python, but packages that depend
-on Python are not necessarily ``PythonPackages``.
+on Python are not necessarily ``PythonPackage``'s.
 
 """""""""""""""""""""""
 Choosing a build system
@@ -878,8 +854,8 @@ and ``pip`` may be a perfectly valid alternative to using Spack. The
 main advantage of Spack over ``pip`` is its ability to compile
 non-Python dependencies. It can also build cythonized versions of a
 package or link to an optimized BLAS/LAPACK library like MKL,
-resulting in calculations that run orders of magnitude faster.
-Spack does not offer a significant advantage to other python-management
+resulting in calculations that run orders of magnitudes faster.
+Spack does not offer a significant advantage over other python-management
 systems for installing and using tools like flake8 and sphinx.
 But if you need packages with non-Python dependencies like
 numpy and scipy, Spack will be very valuable to you.

lib/spack/docs/build_systems/qmakepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -108,6 +108,19 @@ override the ``qmake_args`` method like so:
 
 This method can be used to pass flags as well as variables.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+``*.pro`` file in a sub-directory
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If the ``*.pro`` file used to tell QMake how to build the package is
+found in a sub-directory, you can tell Spack to run all phases in this
+sub-directory by adding the following to the package:
+
+.. code-block:: python
+
+   build_directory = 'src'
+
+
 ^^^^^^^^^^^^^^^^^^^^^^
 External documentation
 ^^^^^^^^^^^^^^^^^^^^^^

lib/spack/docs/build_systems/rocmpackage.rst

@@ -0,0 +1,122 @@
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.
+
+   SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+.. _rocmpackage:
+
+-----------
+ROCmPackage
+-----------
+
+The ``ROCmPackage`` is not a build system but a helper package. Like ``CudaPackage``,
+it provides standard variants, dependencies, and conflicts to facilitate building
+packages using GPUs though for AMD in this case.
+
+You can find the source for this package (and suggestions for setting up your
+``compilers.yaml`` and ``packages.yaml`` files) at
+`<https://github.com/spack/spack/blob/develop/lib/spack/spack/build_systems/rocm.py>`__.
+
+^^^^^^^^
+Variants
+^^^^^^^^
+
+This package provides the following variants:
+
+* **rocm**
+
+  This variant is used to enable/disable building with ``rocm``.  
+  The default is disabled (or ``False``).
+
+* **amdgpu_target**
+
+  This variant supports the optional specification of the AMD GPU architecture.
+  Valid values are the names of the GPUs (e.g., ``gfx701``), which are maintained
+  in the ``amdgpu_targets`` property.
+
+^^^^^^^^^^^^
+Dependencies
+^^^^^^^^^^^^
+
+This package defines basic ``rocm`` dependencies, including ``llvm`` and ``hip``.
+
+^^^^^^^^^
+Conflicts
+^^^^^^^^^
+
+Conflicts are used to prevent builds with known bugs or issues. This package
+already requires that the ``amdgpu_target`` always be specified for ``rocm``
+builds. It also defines a conflict that prevents builds with an ``amdgpu_target``
+when ``rocm`` is disabled.
+
+Refer to `Conflicts <https://spack.readthedocs.io/en/latest/packaging_guide.html?highlight=conflicts#conflicts>`__
+for more information on package conflicts.
+
+^^^^^^^
+Methods
+^^^^^^^
+
+This package provides one custom helper method, which is used to build
+standard AMD hip compiler flags.
+
+**hip_flags**
+
+    This built-in static method returns the appropriately formatted
+    ``--amdgpu-target`` build option for ``hipcc``.
+
+    This method must be explicitly called when you are creating the
+    arguments for your build in order to use the values.
+
+^^^^^
+Usage
+^^^^^
+
+This helper package can be added to your package by adding it as a base
+class of your package.  For example, you can add it to your
+:ref:`CMakePackage <cmakepackage>`-based package as follows:
+
+.. code-block:: python
+   :emphasize-lines: 1,3-7,14-25
+
+    class MyRocmPackage(CMakePackage, ROCmPackage):
+        ...
+        # Ensure +rocm and amdgpu_targets are passed to dependencies
+        depends_on('mydeppackage', when='+rocm')
+        for val in ROCmPackage.amdgpu_targets:
+            depends_on('mydeppackage amdgpu_target={0}'.format(val),
+                       when='amdgpu_target={0}'.format(val))
+        ...
+
+        def cmake_args(self):
+            spec = self.spec
+            args = []
+            ...
+            if '+rocm' in spec:
+                # Set up the hip macros needed by the build
+                args.extend([
+                    '-DENABLE_HIP=ON',
+                    '-DHIP_ROOT_DIR={0}'.format(spec['hip'].prefix])
+                rocm_archs = spec.variants['amdgpu_target'].value
+                if 'none' not in rocm_archs:
+                    args.append('-DHIP_HIPCC_FLAGS=--amdgpu-target={0}'
+                                .format(",".join(rocm_archs)))
+            else:
+                # Ensure build with hip is disabled
+                args.append('-DENABLE_HIP=OFF')
+            ...
+            return args
+        ...
+
+assuming only on the ``ENABLE_HIP``, ``HIP_ROOT_DIR``, and ``HIP_HIPCC_FLAGS``
+macros are required to be set and the only dependency needing rocm options
+is ``mydeppackage``. You will need to customize the flags as needed for your
+build.
+
+This example also illustrates how to check for the ``rocm`` variant using
+``self.spec`` and how to retrieve the ``amdgpu_target`` variant's value
+using ``self.spec.variants['amdgpu_target'].value``.
+
+All five packages using ``ROCmPackage`` as of January 2021 also use the
+:ref:`CudaPackage <cudapackage>`. So it is worth looking at those packages
+to get ideas for creating a package that can support both ``cuda`` and
+``rocm``.

lib/spack/docs/build_systems/rpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -138,7 +138,6 @@ every R package needs this, the ``RPackage`` base class contains:
 .. code-block:: python
 
    extends('r')
-   depends_on('r', type=('build', 'run'))
 
 
 Take a close look at the homepage for ``caret``. If you look at the
@@ -298,8 +297,8 @@ like so:
 
 .. code-block:: python
 
-   def configure_args(self, spec, prefix):
-       mpi_name = spec['mpi'].name
+   def configure_args(self):
+       mpi_name = self.spec['mpi'].name
 
        # The type of MPI. Supported values are:
        # OPENMPI, LAM, MPICH, MPICH2, or CRAY

lib/spack/docs/build_systems/rubypackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -113,7 +113,6 @@ the base class contains:
 .. code-block:: python
 
    extends('ruby')
-   depends_on('ruby', type=('build', 'run'))
 
 
 The ``*.gemspec`` file may contain something like:

lib/spack/docs/build_systems/sconspackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/sippackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -93,10 +93,17 @@ in the site-packages directory:
    $ python
    >>> import setuptools
    >>> setuptools.find_packages()
-   ['QtPy5']
+   [
+       'PyQt5', 'PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtHelp',
+       'PyQt5.QtMultimedia', 'PyQt5.QtMultimediaWidgets', 'PyQt5.QtNetwork',
+       'PyQt5.QtOpenGL', 'PyQt5.QtPrintSupport', 'PyQt5.QtQml',
+       'PyQt5.QtQuick', 'PyQt5.QtSvg', 'PyQt5.QtTest', 'PyQt5.QtWebChannel',
+       'PyQt5.QtWebSockets', 'PyQt5.QtWidgets', 'PyQt5.QtXml',
+       'PyQt5.QtXmlPatterns'
+    ]
 
 
-Large, complex packages like ``QtPy5`` will return a long list of
+Large, complex packages like ``py-pyqt5`` will return a long list of
 packages, while other packages may return an empty list. These packages
 only install a single ``foo.py`` file. In Python packaging lingo,
 a "package" is a directory containing files like:
@@ -108,21 +115,25 @@ a "package" is a directory containing files like:
    foo/baz.py
 
 
-whereas a "module" is a single Python file. Since ``find_packages``
-only returns packages, you'll have to determine the correct module
-names yourself. You can now add these packages and modules to the
-package like so:
+whereas a "module" is a single Python file.
+
+The ``SIPPackage`` base class automatically detects these module
+names for you. If, for whatever reason, the module names detected
+are wrong, you can provide the names yourself by overriding
+``import_modules`` like so:
 
 .. code-block:: python
 
    import_modules = ['PyQt5']
 
 
-When you run ``spack install --test=root py-pyqt5``, Spack will attempt
-to import the ``PyQt5`` module after installation.
+These tests often catch missing dependencies and non-RPATHed
+libraries. Make sure not to add modules/packages containing the word
+"test", as these likely won't end up in the installation directory,
+or may require test dependencies like pytest to be installed.
 
-These tests most often catch missing dependencies and non-RPATHed
-libraries.
+These tests can be triggered by running ``spack install --test=root``
+or by running ``spack test run`` after the installation has finished.
 
 ^^^^^^^^^^^^^^^^^^^^^^
 External documentation

lib/spack/docs/build_systems/wafpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/chain.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/conf.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -128,7 +128,7 @@ def setup(sphinx):
 
 # General information about the project.
 project = u'Spack'
-copyright = u'2013-2019, Lawrence Livermore National Laboratory.'
+copyright = u'2013-2021, Lawrence Livermore National Laboratory.'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

lib/spack/docs/config_yaml.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/configuration.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/containers.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/contribution_guide.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -179,24 +179,26 @@ how to write tests!
    run the unit tests yourself, we suggest you use ``spack unit-test``.
 
 ^^^^^^^^^^^^
-Flake8 Tests
+Style Tests
 ^^^^^^^^^^^^
 
 Spack uses `Flake8 <http://flake8.pycqa.org/en/latest/>`_ to test for
-`PEP 8 <https://www.python.org/dev/peps/pep-0008/>`_ conformance. PEP 8 is
+`PEP 8 <https://www.python.org/dev/peps/pep-0008/>`_ conformance and
+`mypy <https://mypy.readthedocs.io/en/stable/>` for type checking. PEP 8 is
 a series of style guides for Python that provide suggestions for everything
 from variable naming to indentation. In order to limit the number of PRs that
 were mostly style changes, we decided to enforce PEP 8 conformance. Your PR
-needs to comply with PEP 8 in order to be accepted.
+needs to comply with PEP 8 in order to be accepted, and if it modifies the
+spack library it needs to successfully type-check with mypy as well.
 
-Testing for PEP 8 compliance is easy. Simply run the ``spack flake8``
+Testing for compliance with spack's style is easy. Simply run the ``spack style``
 command:
 
 .. code-block:: console
 
-   $ spack flake8
+   $ spack style
 
-``spack flake8`` has a couple advantages over running ``flake8`` by hand:
+``spack style`` has a couple advantages over running the tools by hand:
 
 #. It only tests files that you have modified since branching off of
    ``develop``.
@@ -207,7 +209,9 @@ command:
    checks. For example, URLs are often longer than 80 characters, so we
    exempt them from line length checks. We also exempt lines that start
    with "homepage", "url", "version", "variant", "depends_on", and
-   "extends" in ``package.py`` files.
+   "extends" in ``package.py`` files.  This is now also possible when directly
+   running flake8 if you can use the ``spack`` formatter plugin included with
+   spack.
 
 More approved flake8 exemptions can be found
 `here <https://github.com/spack/spack/blob/develop/.flake8>`_.
@@ -240,13 +244,13 @@ However, if you aren't compliant with PEP 8, flake8 will complain:
 
 Most of the error messages are straightforward, but if you don't understand what
 they mean, just ask questions about them when you submit your PR. The line numbers
-will change if you add or delete lines, so simply run ``spack flake8`` again
+will change if you add or delete lines, so simply run ``spack style`` again
 to update them.
 
 .. tip::
 
    Try fixing flake8 errors in reverse order. This eliminates the need for
-   multiple runs of ``spack flake8`` just to re-compute line numbers and
+   multiple runs of ``spack style`` just to re-compute line numbers and
    makes it much easier to fix errors directly off of the CI output.
 
 .. warning::

lib/spack/docs/developer_guide.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -396,20 +396,42 @@ other Spack modules:
    True
    >>>
 
-You can also run a single command:
+If you prefer using an IPython interpreter, given that IPython is installed
+you can specify the interpreter with ``-i``:
+
+.. code-block:: console
+
+   $ spack python -i ipython
+   Python 3.8.3 (default, May 19 2020, 18:47:26) 
+   Type 'copyright', 'credits' or 'license' for more information
+   IPython 7.17.0 -- An enhanced Interactive Python. Type '?' for help.
+
+
+   Spack version 0.16.0
+   Python 3.8.3, Linux x86_64
+
+   In [1]:
+
+
+With either interpreter you can run a single command:
 
 .. code-block:: console
 
    $ spack python -c 'import distro; distro.linux_distribution()'
-   ('Fedora', '25', 'Workstation Edition')
+   ('Ubuntu', '18.04', 'Bionic Beaver')
+
+   $ spack python -i ipython -c 'import distro; distro.linux_distribution()'
+   Out[1]: ('Ubuntu', '18.04', 'Bionic Beaver')
 
 or a file:
 
 .. code-block:: console
 
    $ spack python ~/test_fetching.py
+   $ spack python -i ipython ~/test_fetching.py
+
+just like you would with the normal ``python`` command. 
 
-just like you would with the normal ``python`` command.
 
 .. _cmd-spack-url:
 

lib/spack/docs/environments.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -441,8 +441,8 @@ Environments can include files with either relative or absolute
 paths. Inline configurations take precedence over included
 configurations, so you don't have to change shared configuration files
 to make small changes to an individual Environment. Included configs
-listed later will have higher precedence, as the included configs are
-applied in order.
+listed earlier will have higher precedence, as the included configs are
+applied in reverse order.
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Manually Editing the Specs List

lib/spack/docs/extensions.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/features.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/getting_started.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -75,6 +75,14 @@ shell integration for :ref:`certain commands <packaging-shell-support>`,
 If you do not want to use Spack's shell support, you can always just run
 the ``spack`` command directly from ``spack/bin/spack``.
 
+When the ``spack`` command is executed it searches for an appropriate
+Python interpreter to use, which can be explicitly overridden by setting
+the ``SPACK_PYTHON`` environment variable.  When sourcing the appropriate shell
+setup script, ``SPACK_PYTHON`` will be set to the interpreter found at
+sourcing time, ensuring future invocations of the ``spack`` command will
+continue to use the same consistent python version regardless of changes in
+the environment.
+
 
 ^^^^^^^^^^^^^^^^^^
 Check Installation

lib/spack/docs/index.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/known_issues.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/mirrors.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/module_file_support.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/package_list.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/packaging_guide.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -612,6 +612,62 @@ it executable, then runs it with some arguments.
        installer = Executable(self.stage.archive_file)
        installer('--prefix=%s' % prefix, 'arg1', 'arg2', 'etc.')
 
+
+^^^^^^^^^^^^^^^^^^^^^^^^
+Deprecating old versions
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+There are many reasons to remove old versions of software:
+
+#. Security vulnerabilities (most serious reason)
+#. Changing build systems that increase package complexity
+#. Changing dependencies/patches/resources/flags that increase package complexity
+#. Maintainer/developer inability/unwillingness to support old versions
+#. No longer available for download (right to be forgotten)
+#. Package or version rename
+
+At the same time, there are many reasons to keep old versions of software:
+
+#. Reproducibility
+#. Requirements for older packages (e.g. some packages still rely on Qt 3)
+
+In general, you should not remove old versions from a ``package.py``. Instead,
+you should first deprecate them using the following syntax:
+
+.. code-block:: python
+
+   version('1.2.3', sha256='...', deprecated=True)
+
+
+This has two effects. First, ``spack info`` will no longer advertise that
+version. Second, commands like ``spack install`` that fetch the package will
+require user approval:
+
+.. code-block:: console
+
+   $ spack install openssl@1.0.1e
+   ==> Warning: openssl@1.0.1e is deprecated and may be removed in a future Spack release.
+   ==>   Fetch anyway? [y/N]
+
+
+If you use ``spack install --deprecated``, this check can be skipped.
+
+This also applies to package recipes that are renamed or removed. You should
+first deprecate all versions before removing a package. If you need to rename
+it, you can deprecate the old package and create a new package at the same
+time.
+
+Version deprecations should always last at least one Spack minor release cycle
+before the version is completely removed. For example, if a version is
+deprecated in Spack 0.16.0, it should not be removed until Spack 0.17.0. No
+version should be removed without such a deprecation process. This gives users
+a chance to complain about the deprecation in case the old version is needed
+for some application. If you require a deprecated version of a package, simply
+submit a PR to remove ``deprecated=True`` from the package. However, you may be
+asked to help maintain this version of the package if the current maintainers
+are unwilling to support this older version.
+
+
 ^^^^^^^^^^^^^^^^
 Download caching
 ^^^^^^^^^^^^^^^^
@@ -1864,14 +1920,21 @@ of ``mpich`` using the following command:
 
    $ srun -N 2 -n 8 spack install -j 4 mpich@3.3.2
 
-This will create eight concurrent four-job installation on two different
+This will create eight concurrent, four-job installs on two different
 nodes.
 
+Alternatively, you could run the same installs on one node by entering
+the following at the command line of a bash shell:
+
+.. code-block:: console
+
+   $ for i in {1..12}; do nohup spack install -j 4 mpich@3.3.2 >> mpich_install.txt 2>&1 &; done
+
 .. note::
 
-   The effective parallelism will be based on the maximum number of
-   packages that can be installed at the same time, which will limited
-   by the number of packages with no (remaining) uninstalled dependencies.
+   The effective parallelism is based on the maximum number of packages
+   that can be installed at the same time, which is limited by the
+   number of packages with no (remaining) uninstalled dependencies.
 
 
 .. _dependencies:
@@ -2737,12 +2800,14 @@ The classes that are currently provided by Spack are:
     |                               | built using CMake                |
     +-------------------------------+----------------------------------+
     | :py:class:`.CudaPackage`      | A helper class for packages that |
-    |                               | use CUDA. It is intended to be   |
-    |                               | used in combination with others  |
+    |                               | use CUDA                         |
     +-------------------------------+----------------------------------+
     | :py:class:`.QMakePackage`     | Specialized class for packages   |
     |                               | build using QMake                |
     +-------------------------------+----------------------------------+
+    | :py:class:`.ROCmPackage`      | A helper class for packages that |
+    |                               | use ROCm                         |
+    +-------------------------------+----------------------------------+
     | :py:class:`.SConsPackage`     | Specialized class for packages   |
     |                               | built using SCons                |
     +-------------------------------+----------------------------------+
@@ -3946,7 +4011,7 @@ using the ``run_before`` decorator.
 
     The API for adding tests is not yet considered stable and may change drastically in future releases.
 
-.. _file-manipulation:
+.. _cmd-spack-test:
 
 ^^^^^^^^^^^^^
 Install Tests
@@ -3956,109 +4021,205 @@ Install Tests
 
    The API for adding and running install tests is not yet considered
    stable and may change drastically in future releases. Packages with
-   upstreamed tests will be refactored to match changes to the API.
+   install tests will be refactored to match changes to the API.
 
-While build-tests are integrated with the build system, install tests
-may be added to Spack packages to be run independently of the install
-method.
+While build tests are integrated with the build system, install tests
+are for testing package installations independent of the build process.
 
-Install tests may be added by defining a ``test`` method with the following signature:
+Install tests are executed in a test stage directory that defaults to
+``~/.spack/test``. You can change the location in the high-level ``config``
+by adding the ``test_stage`` path as follows:
+
+.. code-block:: yaml
+
+   config:
+     test_stage: /path/to/stage
+
+""""""""""""""""""""
+Adding install tests
+""""""""""""""""""""
+
+Install tests are added to a package by defining a ``test`` method
+with the following signature:
 
 .. code-block:: python
 
    def test(self):
 
-These tests will be run in an environment set up to provide access to
-this package and all of its dependencies, including ``test``-type
-dependencies. Inside the ``test`` method, standard python ``assert``
-statements and other error reporting mechanisms can be used. Spack
-will report any errors as a test failure.
+These tests run in an environment that provides access to the
+package and all of its dependencies, including ``test``-type
+dependencies.
 
-Inside the test method, individual tests can be run separately (and
-continue transparently after a test failure) using the ``run_test``
-method. The signature for the ``run_test`` method is:
+Standard python ``assert`` statements and other error reporting
+mechanisms can be used in the ``test`` method. Spack will report
+such errors as test failures.
+
+You can implement multiple tests (or test parts) within the ``test``
+method, where each is run separately and testing continues after
+failures by using the ``run_test`` method. The signature for the
+method is:
 
 .. code-block:: python
 
    def run_test(self, exe, options=[], expected=[], status=0, installed=False,
                 purpose='', skip_missing=False, work_dir=None):
 
-This method will operate in ``work_dir`` if one is specified. It will
-search for an executable in the ``PATH`` variable named ``exe``, and
-if ``installed=True`` it will fail if that executable does not come
-from the prefix of the package being tested. If the executable is not
-found, it will fail the test unless ``skip_missing`` is set to
-``True``. The executable will be run with the options specified, and
-the return code will be checked against the ``status`` argument, which
-can be an integer or list of integers. Spack will also check that
-every string in ``expected`` is a regex matching part of the output of
-the executable. The ``purpose`` argument is recorded in the test log
-for debugging purposes.
+The test fails if there is no executable named ``exe`` found in the
+paths of the ``PATH`` variable **unless** ``skip_missing`` is ``True``.
+The test also fails if the resulting path is not within the prefix of
+the package being tested when ``installed`` is ``True``.
 
-""""""""""""""""""""""""""""""""""""""
-Install tests that require compilation
-""""""""""""""""""""""""""""""""""""""
+The executable runs in ``work_dir``, when specified, using the provided
+``options``. The return code is checked against the ``status`` argument,
+which can be an integer or list of integers representing status codes
+corresponding to successful execution (e.g. ``status=[0,3,7]``).
+Spack also checks that every string in ``expected`` is a regex matching
+part of the output from the test run (e.g.
+``expected=['completed successfully', 'converged in']``). Default behavior
+is to behave as though ``status=[0]`` and ``expected=[]`` are specified.
+
+Output from the test is written to its log file. The ``purpose`` argument
+serves as the heading in text logs to highlight the start of each test part.
+
+"""""""""""""""""""""""""
+Enabling test compilation
+"""""""""""""""""""""""""
 
 Some tests may require access to the compiler with which the package
-was built, especially to test library-only packages. To ensure the
-compiler is configured as part of the test environment, set the
-attribute ``tests_require_compiler = True`` on the package. The
-compiler will be available through the canonical environment variables
-(``CC``, ``CXX``, ``FC``, ``F77``) in the test environment.
+was built, especially to test library-only packages. You must enable
+loading the package's compiler configuration by setting the attribute
+``test_requires_compiler`` to ``True``. Doing so makes the compiler
+available in the test environment through the canonical environment
+variables (``CC``, ``CXX``, ``FC``, ``F77``).
 
-""""""""""""""""""""""""""""""""""""""""""""""""
-Install tests that require build-time components
-""""""""""""""""""""""""""""""""""""""""""""""""
+"""""""""""""""""""""""
+Adding build-time files
+"""""""""""""""""""""""
 
-Some packages cannot be easily tested without components from the
-build-time test suite. For those packages, the
-``cache_extra_test_sources`` method can be used.
+.. note::
+
+    We highly recommend the re-use of build-time tests and input files
+    for testing installed software.  These files are easier to keep
+    synchronized with the software than creating custom install tests.
+
+You can use the ``cache_extra_test_sources`` method copy directories
+and or files from the build stage directory to the package's installation
+directory.
+
+For example, a package method for copying everything in the ``tests``
+subdirectory plus the ``foo.c`` and ``bar.c`` files from ``examples``
+can be implemented as follows:
 
 .. code-block:: python
 
    @run_after('install')
    def cache_test_sources(self):
-       srcs = ['./tests/foo.c', './tests/bar.c']
+       srcs = ['tests', 'examples/foo.c', 'examples/bar.c']
        self.cache_extra_test_sources(srcs)
 
-This method will copy the listed methods into the metadata directory
-of the package at the end of the install phase of the build. They will
-be available to the test method in the directory
-``self._extra_tests_path``.
+The use of the ``run_after`` directive ensures the associated files
+are copied **after** the package is installed during the build process.
 
-While source files are generally recommended, for many packages
-binaries may also technically be cached in this way for later testing.
+The method copies files to the package's metadata directory under
+the ``self.install_test_root``. All files in the package source's
+``tests`` directory for the example above will be copied to the
+``join_path(self.install_test_root, 'tests')`` directory. The two
+files listed under the staged ``examples`` directory will be copied
+to the ``join_path(self.install_test_root, 'examples')`` directory.
+
+.. note::
+
+    While source and input files are generally recommended, binaries
+    may also be cached by the build process for install testing.
+
+"""""""""""""""""""
+Adding custom files
+"""""""""""""""""""
+
+Some tests may require additional files not available from the build.
+Examples include:
+
+- test source files
+- test input files
+- test build scripts
+- expected test output
+
+These extra files should be added to the ``test`` subdirectory of the
+package in the repository. Spack will automaticaly copy any files in
+that directory to the test staging directory during install testing.
+The ``test`` method can access those files from the
+``self.test_suite.current_test_data_dir`` directory.
+
+.. _cmd-spack-test-list:
+
+"""""""""""""""""""
+``spack test list``
+"""""""""""""""""""
+
+Packages available for install testing can be found using the
+``spack test list`` command. The command outputs all installed
+packages that have defined ``test`` methods.
+
+.. _cmd-spack-test-run:
+
+""""""""""""""""""
+``spack test run``
+""""""""""""""""""
+
+Install tests can be run for one or more installed packages using
+the ``spack test run`` command. A ``test suite`` is created from
+the provided specs.  If no specs are provided it will test all specs
+in the active environment or all specs installed in Spack if no
+environment is active.
+
+Test suites can be named using the ``--alias`` option. Unaliased
+Test suites will use the content hash of their specs as their name.
+
+Some of the more commonly used debugging options are:
+
+- ``--fail-fast`` stops testing each package after the first failure
+- ``--fail-first`` stops testing packages after the first failure
+
+Test output is written to a text log file by default but ``junit``
+and ``cdash`` are outputs are available through the ``--log-format``
+option.
+
+.. _cmd-spack-test-results:
+
+""""""""""""""""""""""
+``spack test results``
+""""""""""""""""""""""
+
+The ``spack test results`` command shows results for all completed
+test suites. Providing the alias or content hash limits reporting
+to the corresponding test suite.
+
+The ``--logs`` option includes the output generated by the associated
+test(s) to facilitate debugging.
+
+The ``--failed`` option limits results shown to that of the failed
+tests, if any, of matching packages.
+
+.. _cmd-spack-test-find:
+
+"""""""""""""""""""
+``spack test find``
+"""""""""""""""""""
+
+The ``spack test find`` command lists the aliases or content hashes
+of all test suites whose results are available.
+
+.. _cmd-spack-test-remove:
 
 """""""""""""""""""""
-Running install tests
+``spack test remove``
 """""""""""""""""""""
 
-Install tests can be run using the ``spack test run`` command. The
-``spack test run`` command will create a ``test suite`` out of the
-specs provided to it, or if no specs are provided it will test all
-specs in the active environment, or all specs installed in Spack if no
-environment is active. Test suites can be named using the ``--alias``
-option; test suites not aliased will use the content hash of their
-specs as their name.
+The ``spack test remove`` command removes test suites to declutter
+the test results directory. You are prompted to confirm the removal
+of each test suite **unless** you use the ``--yes-to-all`` option.
 
-Packages to install test can be queried using the ``spack test list``
-command, which outputs all installed packages with defined ``test``
-methods.
-
-Test suites can be found using the ``spack test find`` command. It
-will list all test suites that have been run and have not been removed
-using the ``spack test remove`` command. The ``spack test remove``
-command will remove tests to declutter the test stage. The ``spack
-test results`` command will show results for completed test suites.
-
-The test stage is the working directory for all install tests run with
-Spack. By default, Spack uses ``~/.spack/test`` as the test stage. The
-test stage can be set in the high-level config:
-
-.. code-block:: yaml
-
-   config:
-     test_stage: /path/to/stage
+.. _file-manipulation:
 
 ---------------------------
 File manipulation functions

lib/spack/docs/pipelines.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -122,9 +122,26 @@ pipeline jobs.
 Concretizes the specs in the active environment, stages them (as described in
 :ref:`staging_algorithm`), and writes the resulting ``.gitlab-ci.yml`` to disk.
 
-This sub-command takes two arguments, but the most useful is ``--output-file``,
-which should be an absolute path (including file name) to the generated
-pipeline, if the default (``./.gitlab-ci.yml``) is not desired.
+Using ``--prune-dag`` or ``--no-prune-dag`` configures whether or not jobs are
+generated for specs that are already up to date on the mirror.   If enabling
+DAG pruning using ``--prune-dag``, more information may be required in your
+``spack.yaml`` file, see the :ref:`noop_jobs` section below regarding
+``service-job-attributes``.
+
+The ``--optimize`` argument is experimental and runs the generated pipeline
+document through a series of optimization passes designed to reduce the size
+of the generated file.
+
+The ``--dependencies`` is also experimental and disables what in Gitlab is
+referred to as DAG scheduling, internally using the ``dependencies`` keyword
+rather than ``needs`` to list dependency jobs.  The drawback of using this option
+is that before any job can begin, all jobs in previous stages must first
+complete.  The benefit is that Gitlab allows more dependencies to be listed
+when using ``dependencies`` instead of ``needs``.
+
+The optional ``--output-file`` argument should be an absolute path (including
+file name) to the generated pipeline, and if not given, the default is
+``./.gitlab-ci.yml``.
 
 .. _cmd-spack-ci-rebuild:
 
@@ -223,21 +240,6 @@ takes a boolean and determines whether the pipeline uses artifacts to store and
 pass along the buildcaches from one stage to the next (the default if you don't
 provide this option is ``False``).
 
-The
-``final-stage-rebuild-index`` section controls whether an extra job is added to the
-end of your pipeline (in a stage by itself) which will regenerate the mirror's
-buildcache index.  Under normal operation, each pipeline job that rebuilds a package
-will re-generate the mirror's buildcache index after the buildcache entry for that
-job has been created and pushed to the mirror.  Since jobs in the same stage can run in
-parallel, there is the possibility that at the end of some stage, the index may not
-reflect all the binaries in the buildcache.  Adding the ``final-stage-rebuild-index``
-section ensures that at the end of the pipeline, the index will be in sync with the
-binaries on the mirror.  If the mirror lives in an S3 bucket, this job will need to
-run on a machine with the Python ``boto3`` module installed, and consequently the
-``final-stage-rebuild-index`` needs to specify a list of ``tags`` to pick a runner
-satisfying that condition.  It can also take an ``image`` key so Docker executor type
-runners can pick the right image for the index regeneration job.
-
 The optional ``cdash`` section provides information that will be used by the
 ``spack ci generate`` command (invoked by ``spack ci start``) for reporting
 to CDash.  All the jobs generated from this environment will belong to a
@@ -251,6 +253,76 @@ Take a look at the
 for the gitlab-ci section of the spack environment file, to see precisely what
 syntax is allowed there.
 
+.. _rebuild_index:
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Note about rebuilding buildcache index
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+By default, while a pipeline job may rebuild a package, create a buildcache
+entry, and push it to the mirror, it does not automatically re-generate the
+mirror's buildcache index afterward.  Because the index is not needed by the
+default rebuild jobs in the pipeline, not updating the index at the end of
+each job avoids possible race conditions between simultaneous jobs, and it
+avoids the computational expense of regenerating the index.  This potentially
+saves minutes per job, depending on the number of binary packages in the
+mirror.  As a result, the default is that the mirror's buildcache index may
+not correctly reflect the mirror's contents at the end of a pipeline.
+
+To make sure the buildcache index is up to date at the end of your pipeline,
+spack generates a job to update the buildcache index of the target mirror
+at the end of each pipeline by default.  You can disable this behavior by
+adding ``rebuild-index: False`` inside the ``gitlab-ci`` section of your
+spack environment.  Spack will assign the job any runner attributes found
+on the ``service-job-attributes``, if you have provided that in your
+``spack.yaml``.
+
+.. _noop_jobs:
+
+^^^^^^^^^^^^^^^^^^^^^^^
+Note about "no-op" jobs
+^^^^^^^^^^^^^^^^^^^^^^^
+
+If no specs in an environment need to be rebuilt during a given pipeline run
+(meaning all are already up to date on the mirror), a single succesful job
+(a NO-OP) is still generated to avoid an empty pipeline (which GitLab
+considers to be an error).  An optional ``service-job-attributes`` section
+can be added to your ``spack.yaml`` where you can provide ``tags`` and
+``image`` or ``variables`` for the generated NO-OP job.  This section also
+supports providing ``before_script``, ``script``, and ``after_script``, in
+case you want to take some custom actions in the case of any empty pipeline.
+
+Following is an example of this section added to a ``spack.yaml``:
+
+.. code-block:: yaml
+
+   spack:
+     specs:
+       - openmpi
+     mirrors:
+       cloud_gitlab: https://mirror.spack.io
+     gitlab-ci:
+       mappings:
+         - match:
+             - os=centos8
+           runner-attributes:
+             tags:
+               - custom
+               - tag
+             image: spack/centos7
+       service-job-attributes:
+         tags: ['custom', 'tag']
+         image:
+           name: 'some.image.registry/custom-image:latest'
+           entrypoint: ['/bin/bash']
+         script:
+           - echo "Custom message in a custom script"
+
+The example above illustrates how you can provide the attributes used to run
+the NO-OP job in the case of an empty pipeline.  The only field for the NO-OP
+job that might be generated for you is ``script``, but that will only happen
+if you do not provide one yourself.
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Assignment of specs to runners
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

lib/spack/docs/repositories.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -9,7 +9,7 @@
 Package Repositories
 =============================
 
-Spack comes with over 1,000 built-in package recipes in
+Spack comes with thousands of built-in package recipes in
 ``var/spack/repos/builtin/``.  This is a **package repository** -- a
 directory that Spack searches when it needs to find a package by name.
 You may need to maintain packages for restricted, proprietary or

lib/spack/docs/spack.yaml

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/workflows.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/env/cc

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -125,7 +125,7 @@ case "$command" in
         comp="FC"
         lang_flags=F
         ;;
-    f77|xlf|xlf_r|pgf77|frt|flang)
+    f77|xlf|xlf_r|pgf77)
         command="$SPACK_F77"
         language="Fortran 77"
         comp="F77"
@@ -320,9 +320,13 @@ while [ -n "$1" ]; do
             if [ -z "$arg" ]; then shift; arg="$1"; fi
             if [[ "$arg" = -rpath=* ]]; then
                 rp="${arg#-rpath=}"
+            elif [[ "$arg" = --rpath=* ]]; then
+                rp="${arg#--rpath=}"
             elif [[ "$arg" = -rpath,* ]]; then
                 rp="${arg#-rpath,}"
-            elif [[ "$arg" = -rpath ]]; then
+            elif [[ "$arg" = --rpath,* ]]; then
+                rp="${arg#--rpath,}"
+            elif [[ "$arg" =~ ^-?-rpath$ ]]; then
                 shift; arg="$1"
                 if [[ "$arg" != -Wl,* ]]; then
                     die "-Wl,-rpath was not followed by -Wl,*"
@@ -339,7 +343,9 @@ while [ -n "$1" ]; do
             if [ -z "$arg" ]; then shift; arg="$1"; fi
             if [[ "$arg" = -rpath=* ]]; then
                 rp="${arg#-rpath=}"
-            elif [[ "$arg" = -rpath ]]; then
+            elif [[ "$arg" = --rpath=* ]]; then
+                rp="${arg#--rpath=}"
+            elif [[ "$arg" = -rpath  ]] || [[ "$arg" = --rpath ]]; then
                 shift; arg="$1"
                 if [[ "$arg" != -Xlinker,* ]]; then
                     die "-Xlinker,-rpath was not followed by -Xlinker,*"
@@ -434,7 +440,7 @@ then
         ld)
             flags=("${flags[@]}" -headerpad_max_install_names) ;;
         ccld)
-            flags=("${flags[@]}" -Wl,-headerpad_max_install_names) ;;
+            flags=("${flags[@]}" "-Wl,-headerpad_max_install_names") ;;
     esac
 fi
 
@@ -491,19 +497,19 @@ args+=("${flags[@]}")
 # Insert include directories just prior to any system include directories
 
 for dir in "${includes[@]}";         do args+=("-I$dir"); done
-for dir in "${isystem_includes[@]}";         do args+=("-isystem$dir"); done
+for dir in "${isystem_includes[@]}";         do args+=("-isystem" "$dir"); done
 
 IFS=':' read -ra spack_include_dirs <<< "$SPACK_INCLUDE_DIRS"
 if [[ $mode == cpp || $mode == cc || $mode == as || $mode == ccld ]]; then
     if [[ "$isystem_was_used" == "true" ]] ; then
-	for dir in "${spack_include_dirs[@]}";  do args+=("-isystem$dir"); done
+	for dir in "${spack_include_dirs[@]}";  do args+=("-isystem" "$dir"); done
     else
 	for dir in "${spack_include_dirs[@]}";  do args+=("-I$dir"); done
     fi
 fi
 
 for dir in "${system_includes[@]}";  do args+=("-I$dir"); done
-for dir in "${isystem_system_includes[@]}";  do args+=("-isystem$dir"); done
+for dir in "${isystem_system_includes[@]}";  do args+=("-isystem" "$dir"); done
 
 # Library search paths
 for dir in "${libdirs[@]}";          do args+=("-L$dir"); done
@@ -512,12 +518,12 @@ for dir in "${system_libdirs[@]}";   do args+=("-L$dir"); done
 # RPATHs arguments
 case "$mode" in
     ccld)
-        if [ ! -z "$dtags_to_add" ] ; then args+=("$linker_arg$dtags_to_add") ; fi
+        if [ -n "$dtags_to_add" ] ; then args+=("$linker_arg$dtags_to_add") ; fi
         for dir in "${rpaths[@]}";        do args+=("$rpath$dir"); done
         for dir in "${system_rpaths[@]}"; do args+=("$rpath$dir"); done
         ;;
     ld)
-        if [ ! -z "$dtags_to_add" ] ; then args+=("$dtags_to_add") ; fi
+        if [ -n "$dtags_to_add" ] ; then args+=("$dtags_to_add") ; fi
         for dir in "${rpaths[@]}";        do args+=("-rpath" "$dir"); done
         for dir in "${system_rpaths[@]}"; do args+=("-rpath" "$dir"); done
         ;;

lib/spack/external/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -11,7 +11,7 @@
 
 * Homepage: https://pypi.python.org/pypi/archspec
 * Usage: Labeling, comparison and detection of microarchitectures
-* Version: 0.1.2 (commit 2846749dc5b12ae2b30ff1d3f0270a4a5954710d)
+* Version: 0.1.2 (commit 068b0ebd641211971acf10f39aa876703a34bae4)
 
 argparse
 --------

lib/spack/external/_pytest/assertion/util.py

@@ -5,9 +5,12 @@
 import _pytest._code
 import py
 try:
-    from collections import Sequence
+    from collections.abc import Sequence
 except ImportError:
-    Sequence = list
+    try:
+        from collections import Sequence
+    except ImportError:
+        Sequence = list
 
 
 u = py.builtin._totext

lib/spack/external/_pytest/main.py

@@ -10,9 +10,12 @@
 import _pytest._code
 import py
 try:
-    from collections import MutableMapping as MappingMixin
+    from collections.abc import MutableMapping as MappingMixin
 except ImportError:
-    from UserDict import DictMixin as MappingMixin
+    try:
+        from collections import MutableMapping as MappingMixin
+    except ImportError:
+        from UserDict import DictMixin as MappingMixin
 
 from _pytest.config import directory_arg, UsageError, hookimpl
 from _pytest.outcomes import exit

lib/spack/external/_pytest/python_api.py

@@ -398,7 +398,10 @@ def approx(expected, rel=None, abs=None, nan_ok=False):
        __ https://docs.python.org/3/reference/datamodel.html#object.__ge__
     """
 
-    from collections import Mapping, Sequence
+    if sys.version_info >= (3, 3):
+        from collections.abc import Mapping, Sequence
+    else:
+        from collections import Mapping, Sequence
     from _pytest.compat import STRING_TYPES as String
 
     # Delegate the comparison to a class that knows how to deal with the type

lib/spack/external/archspec/__init__.py

@@ -1,2 +1,2 @@
 """Init file to avoid namespace packages"""
-__version__ = "0.1.1"
+__version__ = "0.1.2"

lib/spack/external/archspec/json/cpu/microarchitectures.json

@@ -75,6 +75,13 @@
             "flags": "-march={name} -mtune=generic"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "name": "x86-64",
+            "flags": "-march={name} -mtune=generic"
+          }
+        ],
         "intel": [
           {
             "versions": ":",
@@ -106,6 +113,12 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "flags": "-march={name} -mtune=generic"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "8.0:",
@@ -143,6 +156,12 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "flags": "-march={name} -mtune=generic"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "8.0:",
@@ -152,7 +171,7 @@
         "intel": [
           {
             "versions": "16.0:",
-            "flags": "-march={name} -mtune={name}}"
+            "flags": "-march={name} -mtune={name}"
           }
         ]
       }
@@ -187,6 +206,12 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "flags": "-march={name} -mtune=generic"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "8.0:",
@@ -229,6 +254,12 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "flags": "-march={name} -mtune=generic"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "8.0:",
@@ -277,6 +308,12 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "8.0:",
@@ -331,6 +368,12 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "8.0:",
@@ -390,6 +433,12 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "8.0:",
@@ -446,6 +495,12 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "8.0:",
@@ -500,6 +555,12 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "8.0:",
@@ -558,6 +619,13 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "name": "knl",
+            "flags": "-march={name} -mtune=generic"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "8.0:",
@@ -621,6 +689,13 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "name": "skylake-avx512",
+            "flags": "-march={name} -mtune=generic"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "8.0:",
@@ -685,6 +760,12 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "8.0:",
@@ -746,6 +827,12 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "11.0:",
@@ -827,6 +914,13 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "name": "icelake-client",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "apple-clang": [
           {
             "versions": "10.0.1:",
@@ -875,6 +969,13 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "name": "amdfam10",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "intel": [
           {
             "versions": "16.0:",
@@ -918,6 +1019,13 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "name": "bdver1",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "intel": [
           {
             "versions": "16.0:",
@@ -965,6 +1073,13 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "name": "bdver2",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "intel": [
           {
             "versions": "16.0:",
@@ -1013,6 +1128,13 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "name": "bdver3",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "intel": [
           {
             "versions": "16.0:",
@@ -1064,6 +1186,13 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "name": "bdver4",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "intel": [
           {
             "versions": "16.0:",
@@ -1119,6 +1248,13 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "name": "znver1",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "intel": [
           {
             "versions": "16.0:",
@@ -1175,6 +1311,13 @@
             "flags": "-march={name} -mtune={name}"
           }
         ],
+        "aocc": [
+          {
+            "versions": "2.2:",
+            "name": "znver2",
+            "flags": "-march={name} -mtune={name}"
+          }
+        ],
         "intel": [
           {
             "versions": "16.0:",

lib/spack/external/jinja2/runtime.py

@@ -315,10 +315,14 @@ def __repr__(self):
 
 # register the context as mapping if possible
 try:
-    from collections import Mapping
+    from collections.abc import Mapping
     Mapping.register(Context)
 except ImportError:
-    pass
+    try:
+        from collections import Mapping
+        Mapping.register(Context)
+    except ImportError:
+        pass
 
 
 class BlockReference(object):

lib/spack/external/jinja2/sandbox.py

@@ -14,7 +14,7 @@
 """
 import types
 import operator
-from collections import Mapping
+import sys
 from jinja2.environment import Environment
 from jinja2.exceptions import SecurityError
 from jinja2._compat import string_types, PY2
@@ -23,6 +23,11 @@
 from markupsafe import EscapeFormatter
 from string import Formatter
 
+if sys.version_info >= (3, 3):
+    from collections.abc import Mapping
+else:
+    from collections import Mapping
+
 
 #: maximum number of items a range may produce
 MAX_RANGE = 100000
@@ -79,7 +84,10 @@
     pass
 
 #: register Python 2.6 abstract base classes
-from collections import MutableSet, MutableMapping, MutableSequence
+if sys.version_info >= (3, 3):
+    from collections.abc import MutableSet, MutableMapping, MutableSequence
+else:
+    from collections import MutableSet, MutableMapping, MutableSequence
 _mutable_set_types += (MutableSet,)
 _mutable_mapping_types += (MutableMapping,)
 _mutable_sequence_types += (MutableSequence,)

lib/spack/external/jinja2/tests.py

@@ -10,11 +10,16 @@
 """
 import operator
 import re
-from collections import Mapping
+import sys
 from jinja2.runtime import Undefined
 from jinja2._compat import text_type, string_types, integer_types
 import decimal
 
+if sys.version_info >= (3, 3):
+    from collections.abc import Mapping
+else:
+    from collections import Mapping
+
 number_re = re.compile(r'^-?\d+(\.\d+)?$')
 regex_type = type(number_re)
 

lib/spack/external/jinja2/utils.py

@@ -482,10 +482,14 @@ def __reversed__(self):
 
 # register the LRU cache as mutable mapping if possible
 try:
-    from collections import MutableMapping
+    from collections.abc import MutableMapping
     MutableMapping.register(LRUCache)
 except ImportError:
-    pass
+    try:
+        from collections import MutableMapping
+        MutableMapping.register(LRUCache)
+    except ImportError:
+        pass
 
 
 def select_autoescape(enabled_extensions=('html', 'htm', 'xml'),

lib/spack/external/markupsafe/__init__.py

@@ -10,10 +10,15 @@
 """
 import re
 import string
-from collections import Mapping
+import sys
 from markupsafe._compat import text_type, string_types, int_types, \
      unichr, iteritems, PY2
 
+if sys.version_info >= (3, 3):
+    from collections.abc import Mapping
+else:
+    from collections import Mapping
+
 __version__ = "1.0"
 
 __all__ = ['Markup', 'soft_unicode', 'escape', 'escape_silent']

lib/spack/external/ordereddict_backport.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/external/py2/typing.py

@@ -0,0 +1,84 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""
+This is a fake set of symbols to allow spack to import typing in python
+versions where we do not support type checking (<3)
+"""
+Annotated = None
+Any = None
+Callable = None
+ForwardRef = None
+Generic = None
+Literal = None
+Optional = None
+Tuple = None
+TypeVar = None
+Union = None
+AbstractSet = None
+ByteString = None
+Container = None
+Hashable = None
+ItemsView = None
+Iterable = None
+Iterator = None
+KeysView = None
+Mapping = None
+MappingView = None
+MutableMapping = None
+MutableSequence = None
+MutableSet = None
+Sequence = None
+Sized = None
+ValuesView = None
+Awaitable = None
+AsyncIterator = None
+AsyncIterable = None
+Coroutine = None
+Collection = None
+AsyncGenerator = None
+AsyncContextManager = None
+Reversible = None
+SupportsAbs = None
+SupportsBytes = None
+SupportsComplex = None
+SupportsFloat = None
+SupportsInt = None
+SupportsRound = None
+ChainMap = None
+Dict = None
+List = None
+OrderedDict = None
+Set = None
+FrozenSet = None
+NamedTuple = None
+Generator = None
+AnyStr = None
+cast = None
+get_args = None
+get_origin = None
+get_type_hints = None
+no_type_check = None
+no_type_check_decorator = None
+NoReturn = None
+
+# these are the typing extension symbols
+ClassVar = None
+Final = None
+Protocol = None
+Type = None
+TypedDict = None
+ContextManager = None
+Counter = None
+Deque = None
+DefaultDict = None
+SupportsIndex = None
+final = None
+IntVar = None
+Literal = None
+NewType = None
+overload = None
+runtime_checkable = None
+Text = None
+TYPE_CHECKING = None

lib/spack/external/ruamel/yaml/comments.py

@@ -9,7 +9,12 @@
 a separate base
 """
 
-from collections import MutableSet
+import sys
+
+if sys.version_info >= (3, 3):
+    from collections.abc import MutableSet
+else:
+    from collections import MutableSet
 
 __all__ = ["CommentedSeq", "CommentedMap", "CommentedOrderedMap",
            "CommentedSet", 'comment_attrib', 'merge_attrib']

lib/spack/external/ruamel/yaml/compat.py

@@ -12,9 +12,12 @@
     from ruamel.ordereddict import ordereddict
 except:
     try:
-        from collections import OrderedDict
+        from collections.abc import OrderedDict
     except ImportError:
-        from ordereddict import OrderedDict
+        try:
+            from collections import OrderedDict
+        except ImportError:
+            from ordereddict import OrderedDict
     # to get the right name import ... as ordereddict doesn't do that
 
     class ordereddict(OrderedDict):

lib/spack/external/ruamel/yaml/constructor.py

@@ -3,7 +3,6 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
-import collections
 import datetime
 import base64
 import binascii
@@ -26,6 +25,12 @@
     from ruamel.yaml.scalarstring import *                           # NOQA
 
 
+if sys.version_info >= (3, 3):
+    from collections.abc import Hashable
+else:
+    from collections import Hashable
+
+
 __all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor',
            'ConstructorError', 'RoundTripConstructor']
 
@@ -163,7 +168,7 @@ def construct_mapping(self, node, deep=False):
             # keys can be list -> deep
             key = self.construct_object(key_node, deep=True)
             # lists are not hashable, but tuples are
-            if not isinstance(key, collections.Hashable):
+            if not isinstance(key, Hashable):
                 if isinstance(key, list):
                     key = tuple(key)
             if PY2:
@@ -175,7 +180,7 @@ def construct_mapping(self, node, deep=False):
                         "found unacceptable key (%s)" %
                         exc, key_node.start_mark)
             else:
-                if not isinstance(key, collections.Hashable):
+                if not isinstance(key, Hashable):
                     raise ConstructorError(
                         "while constructing a mapping", node.start_mark,
                         "found unhashable key", key_node.start_mark)
@@ -959,7 +964,7 @@ def construct_mapping(self, node, maptyp, deep=False):
             # keys can be list -> deep
             key = self.construct_object(key_node, deep=True)
             # lists are not hashable, but tuples are
-            if not isinstance(key, collections.Hashable):
+            if not isinstance(key, Hashable):
                 if isinstance(key, list):
                     key = tuple(key)
             if PY2:
@@ -971,7 +976,7 @@ def construct_mapping(self, node, maptyp, deep=False):
                         "found unacceptable key (%s)" %
                         exc, key_node.start_mark)
             else:
-                if not isinstance(key, collections.Hashable):
+                if not isinstance(key, Hashable):
                     raise ConstructorError(
                         "while constructing a mapping", node.start_mark,
                         "found unhashable key", key_node.start_mark)
@@ -1003,7 +1008,7 @@ def construct_setting(self, node, typ, deep=False):
             # keys can be list -> deep
             key = self.construct_object(key_node, deep=True)
             # lists are not hashable, but tuples are
-            if not isinstance(key, collections.Hashable):
+            if not isinstance(key, Hashable):
                 if isinstance(key, list):
                     key = tuple(key)
             if PY2:
@@ -1015,7 +1020,7 @@ def construct_setting(self, node, typ, deep=False):
                         "found unacceptable key (%s)" %
                         exc, key_node.start_mark)
             else:
-                if not isinstance(key, collections.Hashable):
+                if not isinstance(key, Hashable):
                     raise ConstructorError(
                         "while constructing a mapping", node.start_mark,
                         "found unhashable key", key_node.start_mark)

lib/spack/llnl/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/argparsewriter.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/filesystem.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -23,6 +23,12 @@
 from llnl.util.lang import dedupe, memoized
 from spack.util.executable import Executable
 
+
+if sys.version_info >= (3, 3):
+    from collections.abc import Sequence  # novm
+else:
+    from collections import Sequence
+
 __all__ = [
     'FileFilter',
     'FileList',
@@ -63,7 +69,8 @@
     'touchp',
     'traverse_tree',
     'unset_executable_mode',
-    'working_dir'
+    'working_dir',
+    'keep_modification_time'
 ]
 
 
@@ -1105,7 +1112,7 @@ def find(root, files, recursive=True):
 
     Parameters:
         root (str): The root directory to start searching from
-        files (str or collections.Sequence): Library name(s) to search for
+        files (str or Sequence): Library name(s) to search for
         recurse (bool, optional): if False search only root folder,
             if True descends top-down from the root. Defaults to True.
 
@@ -1168,7 +1175,7 @@ def _find_non_recursive(root, search_files):
 # Utilities for libraries and headers
 
 
-class FileList(collections.Sequence):
+class FileList(Sequence):
     """Sequence of absolute paths to files.
 
     Provides a few convenience methods to manipulate file paths.
@@ -1411,7 +1418,7 @@ def find_headers(headers, root, recursive=False):
     """
     if isinstance(headers, six.string_types):
         headers = [headers]
-    elif not isinstance(headers, collections.Sequence):
+    elif not isinstance(headers, Sequence):
         message = '{0} expects a string or sequence of strings as the '
         message += 'first argument [got {1} instead]'
         message = message.format(find_headers.__name__, type(headers))
@@ -1566,7 +1573,7 @@ def find_system_libraries(libraries, shared=True):
     """
     if isinstance(libraries, six.string_types):
         libraries = [libraries]
-    elif not isinstance(libraries, collections.Sequence):
+    elif not isinstance(libraries, Sequence):
         message = '{0} expects a string or sequence of strings as the '
         message += 'first argument [got {1} instead]'
         message = message.format(find_system_libraries.__name__,
@@ -1620,7 +1627,7 @@ def find_libraries(libraries, root, shared=True, recursive=False):
     """
     if isinstance(libraries, six.string_types):
         libraries = [libraries]
-    elif not isinstance(libraries, collections.Sequence):
+    elif not isinstance(libraries, Sequence):
         message = '{0} expects a string or sequence of strings as the '
         message += 'first argument [got {1} instead]'
         message = message.format(find_libraries.__name__, type(libraries))
@@ -1819,3 +1826,24 @@ def remove_directory_contents(dir):
                 os.unlink(entry)
             else:
                 shutil.rmtree(entry)
+
+
+@contextmanager
+def keep_modification_time(*filenames):
+    """
+    Context manager to keep the modification timestamps of the input files.
+    Tolerates and has no effect on non-existent files and files that are
+    deleted by the nested code.
+
+    Parameters:
+        *filenames: one or more files that must have their modification
+            timestamps unchanged
+    """
+    mtimes = {}
+    for f in filenames:
+        if os.path.exists(f):
+            mtimes[f] = os.path.getmtime(f)
+    yield
+    for f, mtime in mtimes.items():
+        if os.path.exists(f):
+            os.utime(f, (os.path.getatime(f), mtime))

lib/spack/llnl/util/lang.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -9,13 +9,18 @@
 import os
 import re
 import functools
-import collections
 import inspect
 from datetime import datetime, timedelta
 from six import string_types
 import sys
 
 
+if sys.version_info >= (3, 3):
+    from collections.abc import Hashable, MutableMapping  # novm
+else:
+    from collections import Hashable, MutableMapping
+
+
 # Ignore emacs backups when listing modules
 ignore_modules = [r'^\.#', '~$']
 
@@ -189,7 +194,7 @@ def memoized(func):
 
     @functools.wraps(func)
     def _memoized_function(*args):
-        if not isinstance(args, collections.Hashable):
+        if not isinstance(args, Hashable):
             # Not hashable, so just call the function.
             return func(*args)
 
@@ -264,7 +269,7 @@ def setter(name, value):
 
 
 @key_ordering
-class HashableMap(collections.MutableMapping):
+class HashableMap(MutableMapping):
     """This is a hashable, comparable dictionary.  Hash is performed on
        a tuple of the values in the dictionary."""
 

lib/spack/llnl/util/link_tree.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/lock.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/multiproc.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/tty/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/tty/colify.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/tty/color.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/tty/log.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -20,12 +20,17 @@
 from six import string_types
 from six import StringIO
 
+from typing import Optional  # novm
+from types import ModuleType  # novm
+
 import llnl.util.tty as tty
 
+termios = None  # type: Optional[ModuleType]
 try:
-    import termios
+    import termios as term_mod
+    termios = term_mod
 except ImportError:
-    termios = None
+    pass
 
 
 # Use this to strip escape sequences

lib/spack/llnl/util/tty/pty.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/abi.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/architecture.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -56,6 +56,7 @@
 attributes front_os and back_os. The operating system as described earlier,
 will be responsible for compiler detection.
 """
+import contextlib
 import functools
 import inspect
 import warnings
@@ -67,6 +68,8 @@
 from llnl.util.lang import memoized, list_modules, key_ordering
 
 import spack.compiler
+import spack.compilers
+import spack.config
 import spack.paths
 import spack.error as serr
 import spack.util.executable
@@ -235,18 +238,19 @@ class Platform(object):
         Will return a instance of it once it is returned.
     """
 
-    priority        = None   # Subclass sets number. Controls detection order
+    # Subclass sets number. Controls detection order
+    priority        = None   # type: int
 
     #: binary formats used on this platform; used by relocation logic
     binary_formats  = ['elf']
 
-    front_end       = None
-    back_end        = None
-    default         = None   # The default back end target.
+    front_end       = None   # type: str
+    back_end        = None   # type: str
+    default         = None   # type: str # The default back end target.
 
-    front_os        = None
-    back_os         = None
-    default_os      = None
+    front_os        = None   # type: str
+    back_os         = None   # type: str
+    default_os      = None   # type: str
 
     reserved_targets = ['default_target', 'frontend', 'fe', 'backend', 'be']
     reserved_oss = ['default_os', 'frontend', 'fe', 'backend', 'be']
@@ -491,7 +495,7 @@ def arch_for_spec(arch_spec):
 
 
 @memoized
-def all_platforms():
+def _all_platforms():
     classes = []
     mod_path = spack.paths.platform_path
     parent_module = "spack.platforms"
@@ -512,7 +516,7 @@ def all_platforms():
 
 
 @memoized
-def platform():
+def _platform():
     """Detects the platform for this machine.
 
     Gather a list of all available subclasses of platforms.
@@ -521,7 +525,7 @@ def platform():
     a file path (/opt/cray...)
     """
     # Try to create a Platform object using the config file FIRST
-    platform_list = all_platforms()
+    platform_list = _all_platforms()
     platform_list.sort(key=lambda a: a.priority)
 
     for platform_cls in platform_list:
@@ -529,6 +533,19 @@ def platform():
             return platform_cls()
 
 
+#: The "real" platform of the host running Spack. This should not be changed
+#: by any method and is here as a convenient way to refer to the host platform.
+real_platform = _platform
+
+#: The current platform used by Spack. May be swapped by the use_platform
+#: context manager.
+platform = _platform
+
+#: The list of all platform classes. May be swapped by the use_platform
+#: context manager.
+all_platforms = _all_platforms
+
+
 @memoized
 def default_arch():
     """Default ``Arch`` object for this machine.
@@ -563,3 +580,39 @@ def compatible_sys_types():
         arch = Arch(platform(), 'default_os', target)
         compatible_archs.append(str(arch))
     return compatible_archs
+
+
+class _PickleableCallable(object):
+    """Class used to pickle a callable that may substitute either
+    _platform or _all_platforms. Lambda or nested functions are
+    not pickleable.
+    """
+    def __init__(self, return_value):
+        self.return_value = return_value
+
+    def __call__(self):
+        return self.return_value
+
+
+@contextlib.contextmanager
+def use_platform(new_platform):
+    global platform, all_platforms
+
+    msg = '"{0}" must be an instance of Platform'
+    assert isinstance(new_platform, Platform), msg.format(new_platform)
+
+    original_platform_fn, original_all_platforms_fn = platform, all_platforms
+    platform = _PickleableCallable(new_platform)
+    all_platforms = _PickleableCallable([type(new_platform)])
+
+    # Clear configuration and compiler caches
+    spack.config.config.clear_caches()
+    spack.compilers._cache_config_files = []
+
+    yield new_platform
+
+    platform, all_platforms = original_platform_fn, original_all_platforms_fn
+
+    # Clear configuration and compiler caches
+    spack.config.config.clear_caches()
+    spack.compilers._cache_config_files = []

lib/spack/spack/binary_distribution.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -763,6 +763,10 @@ def generate_package_index(cache_prefix):
             url_util.join(cache_prefix, 'index.json.hash'),
             keep_original=False,
             extra_args={'ContentType': 'text/plain'})
+    except Exception as err:
+        msg = 'Encountered problem pushing package index to {0}: {1}'.format(
+            cache_prefix, err)
+        tty.warn(msg)
     finally:
         shutil.rmtree(tmpdir)
 
@@ -823,6 +827,10 @@ def generate_key_index(key_prefix, tmpdir=None):
                 url_util.join(key_prefix, 'index.json'),
                 keep_original=False,
                 extra_args={'ContentType': 'application/json'})
+        except Exception as err:
+            msg = 'Encountered problem pushing key index to {0}: {1}'.format(
+                key_prefix, err)
+            tty.warn(msg)
         finally:
             if remove_tmpdir:
                 shutil.rmtree(tmpdir)
@@ -1081,6 +1089,8 @@ def relocate_package(spec, allow_root):
     """
     Relocate the given package
     """
+    import spack.hooks.sbang as sbang
+
     workdir = str(spec.prefix)
     buildinfo = read_buildinfo_file(workdir)
     new_layout_root = str(spack.store.layout.root)
@@ -1119,9 +1129,7 @@ def relocate_package(spec, allow_root):
     prefix_to_prefix_bin = OrderedDict({})
 
     if old_sbang_install_path:
-        import spack.hooks.sbang as sbang
-        prefix_to_prefix_text[old_sbang_install_path] = \
-            sbang.sbang_install_path()
+        prefix_to_prefix_text[old_sbang_install_path] = sbang.sbang_install_path()
 
     prefix_to_prefix_text[old_prefix] = new_prefix
     prefix_to_prefix_bin[old_prefix] = new_prefix
@@ -1134,7 +1142,6 @@ def relocate_package(spec, allow_root):
     # sbang was a bash script, and it lived in the spack prefix. It is
     # now a POSIX script that lives in the install prefix. Old packages
     # will have the old sbang location in their shebangs.
-    import spack.hooks.sbang as sbang
     orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(old_spack_prefix)
     new_sbang = sbang.sbang_shebang_line()
     prefix_to_prefix_text[orig_sbang] = new_sbang
@@ -1153,7 +1160,7 @@ def is_backup_file(file):
         if not is_backup_file(text_name):
             text_names.append(text_name)
 
-# If we are not installing back to the same install tree do the relocation
+    # If we are not installing back to the same install tree do the relocation
     if old_layout_root != new_layout_root:
         files_to_relocate = [os.path.join(workdir, filename)
                              for filename in buildinfo.get('relocate_binaries')
@@ -1323,7 +1330,7 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
             os.remove(filename)
 
 
-def try_direct_fetch(spec, force=False, full_hash_match=False, mirrors=None):
+def try_direct_fetch(spec, full_hash_match=False, mirrors=None):
     """
     Try to find the spec directly on the configured mirrors
     """
@@ -1361,11 +1368,26 @@ def try_direct_fetch(spec, force=False, full_hash_match=False, mirrors=None):
     return found_specs
 
 
-def get_mirrors_for_spec(spec=None, force=False, full_hash_match=False,
-                         mirrors_to_check=None):
+def get_mirrors_for_spec(spec=None, full_hash_match=False,
+                         mirrors_to_check=None, index_only=False):
     """
     Check if concrete spec exists on mirrors and return a list
     indicating the mirrors on which it can be found
+
+    Args:
+        spec (Spec): The spec to look for in binary mirrors
+        full_hash_match (bool): If True, only includes mirrors where the spec
+            full hash matches the locally computed full hash of the ``spec``
+            argument.  If False, any mirror which has a matching DAG hash
+            is included in the results.
+        mirrors_to_check (dict): Optionally override the configured mirrors
+            with the mirrors in this dictionary.
+        index_only (bool): Do not attempt direct fetching of ``spec.yaml``
+            files from remote mirrors, only consider the indices.
+
+    Return:
+        A list of objects, each containing a ``mirror_url`` and ``spec`` key
+            indicating all mirrors where the spec can be found.
     """
     if spec is None:
         return []
@@ -1391,10 +1413,9 @@ def filter_candidates(candidate_list):
         results = filter_candidates(candidates)
 
     # Maybe we just didn't have the latest information from the mirror, so
-    # try to fetch directly.
-    if not results:
+    # try to fetch directly, unless we are only considering the indices.
+    if not results and not index_only:
         results = try_direct_fetch(spec,
-                                   force=force,
                                    full_hash_match=full_hash_match,
                                    mirrors=mirrors_to_check)
 

lib/spack/spack/build_environment.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -149,12 +149,17 @@ def clean_environment():
     # can affect how some packages find libraries.  We want to make
     # sure that builds never pull in unintended external dependencies.
     env.unset('LD_LIBRARY_PATH')
-    env.unset('LIBRARY_PATH')
-    env.unset('CPATH')
     env.unset('LD_RUN_PATH')
     env.unset('DYLD_LIBRARY_PATH')
     env.unset('DYLD_FALLBACK_LIBRARY_PATH')
 
+    # These vars affect how the compiler finds libraries and include dirs.
+    env.unset('LIBRARY_PATH')
+    env.unset('CPATH')
+    env.unset('C_INCLUDE_PATH')
+    env.unset('CPLUS_INCLUDE_PATH')
+    env.unset('OBJC_INCLUDE_PATH')
+
     # On Cray "cluster" systems, unset CRAY_LD_LIBRARY_PATH to avoid
     # interference with Spack dependencies.
     # CNL requires these variables to be set (or at least some of them,
@@ -412,8 +417,7 @@ def set_build_environment_variables(pkg, env, dirty):
     # directory.  Add that to the path too.
     env_paths = []
     compiler_specific = os.path.join(
-        spack.paths.build_env_path,
-        os.path.dirname(pkg.compiler.link_paths['cc']))
+        spack.paths.build_env_path, os.path.dirname(pkg.compiler.link_paths['cc']))
     for item in [spack.paths.build_env_path, compiler_specific]:
         env_paths.append(item)
         ci = os.path.join(item, 'case-insensitive')

lib/spack/spack/build_systems/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/build_systems/aspell_dict.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/build_systems/autotools.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -8,6 +8,7 @@
 import os.path
 from subprocess import PIPE
 from subprocess import check_call
+from typing import List  # novm
 
 import llnl.util.tty as tty
 import llnl.util.filesystem as fs
@@ -61,7 +62,7 @@ class AutotoolsPackage(PackageBase):
 
     #: Targets for ``make`` during the :py:meth:`~.AutotoolsPackage.build`
     #: phase
-    build_targets = []
+    build_targets = []  # type: List[str]
     #: Targets for ``make`` during the :py:meth:`~.AutotoolsPackage.install`
     #: phase
     install_targets = ['install']
@@ -75,7 +76,7 @@ class AutotoolsPackage(PackageBase):
     #: Set to true to force the autoreconf step even if configure is present
     force_autoreconf = False
     #: Options to be passed to autoreconf when using the default implementation
-    autoreconf_extra_args = []
+    autoreconf_extra_args = []  # type: List[str]
 
     #: If False deletes all the .la files in the prefix folder
     #: after the installation. If True instead it installs them.

lib/spack/spack/build_systems/cmake.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -8,6 +8,7 @@
 import os
 import platform
 import re
+from typing import List  # novm
 
 import spack.build_environment
 from llnl.util.filesystem import working_dir
@@ -74,7 +75,7 @@ class CMakePackage(PackageBase):
     #: system base class
     build_system_class = 'CMakePackage'
 
-    build_targets = []
+    build_targets = []  # type: List[str]
     install_targets = ['install']
 
     build_time_test_callbacks = ['check']

lib/spack/spack/build_systems/cuda.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)