update CHANGELOG.md for 0.14.1

Spack currently cannot run as a background process uninterrupted because some of the logging functions used in the install method (especially to create the dynamic verbosity toggle with the v key) cause the OS to issue a SIGTTOU to Spack when it's backgrounded.

This PR puts the necessary gatekeeping in place so that Spack doesn't do anything that will cause a signal to stop the process when operating as a background process.

.codecov.yml

@@ -4,13 +4,13 @@ coverage:
   range: 60...90
   status:
     project:
-      default: yes
+      default:
+        threshold: 0.3%
 
 ignore:
   - lib/spack/spack/test/.*
   - lib/spack/docs/.*
   - lib/spack/external/.*
   - share/spack/qa/.*
-  - share/spack/spack-completion.bash
 
 comment: off

.coveragerc

@@ -9,6 +9,7 @@ omit =
      lib/spack/spack/test/*
      lib/spack/docs/*
      lib/spack/external/*
+     share/spack/qa/*
 
 [report]
 # Regexes for lines to exclude from consideration

.gitattributes

@@ -0,0 +1 @@
+*.py diff=python

.github/actions/add-maintainers-as-reviewers/entrypoint.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

.github/workflows/linux_build_tests.yaml

@@ -0,0 +1,57 @@
+name: linux builds
+
+on:
+  push:
+    branches:
+    - master
+    - develop
+  pull_request:
+    branches:
+    - master
+    - develop
+    paths-ignore:
+      # Don't run if we only modified packages in the built-in repository
+      - 'var/spack/repos/builtin/**'
+      - '!var/spack/repos/builtin/packages/lz4/**'
+      - '!var/spack/repos/builtin/packages/mpich/**'
+      - '!var/spack/repos/builtin/packages/tut/**'
+      - '!var/spack/repos/builtin/packages/py-setuptools/**'
+      - '!var/spack/repos/builtin/packages/openjpeg/**'
+      - '!var/spack/repos/builtin/packages/r-rcpp/**'
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        package: [lz4, mpich, tut, py-setuptools, openjpeg, r-rcpp]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Cache ccache's store
+      uses: actions/cache@v1
+      with:
+        path: ~/.ccache
+        key: ccache-build-${{ matrix.package }}
+        restore-keys: |
+          ccache-build-${{ matrix.package }}
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Install System Packages
+      run: |
+        sudo apt-get -yqq install ccache gfortran perl perl-base r-base r-base-core r-base-dev findutils openssl libssl-dev libpciaccess-dev
+        R --version
+        perl --version
+    - name: Copy Configuration
+      run: |
+        ccache -M 300M && ccache -z
+        # Set up external deps for build tests, b/c they take too long to compile
+        cp share/spack/qa/configuration/*.yaml etc/spack/
+    - name: Run the build test
+      run: |
+        . share/spack/setup-env.sh
+        SPEC=${{ matrix.package }} share/spack/qa/run-build-tests
+        ccache -s

.github/workflows/minimum_python_versions.yaml

@@ -0,0 +1,30 @@
+name: python version check
+
+on:
+  push:
+    branches:
+      - master
+      - develop
+  pull_request:
+    branches:
+      - master
+      - develop
+jobs:
+  validate:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Setup Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Install Python Packages
+      run: |
+        pip install --upgrade pip
+        pip install --upgrade vermin
+    - name: Minimum Version (Spack's Core)
+      run: vermin --backport argparse -t=2.6- -t=3.5- -v lib/spack/spack/ lib/spack/llnl/ bin/
+    - name: Minimum Version (Repositories)
+      run: vermin --backport argparse -t=2.6- -t=3.5- -v var/spack/repos

.gitlab-ci.yml

@@ -1,14 +0,0 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-generate ci jobs:
-  script:
-    - "./bin/generate-gitlab-ci-yml.sh"
-  tags:
-    - "spack-pre-ci"
-  artifacts:
-    paths:
-      - ci-generation
-    when: always

.travis.yml

@@ -61,61 +61,10 @@ jobs:
       language: generic
       env: [ TEST_SUITE=unit, PYTHON_VERSION=2.7, COVERAGE=true ]
       if: type != pull_request
-# mpich (AutotoolsPackage)
-    - stage: 'build tests'
-      python: '2.7'
-      os: linux
-      language: python
-      env: [ TEST_SUITE=build, 'SPEC=mpich' ]
-# astyle (MakefilePackage)
-    - python: '3.8'
-      os: linux
-      language: python
-      env: [ TEST_SUITE=build, 'SPEC=astyle' ]
-# tut (WafPackage)
-    - python: '3.8'
-      os: linux
-      language: python
-      env: [ TEST_SUITE=build, 'SPEC=tut' ]
-# py-setuptools (PythonPackage)
-    - python: '3.8'
-      os: linux
-      language: python
-      env: [ TEST_SUITE=build, 'SPEC=py-setuptools' ]
-# perl-dbi (PerlPackage)
-#    - python: '3.8'
-#      os: linux
-#      language: python
-#      env: [ TEST_SUITE=build, 'SPEC=perl-dbi' ]
-# openjpeg (CMakePackage + external cmake)
-    - python: '3.8'
-      os: linux
-      language: python
-      env: [ TEST_SUITE=build, 'SPEC=openjpeg' ]
-# r-rcpp (RPackage + external R)
-    - python: '3.8'
-      os: linux
-      language: python
-      env: [ TEST_SUITE=build, 'SPEC=r-rcpp' ]
-# mpich (AutotoolsPackage)
-    - python: '3.8'
-      os: linux
-      language: python
-      env: [ TEST_SUITE=build, 'SPEC=mpich' ]
-    - python: '3.8'
-      stage: 'docker build'
-      os: linux
-      language: python
-      env: TEST_SUITE=docker
-  allow_failures:
-    - env: TEST_SUITE=docker
 
 stages:
   - 'style checks'
   - 'unit tests + documentation'
-  - 'build tests'
-  - name: 'docker build'
-    if: type = push AND branch IN (develop, master)
 
 
 #=============================================================================
@@ -159,7 +108,6 @@ cache:
   pip: true
   ccache: true
   directories:
-    - ~/.mirror
     - ~/.ccache
 
 # Work around Travis's lack of support for Python on OSX
@@ -177,7 +125,7 @@ install:
   - pip install --upgrade pip
   - pip install --upgrade six
   - pip install --upgrade setuptools
-  - pip install --upgrade codecov
+  - pip install --upgrade codecov coverage==4.5.4
   - pip install --upgrade flake8
   - pip install --upgrade pep8-naming
   - if [[ "$TEST_SUITE" == "doc" ]]; then
@@ -192,17 +140,9 @@ before_script:
   # Need this to be able to compute the list of changed files
   - git fetch origin ${TRAVIS_BRANCH}:${TRAVIS_BRANCH}
 
-  # Set up external deps for build tests, b/c they take too long to compile
-  - if [[ "$TEST_SUITE" == "build" ]]; then
-        cp share/spack/qa/configuration/*.yaml etc/spack/;
-    fi
-
 #=============================================================================
 # Building
 #=============================================================================
-services:
-  - docker
-
 script:
   - share/spack/qa/run-$TEST_SUITE-tests
 

CHANGELOG.md

@@ -1,3 +1,137 @@
+# v0.14.1 (2019-03-20)
+
+This is a bugfix release on top of `v0.14.0`.  Specific fixes include:
+
+* several bugfixes for parallel installation (#15339, #15341, #15220, #15197)
+* `spack load` now works with packages that have been renamed (#14348)
+* bugfix for `suite-sparse` installation (#15326)
+* deduplicate identical suffixes added to module names (#14920)
+* fix issues with `configure_args` during module refresh (#11084)
+* increased test coverage and test fixes (#15237, #15354, #15346)
+* remove some unused code (#15431)
+
+# v0.14.0 (2020-02-23)
+
+`v0.14.0` is a major feature release, with 3 highlighted features:
+
+1. **Distributed builds.** Multiple Spack instances will now coordinate
+   properly with each other through locks. This works on a single node
+   (where you've called `spack` several times) or across multiple nodes
+   with a shared filesystem. For example, with SLURM, you could build
+   `trilinos` and its dependencies on 2 24-core nodes, with 3 Spack
+   instances per node and 8 build jobs per instance, with `srun -N 2 -n 6
+   spack install -j 8 trilinos`. This requires a filesystem with locking
+   enabled, but not MPI or any other library for parallelism.
+
+2.  **Build pipelines.** You can also build in parallel through Gitlab
+   CI. Simply create a Spack environment and push it to Gitlab to build
+   on Gitlab runners. Pipeline support is now integreated into a single
+   `spack ci` command, so setting it up is easier than ever.  See the
+   [Pipelines section](https://spack.readthedocs.io/en/v0.14.0/pipelines.html)
+   in the docs.
+
+3. **Container builds.** The new `spack containerize` command allows you
+   to create a Docker or Singularity recipe from any Spack environment.
+   There are options to customize the build if you need them. See the
+   [Container Images section](https://spack.readthedocs.io/en/latest/containers.html)
+   in the docs.
+
+In addition, there are several other new commands, many bugfixes and
+improvements, and `spack load` no longer requires modules, so you can use
+it the same way on your laptop or on your supercomputer.
+
+Spack grew by over 300 packages since our last release in November 2019,
+and the project grew to over 500 contributors.  Thanks to all of you for
+making yet another great release possible. Detailed notes below.
+
+## Major new core features
+* Distributed builds: spack instances coordinate and build in parallel (#13100)
+* New `spack ci` command to manage CI pipelines (#12854)
+* Generate container recipes from environments: `spack containerize` (#14202)
+* `spack load` now works without using modules (#14062, #14628)
+* Garbage collect old/unused installations with `spack gc` (#13534)
+* Configuration files all set environment modifications the same way (#14372,
+  [docs](https://spack.readthedocs.io/en/v0.14.0/configuration.html#environment-modifications))
+* `spack commands --format=bash` auto-generates completion (#14393, #14607)
+* Packages can specify alternate fetch URLs in case one fails (#13881)
+
+## Improvements
+* Improved locking for concurrency with environments (#14676, #14621, #14692)
+* `spack test` sends args to `pytest`, supports better listing (#14319)
+* Better support for aarch64 and cascadelake microarch (#13825, #13780, #13820)
+* Archspec is now a separate library (see https://github.com/archspec/archspec)
+* Many improvements to the `spack buildcache` command (#14237, #14346,
+  #14466, #14467, #14639, #14642, #14659, #14696, #14698, #14714, #14732,
+  #14929, #15003, #15086, #15134)
+
+## Selected Bugfixes
+* Compilers now require an exact match on version (#8735, #14730, #14752)
+* Bugfix for patches that specified specific versions (#13989)
+* `spack find -p` now works in environments (#10019, #13972)
+* Dependency queries work correctly in `spack find` (#14757)
+* Bugfixes for locking upstream Spack instances chains (#13364)
+* Fixes for PowerPC clang optimization flags (#14196)
+* Fix for issue with compilers and specific microarchitectures (#13733, #14798)
+
+## New commands and options
+* `spack ci` (#12854)
+* `spack containerize` (#14202)
+* `spack gc` (#13534)
+* `spack load` accepts `--only package`, `--only dependencies` (#14062, #14628)
+* `spack commands --format=bash` (#14393)
+* `spack commands --update-completion` (#14607)
+* `spack install --with-cache` has new option: `--no-check-signature` (#11107)
+* `spack test` now has `--list`, `--list-long`, and `--list-names` (#14319)
+* `spack install --help-cdash` moves CDash help out of the main help (#13704)
+
+## Deprecations
+* `spack release-jobs` has been rolled into `spack ci`
+* `spack bootstrap` will be removed in a future version, as it is no longer
+  needed to set up modules (see `spack load` improvements above)
+
+## Documentation
+* New section on building container images with Spack (see
+  [docs](https://spack.readthedocs.io/en/latest/containers.html))
+* New section on using `spack ci` command to build pipelines (see
+  [docs](https://spack.readthedocs.io/en/latest/pipelines.html))
+* Document how to add conditional dependencies (#14694)
+* Document how to use Spack to replace Homebrew/Conda (#13083, see
+  [docs](https://spack.readthedocs.io/en/latest/workflows.html#using-spack-to-replace-homebrew-conda))
+
+## Important package changes
+* 3,908 total packages (345 added since 0.13.0)
+* Added first cut at a TensorFlow package (#13112)
+* We now build R without "recommended" packages, manage them w/Spack (#12015)
+* Elpa and OpenBLAS now leverage microarchitecture support (#13655, #14380)
+* Fix `octave` compiler wrapper usage (#14726)
+* Enforce that packages in `builtin` aren't missing dependencies (#13949)
+
+
+# v0.13.4 (2020-02-07)
+
+This release contains several bugfixes:
+
+* bugfixes for invoking python in various environments (#14349, #14496, #14569)
+* brought tab completion up to date (#14392)
+* bugfix for removing extensions from views in order (#12961)
+* bugfix for nondeterministic hashing for specs with externals (#14390)
+
+# v0.13.3 (2019-12-23)
+
+This release contains more major performance improvements for Spack
+environments, as well as bugfixes for mirrors and a `python` issue with
+RHEL8.
+
+* mirror bugfixes: symlinks, duplicate patches, and exception handling (#13789)
+* don't try to fetch `BundlePackages` (#13908)
+* avoid re-fetching patches already added to a mirror (#13908)
+* avoid re-fetching alread added patches (#13908)
+* avoid re-fetching alread added patches (#13908)
+* allow repeated invocations of `spack mirror create` on the same dir (#13908)
+* bugfix for RHEL8 when `python` is unavailable (#14252)
+* improve concretization performance in environments (#14190)
+* improve installation performance in environments (#14263)
+
 # v0.13.2 (2019-12-04)
 
 This release contains major performance improvements for Spack environments, as

COPYRIGHT

@@ -68,10 +68,6 @@ PackageName: py
 PackageHomePage: https://pypi.python.org/pypi/py
 PackageLicenseDeclared: MIT
 
-PackageName: pyqver
-PackageHomePage: https://github.com/ghewgill/pyqver
-PackageLicenseDeclared: BSD-3-Clause
-
 PackageName: pytest
 PackageHomePage: https://pypi.python.org/pypi/pytest
 PackageLicenseDeclared: MIT

LICENSE-MIT

@@ -1,4 +1,4 @@
-Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 Spack Project Developers. See the top-level COPYRIGHT file for details.
 
 Permission is hereby granted, free of charge, to any person obtaining a

README.md

@@ -1,6 +1,7 @@
 # <img src="https://cdn.rawgit.com/spack/spack/develop/share/spack/logo/spack-logo.svg" width="64" valign="middle" alt="Spack"/> Spack
 
 [![Build Status](https://travis-ci.org/spack/spack.svg?branch=develop)](https://travis-ci.org/spack/spack)
+[![Linux Builds](https://github.com/spack/spack/workflows/linux%20builds/badge.svg)](https://github.com/spack/spack/actions)
 [![codecov](https://codecov.io/gh/spack/spack/branch/develop/graph/badge.svg)](https://codecov.io/gh/spack/spack)
 [![Read the Docs](https://readthedocs.org/projects/spack/badge/?version=latest)](https://spack.readthedocs.io)
 [![Slack](https://spackpm.herokuapp.com/badge.svg)](https://spackpm.herokuapp.com)

bin/generate-gitlab-ci-yml.sh

@@ -1,91 +0,0 @@
-#! /usr/bin/env bash
-
-# Remember where we are initially, it's the repo cloned by gitlab-ci
-original_directory=$(pwd)
-. "${original_directory}/share/spack/setup-env.sh"
-
-# Create a temporary working directory
-temp_dir=$(mktemp -d)
-trap 'rm -rf "$temp_dir"' INT TERM QUIT EXIT
-
-if [ -z "${DOWNSTREAM_CI_REPO}" ] ; then
-    echo "ERROR: missing variable: DOWNSTREAM_CI_REPO" >&2
-    exit 1
-fi
-
-if [ -z "${SPACK_RELEASE_ENVIRONMENT_PATH}" ] ; then
-    echo "ERROR: missing variable: SPACK_RELEASE_ENVIRONMENT_PATH" >&2
-    exit 1
-fi
-
-if [ -z "${CDASH_AUTH_TOKEN}" ] ; then
-    echo "WARNING: missing variable: CDASH_AUTH_TOKEN" >&2
-else
-    token_file="${temp_dir}/cdash_auth_token"
-    echo ${CDASH_AUTH_TOKEN} > ${token_file}
-fi
-
-if [ -z "${SPACK_RELEASE_ENVIRONMENT_REPO}" ] ; then
-    echo "Assuming spack repo contains environment" >&2
-    env_repo_dir=${original_directory}
-else
-    echo "Cloning ${SPACK_RELEASE_ENVIRONMENT_REPO} into ${temp_dir}/envrepo" >&2
-    cd ${temp_dir}
-    git clone ${SPACK_RELEASE_ENVIRONMENT_REPO} envrepo
-    cd envrepo
-    env_repo_dir=$(pwd)
-fi
-
-current_branch="$CI_COMMIT_REF_NAME"
-
-# Because want to see generated gitlab-ci file as an artifact,
-# we need to write it within the spack repo cloned by gitlab-ci.
-gen_ci_dir="${original_directory}/ci-generation"
-gen_ci_file="${gen_ci_dir}/.gitlab-ci.yml"
-mkdir -p ${gen_ci_dir}
-
-env_dir="${env_repo_dir}/${SPACK_RELEASE_ENVIRONMENT_PATH}"
-
-if [ ! -f "${env_dir}/spack.yaml" ] ; then
-    echo "ERROR: Cannot find spack environment file in ${env_dir}"
-    exit 1
-fi
-
-cd $env_dir
-
-# The next commands generates the .gitlab-ci.yml (and optionally creates a
-# buildgroup in cdash)
-RELEASE_JOBS_ARGS=("--output-file" "${gen_ci_file}")
-if [ ! -z "${token_file}" ]; then
-    RELEASE_JOBS_ARGS+=("--cdash-credentials" "${token_file}")
-fi
-
-spack release-jobs "${RELEASE_JOBS_ARGS[@]}"
-
-if [[ $? -ne 0 ]]; then
-    echo "spack release-jobs command failed"
-    exit 1
-fi
-
-cd "$original_directory"
-mv .git "$temp_dir/original-git-dir"
-git init .
-
-git config user.email "robot@spack.io"
-git config user.name "Spack Build Bot"
-
-cp ${gen_ci_file} "${original_directory}/.gitlab-ci.yml"
-git add .
-
-echo "git commit"
-commit_message="Auto-generated commit testing"
-commit_message="${commit_message} ${current_branch} (${CI_COMMIT_SHA})"
-git commit --message="${commit_message}"
-
-echo "git push"
-git remote add origin "$DOWNSTREAM_CI_REPO"
-git push --force origin "master:multi-ci-${current_branch}"
-
-rm -rf .git
-mv "$temp_dir/original-git-dir" .git
-git reset --hard HEAD

bin/rebuild-index.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-set -x
-
-SPACK_BIN_DIR="${CI_PROJECT_DIR}/bin"
-export PATH="${SPACK_BIN_DIR}:${PATH}"
-
-spack buildcache update-index -d "$MIRROR_URL"

bin/rebuild-package.sh

@@ -1,399 +0,0 @@
-#!/bin/bash
-
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-###
-### This script represents a gitlab-ci job, corresponding to a single release
-### spec.  As such this script must first decide whether or not the spec it
-### has been assigned is up to date on the remote binary mirror.  If it is
-### not (i.e. the source code has changed in a way that caused a change in the
-### full_hash of the spec), this script will build the package, create a
-### binary cache for it, and then push all related files to the remote binary
-### mirror.  This script also optionally communicates with a remote CDash
-### instance to share status on the package build process.
-###
-### The following environment variables are (possibly) used within this script
-### in order for the various elements function properly.
-###
-### First are two defaults we rely on from gitlab:
-###
-### CI_PROJECT_DIR
-### CI_JOB_NAME
-###
-### The following must be set up in the variables section of gitlab:
-###
-### AWS_ACCESS_KEY_ID
-### AWS_SECRET_ACCESS_KEY
-### SPACK_SIGNING_KEY
-###
-### SPACK_S3_UPLOAD_MIRROR_URL         // only required in the short term for the cloud case
-###
-### The following variabes are defined by the ci generation process and are
-### required:
-###
-### SPACK_ENABLE_CDASH
-### SPACK_ROOT_SPEC
-### SPACK_MIRROR_URL
-### SPACK_JOB_SPEC_PKG_NAME
-### SPACK_COMPILER_ACTION
-###
-### Finally, these variables are optionally defined by the ci generation
-### process, and may or may not be present:
-###
-### SPACK_CDASH_BASE_URL
-### SPACK_CDASH_PROJECT
-### SPACK_CDASH_PROJECT_ENC
-### SPACK_CDASH_BUILD_NAME
-### SPACK_CDASH_SITE
-### SPACK_RELATED_BUILDS
-### SPACK_JOB_SPEC_BUILDGROUP
-###
-
-shopt -s expand_aliases
-
-export FORCE_UNSAFE_CONFIGURE=1
-
-TEMP_DIR="${CI_PROJECT_DIR}/jobs_scratch_dir"
-
-JOB_LOG_DIR="${TEMP_DIR}/logs"
-SPEC_DIR="${TEMP_DIR}/specs"
-LOCAL_MIRROR="${CI_PROJECT_DIR}/local_mirror"
-BUILD_CACHE_DIR="${LOCAL_MIRROR}/build_cache"
-SPACK_BIN_DIR="${CI_PROJECT_DIR}/bin"
-
-if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then
-    CDASH_UPLOAD_URL="${SPACK_CDASH_BASE_URL}/submit.php?project=${SPACK_CDASH_PROJECT_ENC}"
-    DEP_JOB_RELATEBUILDS_URL="${SPACK_CDASH_BASE_URL}/api/v1/relateBuilds.php"
-    declare -a JOB_DEPS_PKG_NAMES
-fi
-
-export SPACK_ROOT=${CI_PROJECT_DIR}
-# export PATH="${SPACK_BIN_DIR}:${PATH}"
-export GNUPGHOME="${CI_PROJECT_DIR}/opt/spack/gpg"
-
-. "${CI_PROJECT_DIR}/share/spack/setup-env.sh"
-
-mkdir -p ${JOB_LOG_DIR}
-mkdir -p ${SPEC_DIR}
-
-cleanup() {
-    set +x
-
-    if [ -z "$exit_code" ] ; then
-
-        exit_code=$1
-        if [ -z "$exit_code" ] ; then
-            exit_code=0
-        fi
-
-        restore_io
-
-        if [ "$( type -t finalize )" '=' 'function' ] ; then
-            finalize "$JOB_LOG_DIR/cdash_log.txt"
-        fi
-
-        # We can clean these out later on, once we have a good sense for
-        # how the logging infrastructure is working
-        # rm -rf "$JOB_LOG_DIR"
-    fi
-
-    \exit $exit_code
-}
-
-alias exit='cleanup'
-
-begin_logging() {
-    trap "cleanup 1; \\exit \$exit_code" INT TERM QUIT
-    trap "cleanup 0; \\exit \$exit_code" EXIT
-
-    rm -rf "$JOB_LOG_DIR/cdash_log.txt"
-
-    # NOTE: Here, some redirects are set up
-    exec 3>&1 # fd 3 is now a dup of stdout
-    exec 4>&2 # fd 4 is now a dup of stderr
-
-    # stdout and stderr are joined and redirected to the log
-    exec &> "$JOB_LOG_DIR/cdash_log.txt"
-
-    set -x
-}
-
-restore_io() {
-    exec  >&-
-    exec 2>&-
-
-    exec  >&3
-    exec 2>&4
-
-    exec 3>&-
-    exec 4>&-
-}
-
-finalize() {
-    # If you define a finalize function:
-    #  - it will always be called at the very end of the script
-    #  - the log file will be passed in as the first argument, and
-    #  - the code in this function will not be logged.
-    echo "The full log file is located at $1"
-    # TODO: send this log data to cdash!
-}
-
-check_error()
-{
-    local last_exit_code=$1
-    local last_cmd=$2
-    if [[ ${last_exit_code} -ne 0 ]]; then
-        echo "${last_cmd} exited with code ${last_exit_code}"
-        echo "TERMINATING JOB"
-        exit 1
-    else
-        echo "${last_cmd} completed successfully"
-    fi
-}
-
-extract_build_id()
-{
-    LINES_TO_SEARCH=$1
-    regex="buildSummary\.php\?buildid=([[:digit:]]+)"
-    SINGLE_LINE_OUTPUT=$(echo ${LINES_TO_SEARCH} | tr -d '\n')
-
-    if [[ ${SINGLE_LINE_OUTPUT} =~ ${regex} ]]; then
-        echo "${BASH_REMATCH[1]}"
-    else
-        echo "NONE"
-    fi
-}
-
-get_relate_builds_post_data()
-{
-  cat <<EOF
-{
-  "project": "${1}",
-  "buildid": ${2},
-  "relatedid": ${3},
-  "relationship": "depends on"
-}
-EOF
-}
-
-gen_full_specs_for_job_and_deps() {
-    SPEC_YAML_PATH="${SPEC_DIR}/${SPACK_JOB_SPEC_PKG_NAME}.yaml"
-    local spec_names_to_save="${SPACK_JOB_SPEC_PKG_NAME}"
-
-    if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then
-        IFS=';' read -ra DEPS <<< "${SPACK_RELATED_BUILDS}"
-        for i in "${DEPS[@]}"; do
-            depPkgName="${i}"
-            spec_names_to_save="${spec_names_to_save} ${depPkgName}"
-            JOB_DEPS_PKG_NAMES+=("${depPkgName}")
-        done
-    fi
-
-    if [ "${SPACK_COMPILER_ACTION}" == "FIND_ANY" ]; then
-        # This corresponds to a bootstrapping phase where we need to
-        # rely on any available compiler to build the package (i.e. the
-        # compiler needed to be stripped from the spec), and thus we need
-        # to concretize the root spec again.
-        spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec "${SPACK_ROOT_SPEC}" --yaml-dir "${SPEC_DIR}"
-    else
-        # in this case, either we're relying on Spack to install missing compiler
-        # bootstrapped in a previous phase, or else we only had one phase (like a
-        # site which already knows what compilers are available on it's runners),
-        # so we don't want to concretize that root spec again.  The reason we need
-        # this in the first case (bootstrapped compiler), is that we can't concretize
-        # a spec at this point if we're going to ask spack to "install_missing_compilers".
-        tmp_dir=$(mktemp -d)
-        TMP_YAML_PATH="${tmp_dir}/root.yaml"
-        ROOT_SPEC_YAML=$(spack python -c "import base64 ; import zlib ; print(str(zlib.decompress(base64.b64decode('${SPACK_ROOT_SPEC}')).decode('utf-8')))")
-        echo "${ROOT_SPEC_YAML}" > "${TMP_YAML_PATH}"
-        spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec-yaml "${TMP_YAML_PATH}" --yaml-dir "${SPEC_DIR}"
-        rm -rf ${tmp_dir}
-    fi
-}
-
-begin_logging
-
-echo "Running job for spec: ${CI_JOB_NAME}"
-
-# This should create the directory we referred to as GNUPGHOME earlier
-spack gpg list
-
-# Importing the secret key using gpg2 directly should allow to
-# sign and verify both
-set +x
-KEY_IMPORT_RESULT=`echo ${SPACK_SIGNING_KEY} | base64 --decode | gpg2 --import`
-check_error $? "gpg2 --import"
-set -x
-
-spack gpg list --trusted
-spack gpg list --signing
-
-# To have spack install missing compilers, we need to add a custom
-# configuration scope, then we pass that to the package installation
-# command
-CUSTOM_CONFIG_SCOPE_DIR="${TEMP_DIR}/config_scope"
-mkdir -p "${CUSTOM_CONFIG_SCOPE_DIR}"
-CUSTOM_CONFIG_SCOPE_ARG=""
-
-if [ "${SPACK_COMPILER_ACTION}" == "INSTALL_MISSING" ]; then
-    echo "Make sure bootstrapped compiler will be installed"
-    custom_config_file_path="${CUSTOM_CONFIG_SCOPE_DIR}/config.yaml"
-      cat <<CONFIG_STUFF > "${custom_config_file_path}"
-config:
-  install_missing_compilers: true
-CONFIG_STUFF
-    CUSTOM_CONFIG_SCOPE_ARG="-C ${CUSTOM_CONFIG_SCOPE_DIR}"
-    # Configure the binary mirror where, if needed, this jobs compiler
-    # was installed in binary pacakge form, then tell spack to
-    # install_missing_compilers.
-elif [ "${SPACK_COMPILER_ACTION}" == "FIND_ANY" ]; then
-    echo "Just find any available compiler"
-    spack compiler find
-else
-    echo "No compiler action to be taken"
-fi
-
-# Finally, list the compilers spack knows about
-echo "Compiler Configurations:"
-spack config get compilers
-
-# Write full-deps yamls for this job spec and its dependencies
-gen_full_specs_for_job_and_deps
-
-# Make the build_cache directory if it doesn't exist
-mkdir -p "${BUILD_CACHE_DIR}"
-
-# Get buildcache name so we can write a CDash build id file in the right place.
-# If we're unable to get the buildcache name, we may have encountered a problem
-# concretizing the spec, or some other issue that will eventually cause the job
-# to fail.
-JOB_BUILD_CACHE_ENTRY_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${SPEC_YAML_PATH}"`
-if [[ $? -ne 0 ]]; then
-    echo "ERROR, unable to get buildcache entry name for job ${CI_JOB_NAME}"
-    exit 1
-fi
-
-if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then
-    # Whether we have to build the spec or download it pre-built, we expect to find
-    # the cdash build id file sitting in this location afterwards.
-    JOB_CDASH_ID_FILE="${BUILD_CACHE_DIR}/${JOB_BUILD_CACHE_ENTRY_NAME}.cdashid"
-fi
-
-# Finally, we can check the spec we have been tasked with build against
-# the built binary on the remote mirror to see if it needs to be rebuilt
-spack -d buildcache check --spec-yaml "${SPEC_YAML_PATH}" --mirror-url "${SPACK_MIRROR_URL}" --rebuild-on-error
-
-if [[ $? -ne 0 ]]; then
-    # Configure mirror
-    spack mirror add local_artifact_mirror "file://${LOCAL_MIRROR}"
-
-    if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then
-        JOB_CDASH_ID="NONE"
-
-        # Install package, using the buildcache from the local mirror to
-        # satisfy dependencies.
-        BUILD_ID_LINE=`spack -d -k -v "${CUSTOM_CONFIG_SCOPE_ARG}" install --keep-stage --cdash-upload-url "${CDASH_UPLOAD_URL}" --cdash-build "${SPACK_CDASH_BUILD_NAME}" --cdash-site "${SPACK_CDASH_SITE}" --cdash-track "${SPACK_JOB_SPEC_BUILDGROUP}" -f "${SPEC_YAML_PATH}" | grep "buildSummary\\.php"`
-        check_error $? "spack install"
-
-        # By parsing the output of the "spack install" command, we can get the
-        # buildid generated for us by CDash
-        JOB_CDASH_ID=$(extract_build_id "${BUILD_ID_LINE}")
-
-        # Write the .cdashid file to the buildcache as well
-        echo "${JOB_CDASH_ID}" >> ${JOB_CDASH_ID_FILE}
-    else
-        spack -d -k -v "${CUSTOM_CONFIG_SCOPE_ARG}" install --keep-stage -f "${SPEC_YAML_PATH}"
-    fi
-
-    # Copy some log files into an artifact location, once we have a way
-    # to provide a spec.yaml file to more spack commands (e.g. "location")
-    # stage_dir=$(spack location --stage-dir -f "${SPEC_YAML_PATH}")
-    # build_log_file=$(find -L "${stage_dir}" | grep "spack-build\\.out")
-    # config_log_file=$(find -L "${stage_dir}" | grep "config\\.log")
-    # cp "${build_log_file}" "${JOB_LOG_DIR}/"
-    # cp "${config_log_file}" "${JOB_LOG_DIR}/"
-
-    # Create buildcache entry for this package, reading the spec from the yaml
-    # file.
-    spack -d buildcache create --spec-yaml "${SPEC_YAML_PATH}" -a -f -d "${LOCAL_MIRROR}" --no-rebuild-index
-    check_error $? "spack buildcache create"
-
-    # TODO: The upload-s3 command should eventually be replaced with something
-    # like: "spack buildcache put <mirror> <spec>", when that subcommand is
-    # properly implemented.
-    if [ ! -z "${SPACK_S3_UPLOAD_MIRROR_URL}" ] ; then
-        spack -d upload-s3 spec --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}" --endpoint-url "${SPACK_S3_UPLOAD_MIRROR_URL}"
-        check_error $? "spack upload-s3 spec"
-    else
-        spack -d buildcache copy --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}" --destination-url "${SPACK_MIRROR_URL}"
-    fi
-else
-    echo "spec ${CI_JOB_NAME} is already up to date on remote mirror, downloading it"
-
-    # Configure remote mirror so we can download buildcache entry
-    spack mirror add remote_binary_mirror ${SPACK_MIRROR_URL}
-
-    # Now download it
-    BUILDCACHE_DL_ARGS=("--spec-yaml" "${SPEC_YAML_PATH}" "--path" "${BUILD_CACHE_DIR}/" )
-    if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then
-        BUILDCACHE_DL_ARGS+=( "--require-cdashid" )
-    fi
-    spack -d buildcache download "${BUILDCACHE_DL_ARGS[@]}"
-    check_error $? "spack buildcache download"
-fi
-
-# The next step is to relate this job to the jobs it depends on
-if [ "${SPACK_ENABLE_CDASH}" == "True" ] ; then
-    if [ -f "${JOB_CDASH_ID_FILE}" ]; then
-        JOB_CDASH_BUILD_ID=$(<${JOB_CDASH_ID_FILE})
-
-        if [ "${JOB_CDASH_BUILD_ID}" == "NONE" ]; then
-            echo "ERROR: unable to read this jobs id from ${JOB_CDASH_ID_FILE}"
-            exit 1
-        fi
-
-        # Now get CDash ids for dependencies and "relate" each dependency build
-        # with this jobs build
-        for DEP_PKG_NAME in "${JOB_DEPS_PKG_NAMES[@]}"; do
-            echo "Getting cdash id for dependency --> ${DEP_PKG_NAME} <--"
-            DEP_SPEC_YAML_PATH="${SPEC_DIR}/${DEP_PKG_NAME}.yaml"
-            DEP_JOB_BUILDCACHE_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${DEP_SPEC_YAML_PATH}"`
-
-            if [[ $? -eq 0 ]]; then
-                DEP_JOB_ID_FILE="${BUILD_CACHE_DIR}/${DEP_JOB_BUILDCACHE_NAME}.cdashid"
-                echo "DEP_JOB_ID_FILE path = ${DEP_JOB_ID_FILE}"
-
-                if [ -f "${DEP_JOB_ID_FILE}" ]; then
-                    DEP_JOB_CDASH_BUILD_ID=$(<${DEP_JOB_ID_FILE})
-                    echo "File ${DEP_JOB_ID_FILE} contained value ${DEP_JOB_CDASH_BUILD_ID}"
-                    echo "Relating builds -> ${SPACK_CDASH_BUILD_NAME} (buildid=${JOB_CDASH_BUILD_ID}) depends on ${DEP_PKG_NAME} (buildid=${DEP_JOB_CDASH_BUILD_ID})"
-                    relateBuildsPostBody="$(get_relate_builds_post_data "${SPACK_CDASH_PROJECT}" ${JOB_CDASH_BUILD_ID} ${DEP_JOB_CDASH_BUILD_ID})"
-                    relateBuildsResult=`curl "${DEP_JOB_RELATEBUILDS_URL}" -H "Content-Type: application/json" -H "Accept: application/json" -d "${relateBuildsPostBody}"`
-                    echo "Result of curl request: ${relateBuildsResult}"
-                else
-                    echo "ERROR: Did not find expected .cdashid file for dependency: ${DEP_JOB_ID_FILE}"
-                    exit 1
-                fi
-            else
-                echo "ERROR: Unable to get buildcache entry name for ${DEP_SPEC_NAME}"
-                exit 1
-            fi
-        done
-    else
-        echo "ERROR: Did not find expected .cdashid file ${JOB_CDASH_ID_FILE}"
-        exit 1
-    fi
-fi
-
-# Show the size of the buildcache and a list of what's in it, directly
-# in the gitlab log output
-(
-    restore_io
-    du -sh ${BUILD_CACHE_DIR}
-    find ${BUILD_CACHE_DIR} -maxdepth 3 -type d -ls
-)
-
-echo "End of rebuild package script"

bin/sbang

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -103,10 +103,10 @@ interpreter_f="${interpreter_v[0]}"
 
 # Invoke any interpreter found, or raise an error if none was found.
 if [[ -n "$interpreter_f" ]]; then
-    if [[ "${interpreter_f##*/}" = "perl" ]]; then
-        exec $interpreter_v -x "$@"
+    if [[ "${interpreter_f##*/}" = "perl"* ]]; then
+        exec $interpreter -x "$@"
     else
-        exec $interpreter_v "$@"
+        exec $interpreter "$@"
     fi
 else
     echo "error: sbang found no interpreter in $script"

bin/spack

@@ -1,7 +1,7 @@
 #!/bin/sh
 # -*- python -*-
 #
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

bin/spack-python

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

etc/spack/defaults/config.yaml

@@ -137,7 +137,7 @@ config:
   # when Spack needs to manage its own package metadata and all operations are
   # expected to complete within the default time limit. The timeout should
   # therefore generally be left untouched.
-  db_lock_timeout: 120
+  db_lock_timeout: 3
 
 
   # How long to wait when attempting to modify a package (e.g. to install it).

etc/spack/defaults/packages.yaml

@@ -40,9 +40,11 @@ packages:
       pil: [py-pillow]
       pkgconfig: [pkgconf, pkg-config]
       scalapack: [netlib-scalapack]
+      sycl: [hipsycl]
       szip: [libszip, libaec]
       tbb: [intel-tbb]
       unwind: [libunwind]
+      sycl: [hipsycl]
     permissions:
       read: world
       write: user

lib/spack/docs/basic_usage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -232,6 +232,50 @@ remove dependent packages *before* removing their dependencies or use the
 
 .. _nondownloadable:
 
+^^^^^^^^^^^^^^^^^^
+Garbage collection
+^^^^^^^^^^^^^^^^^^
+
+When Spack builds software from sources, if often installs tools that are needed
+just to build or test other software. These are not necessary at runtime.
+To support cases where removing these tools can be a benefit Spack provides
+the ``spack gc`` ("garbage collector") command, which will uninstall all unneeded packages:
+
+.. code-block:: console
+
+   $ spack find
+   ==> 24 installed packages
+   -- linux-ubuntu18.04-broadwell / gcc@9.0.1 ----------------------
+   autoconf@2.69    findutils@4.6.0  libiconv@1.16        libszip@2.1.1  m4@1.4.18    openjpeg@2.3.1  pkgconf@1.6.3  util-macros@1.19.1
+   automake@1.16.1  gdbm@1.18.1      libpciaccess@0.13.5  libtool@2.4.6  mpich@3.3.2  openssl@1.1.1d  readline@8.0   xz@5.2.4
+   cmake@3.16.1     hdf5@1.10.5      libsigsegv@2.12      libxml2@2.9.9  ncurses@6.1  perl@5.30.0     texinfo@6.5    zlib@1.2.11
+
+   $ spack gc
+   ==> The following packages will be uninstalled:
+
+       -- linux-ubuntu18.04-broadwell / gcc@9.0.1 ----------------------
+       vn47edz autoconf@2.69    6m3f2qn findutils@4.6.0  ubl6bgk libtool@2.4.6  pksawhz openssl@1.1.1d  urdw22a readline@8.0
+       ki6nfw5 automake@1.16.1  fklde6b gdbm@1.18.1      b6pswuo m4@1.4.18      k3s2csy perl@5.30.0     lp5ya3t texinfo@6.5
+       ylvgsov cmake@3.16.1     5omotir libsigsegv@2.12  leuzbbh ncurses@6.1    5vmfbrq pkgconf@1.6.3   5bmv4tg util-macros@1.19.1
+
+   ==> Do you want to proceed? [y/N] y
+
+   [ ... ]
+
+   $ spack find
+   ==> 9 installed packages
+   -- linux-ubuntu18.04-broadwell / gcc@9.0.1 ----------------------
+   hdf5@1.10.5  libiconv@1.16  libpciaccess@0.13.5  libszip@2.1.1  libxml2@2.9.9  mpich@3.3.2  openjpeg@2.3.1  xz@5.2.4  zlib@1.2.11
+
+In the example above Spack went through all the packages in the DB
+and removed everything that is not either:
+
+1. A package installed upon explicit request of the user
+2. A ``link`` or ``run`` dependency, even transitive, of one of the packages at point 1.
+
+You can check :ref:`cmd-spack-find-metadata` to see how to query for explicitly installed packages
+or :ref:`dependency-types` for a more thorough treatment of dependency types.
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 Non-Downloadable Tarballs
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -414,6 +458,8 @@ Packages are divided into groups according to their architecture and
 compiler.  Within each group, Spack tries to keep the view simple, and
 only shows the version of installed packages.
 
+.. _cmd-spack-find-metadata:
+
 """"""""""""""""""""""""""""""""
 Viewing more metadata
 """"""""""""""""""""""""""""""""
@@ -883,11 +929,13 @@ in GNU Autotools. If all flags are set, the order is
 Compiler environment variables and additional RPATHs
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-In the exceptional case a compiler requires setting special environment
-variables, like an explicit library load path. These can bet set in an
-extra section in the compiler configuration (the supported environment
-modification commands are: ``set``, ``unset``, ``append-path``, and
-``prepend-path``). The user can also specify additional ``RPATHs`` that the
+Sometimes compilers require setting special environment variables to
+operate correctly. Spack handles these cases by allowing custom environment
+modifications in the ``environment`` attribute of the compiler configuration
+section. See also the :ref:`configuration_environment_variables` section
+of the configuration files docs for more information.
+
+It is also possible to specify additional ``RPATHs`` that the
 compiler will add to all executables generated by that compiler.  This is
 useful for forcing certain compilers to RPATH their own runtime libraries, so
 that executables will run without the need to set ``LD_LIBRARY_PATH``.
@@ -904,28 +952,19 @@ that executables will run without the need to set ``LD_LIBRARY_PATH``.
           fc: /opt/gcc/bin/gfortran
         environment:
           unset:
-            BAD_VARIABLE: # The colon is required but the value must be empty
+            - BAD_VARIABLE
           set:
             GOOD_VARIABLE_NUM: 1
             GOOD_VARIABLE_STR: good
-          prepend-path:
+          prepend_path:
             PATH: /path/to/binutils
-          append-path:
+          append_path:
             LD_LIBRARY_PATH: /opt/gcc/lib
         extra_rpaths:
         - /path/to/some/compiler/runtime/directory
         - /path/to/some/other/compiler/runtime/directory
 
 
-.. note::
-
-   The section `environment` is interpreted as an ordered dictionary, which
-   means two things. First, environment modification are applied in the order
-   they are specified in the configuration file. Second, you cannot express
-   environment modifications that require mixing different commands, i.e. you
-   cannot `set` one variable, than `prepend-path` to another one, and than
-   again `set` a third one.
-
 ^^^^^^^^^^^^^^^^^^^^^^^
 Architecture specifiers
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -1271,10 +1310,9 @@ directly when you run ``python``:
 Using Extensions
 ^^^^^^^^^^^^^^^^
 
-There are three ways to get ``numpy`` working in Python.  The first is
-to use :ref:`shell-support`.  You can simply ``load`` the
-module for the extension, and it will be added to the ``PYTHONPATH``
-in your current shell:
+There are four ways to get ``numpy`` working in Python.  The first is
+to use :ref:`shell-support`.  You can simply ``load`` the extension,
+and it will be added to the ``PYTHONPATH`` in your current shell:
 
 .. code-block:: console
 
@@ -1284,11 +1322,29 @@ in your current shell:
 Now ``import numpy`` will succeed for as long as you keep your current
 session open.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Loading Extensions via Modules
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Instead of using Spack's environment modification capabilities through
+the ``spack load`` command, you can load numpy through your
+environment modules (using ``environment-modules`` or ``lmod``). This
+will also add the extension to the ``PYTHONPATH`` in your current
+shell.
+
+.. code-block:: console
+
+   $ module load <name of numpy module>
+
+If you do not know the name of the specific numpy module you wish to
+load, you can use the ``spack module tcl|lmod loads`` command to get
+the name of the module from the Spack spec.
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Activating Extensions in a View
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The second way to use extensions is to create a view, which merges the
+Another way to use extensions is to create a view, which merges the
 python installation along with the extensions into a single prefix.
 See :ref:`filesystem-views` for a more in-depth description of views and
 :ref:`cmd-spack-view` for usage of the ``spack view`` command.

lib/spack/docs/binary_caches.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_settings.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -58,9 +58,9 @@ directory. Here's an example of an external configuration:
    packages:
      openmpi:
        paths:
-         openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7: /opt/openmpi-1.4.3
-         openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7+debug: /opt/openmpi-1.4.3-debug
-         openmpi@1.6.5%intel@10.1 arch=linux-x86_64-debian7: /opt/openmpi-1.6.5-intel
+         openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64: /opt/openmpi-1.4.3
+         openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64+debug: /opt/openmpi-1.4.3-debug
+         openmpi@1.6.5%intel@10.1 arch=linux-debian7-x86_64: /opt/openmpi-1.6.5-intel
 
 This example lists three installations of OpenMPI, one built with GCC,
 one built with GCC and debug information, and another built with Intel.
@@ -107,9 +107,9 @@ be:
    packages:
      openmpi:
        paths:
-         openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7: /opt/openmpi-1.4.3
-         openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7+debug: /opt/openmpi-1.4.3-debug
-         openmpi@1.6.5%intel@10.1 arch=linux-x86_64-debian7: /opt/openmpi-1.6.5-intel
+         openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64: /opt/openmpi-1.4.3
+         openmpi@1.4.3%gcc@4.4.7 arch=linux-debian7-x86_64+debug: /opt/openmpi-1.4.3-debug
+         openmpi@1.6.5%intel@10.1 arch=linux-debian7-x86_64: /opt/openmpi-1.6.5-intel
        buildable: False
 
 The addition of the ``buildable`` flag tells Spack that it should never build

lib/spack/docs/build_systems.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -56,6 +56,7 @@ on these ideas for each distinct build system that Spack supports:
    :maxdepth: 1
    :caption: Other
 
+   build_systems/bundlepackage
    build_systems/cudapackage
    build_systems/intelpackage
    build_systems/custompackage

lib/spack/docs/build_systems/autotoolspackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/bundlepackage.rst

@@ -0,0 +1,52 @@
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.
+
+   SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+.. _bundlepackage:
+
+-------------
+BundlePackage
+-------------
+
+``BundlePackage`` represents a set of packages that are expected to work well
+together, such as a collection of commonly used software libraries.  The
+associated software is specified as bundle dependencies.
+
+
+^^^^^^^^
+Creation
+^^^^^^^^
+
+Be sure to specify the ``bundle`` template if you are using ``spack create``
+to generate a package from the template.  For example, use the following
+command to create a bundle package whose class name will be ``Mybundle``:
+
+.. code-block:: console
+
+    $ spack create --template bundle --name mybundle
+
+
+
+^^^^^^
+Phases
+^^^^^^
+
+The ``BundlePackage`` base class does not provide any phases by default
+since the bundle does not represent a build system.
+
+
+^^^
+URL
+^^^
+
+The ``url`` property does not have meaning since there is no package-specific
+code to fetch.
+
+
+^^^^^^^
+Version
+^^^^^^^
+
+At least one ``version`` must be specified in order for the package to
+build.

lib/spack/docs/build_systems/cmakepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/cudapackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/custompackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/intelpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/makefilepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/mesonpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/octavepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/perlpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/pythonpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/qmakepackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/rpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/rubypackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/sconspackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/sippackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/build_systems/wafpackage.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/chain.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/conf.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -176,7 +176,25 @@ def setup(sphinx):
 #show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+# We use our own extension of the default style with a few modifications
+from pygments.style import Style
+from pygments.styles.default import DefaultStyle
+from pygments.token import Generic, Comment, Text
+
+class SpackStyle(DefaultStyle):
+    styles = DefaultStyle.styles.copy()
+    background_color       = "#f4f4f8"
+    styles[Generic.Output] = "#355"
+    styles[Generic.Prompt] = "bold #346ec9"
+
+import pkg_resources
+dist = pkg_resources.Distribution(__file__)
+sys.path.append('.')  # make 'conf' module findable
+ep = pkg_resources.EntryPoint.parse('spack = conf:SpackStyle', dist=dist)
+dist._ep_map = {'pygments.styles': {'plugin1': ep}}
+pkg_resources.working_set.add(dist)
+
+pygments_style = 'spack'
 
 # A list of ignored prefixes for module index sorting.
 #modindex_common_prefix = []

lib/spack/docs/config_yaml.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -30,11 +30,21 @@ Default is ``$spack/opt/spack``.
 ``install_hash_length`` and ``install_path_scheme``
 ---------------------------------------------------
 
-The default Spack installation path can be very long and can create
-problems for scripts with hardcoded shebangs. There are two parameters
-to help with that. Firstly, the ``install_hash_length`` parameter can
-set the length of the hash in the installation path from 1 to 32. The
-default path uses the full 32 characters.
+The default Spack installation path can be very long and can create problems
+for scripts with hardcoded shebangs. Additionally, when using the Intel
+compiler, and if there is also a long list of dependencies, the compiler may
+segfault. If you see the following:
+
+     .. code-block:: console
+
+       : internal error: ** The compiler has encountered an unexpected problem.
+       ** Segmentation violation signal raised. **
+       Access violation or stack overflow. Please contact Intel Support for assistance.
+
+it may be because variables containing dependency specs may be too long. There
+are two parameters to help with long path names. Firstly, the
+``install_hash_length`` parameter can set the length of the hash in the
+installation path from 1 to 32. The default path uses the full 32 characters.
 
 Secondly, it is also possible to modify the entire installation
 scheme. By default Spack uses

lib/spack/docs/configuration.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -427,6 +427,33 @@ home directory, and ``~user`` will expand to a specified user's home
 directory. The ``~`` must appear at the beginning of the path, or Spack
 will not expand it.
 
+.. _configuration_environment_variables:
+
+-------------------------
+Environment Modifications
+-------------------------
+
+Spack allows to prescribe custom environment modifications in a few places
+within its configuration files. Every time these modifications are allowed
+they are specified as a dictionary, like in the following example:
+
+.. code-block:: yaml
+
+   environment:
+     set:
+       LICENSE_FILE: '/path/to/license'
+     unset:
+     - CPATH
+     - LIBRARY_PATH
+     append_path:
+       PATH: '/new/bin/dir'
+
+The possible actions that are permitted are ``set``, ``unset``, ``append_path``,
+``prepend_path`` and finally ``remove_path``. They all require a dictionary
+of variable names mapped to the values used for the modification.
+The only exception is ``unset`` that requires just a list of variable names.
+No particular order is ensured on the execution of each of these modifications.
+
 ----------------------------
 Seeing Spack's Configuration
 ----------------------------

lib/spack/docs/containers.rst

@@ -0,0 +1,307 @@
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.
+
+   SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+.. _containers:
+
+================
+Container Images
+================
+
+Spack can be an ideal tool to setup images for containers since all the
+features discussed in :ref:`environments` can greatly help to manage
+the installation of software during the image build process. Nonetheless,
+building a production image from scratch still requires a lot of
+boilerplate to:
+
+- Get Spack working within the image, possibly running as root
+- Minimize the physical size of the software installed
+- Properly update the system software in the base image
+
+To facilitate users with these tedious tasks, Spack provides a command
+to automatically generate recipes for container images based on
+Environments:
+
+.. code-block:: console
+
+   $ ls
+   spack.yaml
+
+   $ spack containerize
+   # Build stage with Spack pre-installed and ready to be used
+   FROM spack/centos7:latest as builder
+
+   # What we want to install and how we want to install it
+   # is specified in a manifest file (spack.yaml)
+   RUN mkdir /opt/spack-environment \
+   &&  (echo "spack:" \
+   &&   echo "  specs:" \
+   &&   echo "  - gromacs+mpi" \
+   &&   echo "  - mpich" \
+   &&   echo "  concretization: together" \
+   &&   echo "  config:" \
+   &&   echo "    install_tree: /opt/software" \
+   &&   echo "  view: /opt/view") > /opt/spack-environment/spack.yaml
+
+   # Install the software, remove unecessary deps
+   RUN cd /opt/spack-environment && spack install && spack gc -y
+
+   # Strip all the binaries
+   RUN find -L /opt/view/* -type f -exec readlink -f '{}' \; | \
+       xargs file -i | \
+       grep 'charset=binary' | \
+       grep 'x-executable\|x-archive\|x-sharedlib' | \
+       awk -F: '{print $1}' | xargs strip -s
+
+   # Modifications to the environment that are necessary to run
+   RUN cd /opt/spack-environment && \
+       spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
+
+
+   # Bare OS image to run the installed executables
+   FROM centos:7
+
+   COPY --from=builder /opt/spack-environment /opt/spack-environment
+   COPY --from=builder /opt/software /opt/software
+   COPY --from=builder /opt/view /opt/view
+   COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh
+
+   RUN yum update -y && yum install -y epel-release && yum update -y                                   \
+    && yum install -y libgomp \
+    && rm -rf /var/cache/yum  && yum clean all
+
+   RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc
+
+
+   LABEL "app"="gromacs"
+   LABEL "mpi"="mpich"
+
+   ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"]
+
+
+The bits that make this automation possible are discussed in details
+below. All the images generated in this way will be based on
+multi-stage builds with:
+
+- A fat ``build`` stage containing common build tools and Spack itself
+- A minimal ``final`` stage containing only the software requested by the user
+
+-----------------
+Spack Base Images
+-----------------
+
+Docker images with Spack preinstalled and ready to be used are
+built on `Docker Hub <https://hub.docker.com/u/spack>`_
+at every push to ``develop`` or to a release branch. The OS that
+are currently supported are summarized in the table below:
+
+.. _containers-supported-os:
+
+.. list-table:: Supported operating systems
+   :header-rows: 1
+
+   * - Operating System
+     - Base Image
+     - Spack Image
+   * - Ubuntu 16.04
+     - ``ubuntu:16.04``
+     - ``spack/ubuntu-xenial``
+   * - Ubuntu 18.04
+     - ``ubuntu:16.04``
+     - ``spack/ubuntu-bionic``
+   * - CentOS 6
+     - ``centos:6``
+     - ``spack/centos6``
+   * - CentOS 7
+     - ``centos:7``
+     - ``spack/centos7``
+
+All the images are tagged with the corresponding release of Spack:
+
+.. image:: dockerhub_spack.png
+
+with the exception of the ``latest`` tag that points to the HEAD
+of the ``develop`` branch. These images are available for anyone
+to use and take care of all the repetitive tasks that are necessary
+to setup Spack within a container. All the container recipes generated
+automatically by Spack use them as base images for their ``build`` stage.
+
+
+-------------------------
+Environment Configuration
+-------------------------
+
+Any Spack Environment can be used for the automatic generation of container
+recipes. Sensible defaults are provided for things like the base image or the
+version of Spack used in the image. If a finer tuning is needed it can be
+obtained by adding the relevant metadata under the ``container`` attribute
+of environments:
+
+.. code-block:: yaml
+
+   spack:
+     specs:
+     - gromacs+mpi
+     - mpich
+
+     container:
+       # Select the format of the recipe e.g. docker,
+       # singularity or anything else that is currently supported
+       format: docker
+
+       # Select from a valid list of images
+       base:
+         image: "centos:7"
+         spack: develop
+
+       # Whether or not to strip binaries
+       strip: true
+
+       # Additional system packages that are needed at runtime
+       os_packages:
+       - libgomp
+
+       # Extra instructions
+       extra_instructions:
+         final: |
+   RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc
+
+       # Labels for the image
+       labels:
+         app: "gromacs"
+         mpi: "mpich"
+
+The tables below describe the configuration options that are currently supported:
+
+.. list-table:: General configuration options for the ``container`` section of ``spack.yaml``
+   :header-rows: 1
+
+   * - Option Name
+     - Description
+     - Allowed Values
+     - Required
+   * - ``format``
+     - The format of the recipe
+     - ``docker`` or ``singularity``
+     - Yes
+   * - ``base:image``
+     - Base image for ``final`` stage
+     - See :ref:`containers-supported-os`
+     - Yes
+   * - ``base:spack``
+     - Version of Spack
+     - Valid tags for ``base:image``
+     - Yes
+   * - ``strip``
+     - Whether to strip binaries
+     - ``true`` (default) or ``false``
+     - No
+   * - ``os_packages``
+     - System packages to be installed
+     - Valid packages for the ``final`` OS
+     - No
+   * - ``extra_instructions:build``
+     - Extra instructions (e.g. `RUN`, `COPY`, etc.) at the end of the ``build`` stage
+     - Anything understood by the current ``format``
+     - No
+   * - ``extra_instructions:final``
+     - Extra instructions (e.g. `RUN`, `COPY`, etc.) at the end of the ``final`` stage
+     - Anything understood by the current ``format``
+     - No
+   * - ``labels``
+     - Labels to tag the image
+     - Pairs of key-value strings
+     - No
+
+.. list-table:: Configuration options specific to Singularity
+   :header-rows: 1
+
+   * - Option Name
+     - Description
+     - Allowed Values
+     - Required
+   * - ``singularity:runscript``
+     - Content of ``%runscript``
+     - Any valid script
+     - No
+   * - ``singularity:startscript``
+     - Content of ``%startscript``
+     - Any valid script
+     - No
+   * - ``singularity:test``
+     - Content of ``%test``
+     - Any valid script
+     - No
+   * - ``singularity:help``
+     - Description of the image
+     - Description string
+     - No
+
+Once the Environment is properly configured a recipe for a container
+image can be printed to standard output by issuing the following
+command from the directory where the ``spack.yaml`` resides:
+
+.. code-block:: console
+
+   $ spack containerize
+
+The example ``spack.yaml`` above would produce for instance the
+following ``Dockerfile``:
+
+.. code-block:: docker
+
+   # Build stage with Spack pre-installed and ready to be used
+   FROM spack/centos7:latest as builder
+
+   # What we want to install and how we want to install it
+   # is specified in a manifest file (spack.yaml)
+   RUN mkdir /opt/spack-environment \
+   &&  (echo "spack:" \
+   &&   echo "  specs:" \
+   &&   echo "  - gromacs+mpi" \
+   &&   echo "  - mpich" \
+   &&   echo "  concretization: together" \
+   &&   echo "  config:" \
+   &&   echo "    install_tree: /opt/software" \
+   &&   echo "  view: /opt/view") > /opt/spack-environment/spack.yaml
+
+   # Install the software, remove unecessary deps
+   RUN cd /opt/spack-environment && spack install && spack gc -y
+
+   # Strip all the binaries
+   RUN find -L /opt/view/* -type f -exec readlink -f '{}' \; | \
+       xargs file -i | \
+       grep 'charset=binary' | \
+       grep 'x-executable\|x-archive\|x-sharedlib' | \
+       awk -F: '{print $1}' | xargs strip -s
+
+   # Modifications to the environment that are necessary to run
+   RUN cd /opt/spack-environment && \
+       spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
+
+
+   # Bare OS image to run the installed executables
+   FROM centos:7
+
+   COPY --from=builder /opt/spack-environment /opt/spack-environment
+   COPY --from=builder /opt/software /opt/software
+   COPY --from=builder /opt/view /opt/view
+   COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh
+
+   RUN yum update -y && yum install -y epel-release && yum update -y                                   \
+    && yum install -y libgomp \
+    && rm -rf /var/cache/yum  && yum clean all
+
+   RUN echo 'export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][gromacs]\[$(tput setaf 2)\]\u\[$(tput sgr0)\]:\w $ \[$(tput sgr0)\]"' >> ~/.bashrc
+
+
+   LABEL "app"="gromacs"
+   LABEL "mpi"="mpich"
+
+   ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l"]
+
+.. note::
+   Spack can also produce Singularity definition files to build the image. The
+   minimum version of Singularity required to build a SIF (Singularity Image Format)
+   from them is ``3.5.3``.

lib/spack/docs/contribution_guide.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -64,6 +64,8 @@ If you take a look in ``$SPACK_ROOT/.travis.yml``, you'll notice that we test
 against Python 2.6, 2.7, and 3.4-3.7 on both macOS and Linux. We currently
 perform 3 types of tests:
 
+.. _cmd-spack-test:
+
 ^^^^^^^^^^
 Unit Tests
 ^^^^^^^^^^
@@ -86,40 +88,83 @@ To run *all* of the unit tests, use:
 
    $ spack test
 
-These tests may take several minutes to complete. If you know you are only
-modifying a single Spack feature, you can run a single unit test at a time:
+These tests may take several minutes to complete. If you know you are
+only modifying a single Spack feature, you can run subsets of tests at a
+time.  For example, this would run all the tests in
+``lib/spack/spack/test/architecture.py``:
 
 .. code-block:: console
 
-   $ spack test architecture
+   $ spack test architecture.py
 
-This allows you to develop iteratively: make a change, test that change, make
-another change, test that change, etc. To get a list of all available unit
-tests, run:
+And this would run the ``test_platform`` test from that file:
+
+.. code-block:: console
+
+   $ spack test architecture.py::test_platform
+
+This allows you to develop iteratively: make a change, test that change,
+make another change, test that change, etc.  We use `pytest
+<http://pytest.org/>`_ as our tests fromework, and these types of
+arguments are just passed to the ``pytest`` command underneath. See `the
+pytest docs
+<http://doc.pytest.org/en/latest/usage.html#specifying-tests-selecting-tests>`_
+for more details on test selection syntax.
+
+``spack test`` has a few special options that can help you understand
+what tests are available.  To get a list of all available unit test
+files, run:
 
 .. command-output:: spack test --list
+   :ellipsis: 5
 
-A more detailed list of available unit tests can be found by running
-``spack test --long-list``.
+To see a more detailed list of available unit tests, use ``spack test
+--list-long``:
 
-By default, ``pytest`` captures the output of all unit tests. If you add print
-statements to a unit test and want to see the output, simply run:
+.. command-output:: spack test --list-long
+   :ellipsis: 10
+
+And to see the fully qualified names of all tests, use ``--list-names``:
+
+.. command-output:: spack test --list-names
+   :ellipsis: 5
+
+You can combine these with ``pytest`` arguments to restrict which tests
+you want to know about.  For example, to see just the tests in
+``architecture.py``:
+
+.. command-output:: spack test --list-long architecture.py
+
+You can also combine any of these options with a ``pytest`` keyword
+search.  For example, to see the names of all tests that have "spec"
+or "concretize" somewhere in their names:
+
+.. command-output:: spack test --list-names -k "spec and concretize"
+
+By default, ``pytest`` captures the output of all unit tests, and it will
+print any captured output for failed tests. Sometimes it's helpful to see
+your output interactively, while the tests run (e.g., if you add print
+statements to a unit tests).  To see the output *live*, use the ``-s``
+argument to ``pytest``:
 
 .. code-block:: console
 
-   $ spack test -s -k architecture
+   $ spack test -s architecture.py::test_platform
 
-Unit tests are crucial to making sure bugs aren't introduced into Spack. If you
-are modifying core Spack libraries or adding new functionality, please consider
-adding new unit tests or strengthening existing tests.
+Unit tests are crucial to making sure bugs aren't introduced into
+Spack. If you are modifying core Spack libraries or adding new
+functionality, please add new unit tests for your feature, and consider
+strengthening existing tests.  You will likely be asked to do this if you
+submit a pull request to the Spack project on GitHub.  Check out the
+`pytest docs <http://pytest.org/>`_ and feel free to ask for guidance on
+how to write tests!
 
 .. note::
 
-   There is also a ``run-unit-tests`` script in ``share/spack/qa`` that
-   runs the unit tests. Afterwards, it reports back to Codecov with the
-   percentage of Spack that is covered by unit tests. This script is
-   designed for Travis CI. If you want to run the unit tests yourself, we
-   suggest you use ``spack test``.
+   You may notice the ``share/spack/qa/run-unit-tests`` script in the
+   repository.  This script is designed for Travis CI.  It runs the unit
+   tests and reports coverage statistics back to Codecov. If you want to
+   run the unit tests yourself, we suggest you use ``spack test``.
 
 ^^^^^^^^^^^^
 Flake8 Tests
@@ -223,8 +268,7 @@ documentation. In order to prevent things like broken links and missing imports,
 we added documentation tests that build the documentation and fail if there
 are any warning or error messages.
 
-Building the documentation requires several dependencies, all of which can be
-installed with Spack:
+Building the documentation requires several dependencies:
 
 * sphinx
 * sphinxcontrib-programoutput
@@ -234,11 +278,18 @@ installed with Spack:
 * mercurial
 * subversion
 
+All of these can be installed with Spack, e.g.
+
+.. code-block:: console
+
+   $ spack install py-sphinx py-sphinxcontrib-programoutput py-sphinx-rtd-theme graphviz git mercurial subversion
+
 .. warning::
 
    Sphinx has `several required dependencies <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/py-sphinx/package.py>`_.
-   If you installed ``py-sphinx`` with Spack, make sure to add all of these
-   dependencies to your ``PYTHONPATH``. The easiest way to do this is to run:
+   If you're using a ``python`` from Spack and you installed
+   ``py-sphinx`` and friends, you need to make them available to your
+   ``python``. The easiest way to do this is to run:
 
    .. code-block:: console
 
@@ -246,8 +297,10 @@ installed with Spack:
       $ spack activate py-sphinx-rtd-theme
       $ spack activate py-sphinxcontrib-programoutput
 
-   so that all of the dependencies are symlinked to a central location.
-   If you see an error message like:
+   so that all of the dependencies are symlinked into that Python's
+   tree.  Alternatively, you could arrange for their library
+   directories to be added to PYTHONPATH.  If you see an error message
+   like:
 
    .. code-block:: console
 

lib/spack/docs/developer_guide.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -363,12 +363,12 @@ Developer commands
 ``spack doc``
 ^^^^^^^^^^^^^
 
-.. _cmd-spack-test:
-
 ^^^^^^^^^^^^^^
 ``spack test``
 ^^^^^^^^^^^^^^
 
+See the :ref:`contributor guide section <cmd-spack-test>` on ``spack test``.
+
 .. _cmd-spack-python:
 
 ^^^^^^^^^^^^^^^^

lib/spack/docs/docker_for_developers.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/environments.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -49,6 +49,8 @@ Spack uses a "manifest and lock" model similar to `Bundler gemfiles
 managers. The user input file is named ``spack.yaml`` and the lock
 file is named ``spack.lock``
 
+.. _environments-using:
+
 ------------------
 Using Environments
 ------------------
@@ -382,11 +384,12 @@ the Environment.
 Loading
 ^^^^^^^
 
-Once an environment has been installed, the following creates a load script for it:
+Once an environment has been installed, the following creates a load
+script for it:
 
 .. code-block:: console
 
-   $ spack env myenv loads -r
+   $ spack env loads -r
 
 This creates a file called ``loads`` in the environment directory.
 Sourcing that file in Bash will make the environment available to the

lib/spack/docs/extensions.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -9,12 +9,6 @@
 Custom Extensions
 =================
 
-.. warning::
-
-   The support for extending Spack with custom commands is still experimental.
-   Users should expect APIs or prescribed directory structures to
-   change at any time.
-
 *Spack extensions* permit you to extend Spack capabilities by deploying your
 own custom commands or logic in an arbitrary location on your filesystem.
 This might be extremely useful e.g. to develop and maintain a command whose purpose is

lib/spack/docs/features.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/getting_started.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -97,7 +97,7 @@ Check Installation
 With Spack installed, you should be able to run some basic Spack
 commands.  For example:
 
-.. command-output:: spack spec netcdf
+.. command-output:: spack spec netcdf-c
 
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^

lib/spack/docs/index.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -66,6 +66,7 @@ or refer to the full manual below.
    config_yaml
    build_settings
    environments
+   containers
    mirrors
    module_file_support
    repositories
@@ -74,6 +75,7 @@ or refer to the full manual below.
    package_list
    chain
    extensions
+   pipelines
 
 .. toctree::
    :maxdepth: 2

lib/spack/docs/known_issues.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/mirrors.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/module_file_support.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -119,7 +119,7 @@ For example this will add the ``mpich`` package built with ``gcc`` to your path:
 
    # ... wait for install ...
 
-   $ spack load mpich %gcc@4.4.7    # modules
+   $ spack load mpich %gcc@4.4.7
    $ which mpicc
    ~/spack/opt/linux-debian7-x86_64/gcc@4.4.7/mpich@3.0.4/bin/mpicc
 
@@ -129,27 +129,29 @@ want to use a package, you can type unload or unuse similarly:
 
 .. code-block:: console
 
-   $ spack unload mpich %gcc@4.4.7  # modules
+   $ spack unload mpich %gcc@4.4.7
 
 .. note::
 
-   The ``load`` and ``unload`` subcommands are
-   only available if you have enabled Spack's shell support *and* you
-   have environment-modules installed on your machine.
+   The ``load`` and ``unload`` subcommands are only available if you
+   have enabled Spack's shell support. These command DO NOT use the
+   underlying Spack-generated module files.
 
-^^^^^^^^^^^^^^^^^^^^^^
-Ambiguous module names
-^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^
+Ambiguous specs
+^^^^^^^^^^^^^^^
 
-If a spec used with load/unload or use/unuse is ambiguous (i.e. more
-than one installed package matches it), then Spack will warn you:
+If a spec used with load/unload or is ambiguous (i.e. more than one
+installed package matches it), then Spack will warn you:
 
 .. code-block:: console
 
    $ spack load libelf
-   ==> Error: Multiple matches for spec libelf.  Choose one:
-   libelf@0.8.13%gcc@4.4.7 arch=linux-debian7-x86_64
-   libelf@0.8.13%intel@15.0.0 arch=linux-debian7-x86_64
+   ==> Error: libelf matches multiple packages.
+   Matching packages:
+     libelf@0.8.13%gcc@4.4.7 arch=linux-debian7-x86_64
+     libelf@0.8.13%intel@15.0.0 arch=linux-debian7-x86_64
+   Use a more specific spec
 
 You can either type the ``spack load`` command again with a fully
 qualified argument, or you can add just enough extra constraints to
@@ -171,8 +173,15 @@ To identify just the one built with the Intel compiler.
 ``spack module tcl loads``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-In some cases, it is desirable to load not just a module, but also all
-the modules it depends on.  This is not required for most modules
+In some cases, it is desirable to use a Spack-generated module, rather
+than relying on Spack's built-in user-environment modification
+capabilities. To translate a spec into a module name, use ``spack
+module tcl loads`` or ``spack module lmod loads`` depending on the
+module system desired.
+
+
+To load not just a module, but also all the modules it depends on, use
+the ``--dependencies`` option. This is not required for most modules
 because Spack builds binaries with RPATH support.  However, not all
 packages use RPATH to find their dependencies: this can be true in
 particular for Python extensions, which are currently *not* built with

lib/spack/docs/package_list.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/packaging_guide.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -553,6 +553,34 @@ version. This is useful for packages that have an easy to extrapolate URL, but
 keep changing their URL format every few releases. With this method, you only
 need to specify the ``url`` when the URL changes.
 
+"""""""""""""""""""""""
+Mirrors of the main URL
+"""""""""""""""""""""""
+
+Spack supports listing mirrors of the main URL in a package by defining
+the ``urls`` attribute:
+
+.. code-block:: python
+
+  class Foo(Package):
+
+    urls = [
+        'http://example.com/foo-1.0.tar.gz',
+        'http://mirror.com/foo-1.0.tar.gz'
+    ]
+
+instead of just a single ``url``. This attribute is a list of possible URLs that
+will be tried in order when fetching packages. Notice that either one of ``url``
+or ``urls`` can be present in a package, but not both at the same time.
+
+A well-known case of packages that can be fetched from multiple mirrors is that
+of GNU. For that, Spack goes a step further and defines a mixin class that
+takes care of all of the plumbing and requires packagers to just define a proper
+``gnu_mirror_path`` attribute:
+
+.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/autoconf/package.py
+   :lines: 9-18
+
 ^^^^^^^^^^^^^^^^^^^^^^^^
 Skipping the expand step
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -901,6 +929,9 @@ Git fetching supports the following parameters to ``version``:
 * ``tag``: Name of a tag to fetch.
 * ``commit``: SHA hash (or prefix) of a commit to fetch.
 * ``submodules``: Also fetch submodules recursively when checking out this repository.
+* ``submodules_delete``: A list of submodules to forcibly delete from the repository
+  after fetching. Useful if a version in the repository has submodules that
+  have disappeared/are no longer accessible.
 * ``get_full_repo``: Ensure the full git history is checked out with all remote
   branch information. Normally (``get_full_repo=False``, the default), the git
   option ``--depth 1`` will be used if the version of git and the specified
@@ -1479,8 +1510,8 @@ that the same package with different patches applied will have different
 hash identifiers.  To ensure that the hashing scheme is consistent, you
 must use a ``sha256`` checksum for the patch.  Patches will be fetched
 from their URLs, checked, and applied to your source code.  You can use
-the ``spack sha256`` command to generate a checksum for a patch file or
-URL.
+the GNU utils ``sha256sum`` or the macOS ``shasum -a 256`` commands to
+generate a checksum for a patch file.
 
 Spack can also handle compressed patches.  If you use these, Spack needs
 a little more help.  Specifically, it needs *two* checksums: the
@@ -1922,6 +1953,8 @@ issues with 1.64.0, 1.65.0, and 1.66.0, you can say:
    depends_on('boost@1.59.0:1.63,1.65.1,1.67.0:')
 
 
+.. _dependency-types:
+
 ^^^^^^^^^^^^^^^^
 Dependency types
 ^^^^^^^^^^^^^^^^
@@ -1959,6 +1992,28 @@ inject the dependency's ``prefix/lib`` directory, but the package needs to
 be in ``PATH`` and ``PYTHONPATH`` during the build process and later when
 a user wants to run the package.
 
+^^^^^^^^^^^^^^^^^^^^^^^^
+Conditional dependencies
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+You may have a package that only requires a dependency under certain
+conditions. For example, you may have a package that has optional MPI support,
+- MPI is only a dependency when you want to enable MPI support for the
+package. In that case, you could say something like:
+
+.. code-block:: python
+
+   variant('mpi', default=False)
+   depends_on('mpi', when='+mpi')
+
+``when`` can include constraints on the variant, version, compiler, etc. and
+the :mod:`syntax<spack.spec>` is the same as for Specs written on the command
+line.
+
+If a dependency/feature of a package isn't typically used, you can save time
+by making it conditional (since Spack will not build the dependency unless it
+is required for the Spec).
+
 .. _dependency_dependency_patching:
 
 ^^^^^^^^^^^^^^^^^^^
@@ -4399,7 +4454,7 @@ translate variant flags into CMake definitions.  For example:
 
 .. code-block:: python
 
-   def configure_args(self):
+   def cmake_args(self):
        spec = self.spec
        return [
            '-DUSE_EVERYTRACE=%s' % ('YES' if '+everytrace' in spec else 'NO'),

lib/spack/docs/pipelines.rst

@@ -0,0 +1,439 @@
+.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+   Spack Project Developers. See the top-level COPYRIGHT file for details.
+
+   SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+.. _pipelines:
+
+=========
+Pipelines
+=========
+
+Spack provides commands that support generating and running automated build
+pipelines designed for Gitlab CI.  At the highest level it works like this:
+provide a spack environment describing the set of packages you care about,
+and include within that environment file a description of how those packages
+should be mapped to Gitlab runners.  Spack can then generate a ``.gitlab-ci.yml``
+file containing job descriptions for all your packages that can be run by a
+properly configured Gitlab CI instance.  When run, the generated pipeline will
+build and deploy binaries, and it can optionally report to a CDash instance
+regarding the health of the builds as they evolve over time.
+
+------------------------------
+Getting started with pipelines
+------------------------------
+
+It is fairly straightforward to get started with automated build pipelines.  At
+a minimum, you'll need to set up a Gitlab instance (more about Gitlab CI
+`here <https://about.gitlab.com/product/continuous-integration/>`_) and configure
+at least one `runner <https://docs.gitlab.com/runner/>`_.  Then the basic steps
+for setting up a build pipeline are as follows:
+
+#. Create a repository on your gitlab instance
+#. Add a ``spack.yaml`` at the root containing your pipeline environment (see
+   below for details)
+#. Add a ``.gitlab-ci.yml`` at the root containing a single job, similar to
+   this one:
+
+   .. code-block:: yaml
+
+      pipeline-job:
+        tags:
+          - <custom-tag>
+          ...
+        script:
+          - spack ci start
+
+#. Add any secrets required by the CI process to environment variables using the
+   CI web ui
+#. Push a commit containing the ``spack.yaml`` and ``.gitlab-ci.yml`` mentioned above
+   to the gitlab repository
+
+The ``<custom-tag>``, above, is used to pick one of your configured runners,
+while the use of the ``spack ci start`` command implies that runner has an
+appropriate version of spack installed and configured for use.  Of course, there
+are myriad ways to customize the process.  You can configure CDash reporting
+on the progress of your builds, set up S3 buckets to mirror binaries built by
+the pipeline, clone a custom spack repository/ref for use by the pipeline, and
+more.
+
+While it is possible to set up pipelines on gitlab.com, the builds there are
+limited to 60 minutes and generic hardware.  It is also possible to
+`hook up <https://about.gitlab.com/blog/2018/04/24/getting-started-gitlab-ci-gcp>`_
+Gitlab to Google Kubernetes Engine (`GKE <https://cloud.google.com/kubernetes-engine/>`_)
+or Amazon Elastic Kubernetes Service (`EKS <https://aws.amazon.com/eks>`_), though those
+topics are outside the scope of this document.
+
+-----------------------------------
+Spack commands supporting pipelines
+-----------------------------------
+
+Spack provides a command `ci` with sub-commands for doing various things related
+to automated build pipelines.  All of the ``spack ci ...`` commands must be run
+from within a environment, as each one makes use of the environment for different
+purposes.  Additionally, some options to the commands (or conditions present in
+the spack environment file) may require particular environment variables to be
+set in order to function properly.  Examples of these are typically secrets
+needed for pipeline operation that should not be visible in a spack environment
+file.  These environment variables are described in more detail
+:ref:`ci_environment_variables`.
+
+.. _cmd_spack_ci:
+
+^^^^^^^^^^^^^^^^^^
+``spack ci``
+^^^^^^^^^^^^^^^^^^
+
+Super-command for functionality related to generating pipelines and executing
+pipeline jobs.
+
+.. _cmd_spack_ci_start:
+
+^^^^^^^^^^^^^^^^^^
+``spack ci start``
+^^^^^^^^^^^^^^^^^^
+
+Currently this command is a short-cut to first run ``spack ci generate``, followed
+by ``spack ci pushyaml``.
+
+.. _cmd_spack_ci_generate:
+
+^^^^^^^^^^^^^^^^^^^^^
+``spack ci generate``
+^^^^^^^^^^^^^^^^^^^^^
+
+Concretizes the specs in the active environment, stages them (as described in
+:ref:`staging_algorithm`), and writes the resulting ``.gitlab-ci.yml`` to disk.
+
+.. _cmd_spack_ci_pushyaml:
+
+^^^^^^^^^^^^^^^^^^^^^
+``spack ci pushyaml``
+^^^^^^^^^^^^^^^^^^^^^
+
+Generates a commit containing the generated ``.gitlab-ci.yml`` and pushes it to a
+``DOWNSTREAM_CI_REPO``, which is frequently the same repository.  The branch
+created has the same name as the current branch being tested, but has ``multi-ci-``
+prepended to the branch name.  Once Gitlab CI has full support for dynamically
+defined workloads, this command will be deprecated.
+
+.. _cmd_spack_ci_rebuild:
+
+^^^^^^^^^^^^^^^^^^^^
+``spack ci rebuild``
+^^^^^^^^^^^^^^^^^^^^
+
+This sub-command is responsible for ensuring a single spec from the release
+environment is up to date on the remote mirror configured in the environment,
+and as such, corresponds to a single job in the ``.gitlab-ci.yml`` file.
+
+------------------------------------
+A pipeline-enabled spack environment
+------------------------------------
+
+Here's an example of a spack environment file that has been enhanced with
+sections desribing a build pipeline:
+
+.. code-block:: yaml
+
+   spack:
+     definitions:
+     - pkgs:
+       - readline@7.0
+     - compilers:
+       - '%gcc@5.5.0'
+     - oses:
+       - os=ubuntu18.04
+       - os=centos7
+     specs:
+     - matrix:
+       - [$pkgs]
+       - [$compilers]
+       - [$oses]
+     mirrors:
+       cloud_gitlab: https://mirror.spack.io
+     gitlab-ci:
+       mappings:
+         - match:
+             - os=ubuntu18.04
+           runner-attributes:
+             tags:
+               - spack-k8s
+             image: spack/spack_builder_ubuntu_18.04
+         - match:
+             - os=centos7
+           runner-attributes:
+             tags:
+               - spack-k8s
+             image: spack/spack_builder_centos_7
+     cdash:
+       build-group: Release Testing
+       url: https://cdash.spack.io
+       project: Spack
+       site: Spack AWS Gitlab Instance
+
+Hopefully, the ``definitions``, ``specs``, ``mirrors``, etc. sections are already
+familiar, as they are part of spack :ref:`environments`.  So let's take a more
+in-depth look some of the pipeline-related sections in that environment file
+that might not be as familiar.
+
+The ``gitlab-ci`` section is used to configure how the pipeline workload should be
+generated, mainly how the jobs for building specs should be assigned to the
+configured runners on your instance.  Each entry within the list of ``mappings``
+corresponds to a known gitlab runner, where the ``match`` section is used
+in assigning a release spec to one of the runners, and the ``runner-attributes``
+section is used to configure the spec/job for that particular runner.
+
+There are other pipeline options you can configure within the ``gitlab-ci`` section
+as well.  The ``bootstrap`` section allows you to specify lists of specs from
+your ``definitions`` that should be staged ahead of the environment's ``specs`` (this
+section is described in more detail below).  The ``enable-artifacts-buildcache`` key
+takes a boolean and determines whether the pipeline uses artifacts to store and
+pass along the buildcaches from one stage to the next (the default if you don't
+provide this option is ``False``).  The ``enable-debug-messages`` key takes a boolean
+and allows you to choose whether the pipeline build jobs are run as ``spack -d ci rebuild``
+or just ``spack ci rebuild`` (the default is not to enable debug messages).  The
+``final-stage-rebuild-index`` section controls whether an extra job is added to the
+end of your pipeline (in a stage by itself) which will regenerate the mirror's
+buildcache index.  Under normal operation, each pipeline job that rebuilds a package
+will re-generate the mirror's buildcache index after the buildcache entry for that
+job has been created and pushed to the mirror.  Since jobs in the same stage can run in
+parallel, there is the possibility that at the end of some stage, the index may not
+reflect all the binaries in the buildcache.  Adding the ``final-stage-rebuild-index``
+section ensures that at the end of the pipeline, the index will be in sync with the
+binaries on the mirror.  If the mirror lives in an S3 bucket, this job will need to
+run on a machine with the Python ``boto3`` module installed, and consequently the
+``final-stage-rebuild-index`` needs to specify a list of ``tags`` to pick a runner
+satisfying that condition.  It can also take an ``image`` key so Docker executor type
+runners can pick the right image for the index regeneration job.
+
+The optional ``cdash`` section provides information that will be used by the
+``spack ci generate`` command (invoked by ``spack ci start``) for reporting
+to CDash.  All the jobs generated from this environment will belong to a
+"build group" within CDash that can be tracked over time.  As the release
+progresses, this build group may have jobs added or removed. The url, project,
+and site are used to specify the CDash instance to which build results should
+be reported.
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Assignment of specs to runners
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``mappings`` section corresponds to a list of runners, and during assignment
+of specs to runners, the list is traversed in order looking for matches, the
+first runner that matches a release spec is assigned to build that spec.  The
+``match`` section within each runner mapping section is a list of specs, and
+if any of those specs match the release spec (the ``spec.satisfies()`` method
+is used), then that runner is considered a match.
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Configuration of specs/jobs for a runner
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Once a runner has been chosen to build a release spec, the ``runner-attributes``
+section provides information determining details of the job in the context of
+the runner.  The ``runner-attributes`` section must have a ``tags`` key, which
+is a list containing at least one tag used to select the runner from among the
+runners known to the gitlab instance.  For Docker executor type runners, the
+``image`` key is used to specify the Docker image used to build the release spec
+(and could also appear as a dictionary with a ``name`` specifying the image name,
+as well as an ``entrypoint`` to override whatever the default for that image is).
+For other types of runners the ``variables`` key will be useful to pass any
+information on to the runner that it needs to do its work (e.g. scheduler
+parameters, etc.).
+
+.. _staging_algorithm:
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Summary of ``.gitlab-ci.yml`` generation algorithm
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+All specs yielded by the matrix (or all the specs in the environment) have their
+dependencies computed, and the entire resulting set of specs are staged together
+before being run through the ``gitlab-ci/mappings`` entries, where each staged
+spec is assigned a runner.  "Staging" is the name we have given to the process
+of figuring out in what order the specs should be built, taking into consideration
+Gitlab CI rules about jobs/stages.  In the staging process the goal is to maximize
+the number of jobs in any stage of the pipeline, while ensuring that the jobs in
+any stage only depend on jobs in previous stages (since those jobs are guaranteed
+to have completed already).  As a runner is determined for a job, the information
+in the ``runner-attributes`` is used to populate various parts of the job
+description that will be used by Gitlab CI. Once all the jobs have been assigned
+a runner, the ``.gitlab-ci.yml`` is written to disk.
+
+The short example provided above would result in the ``readline``, ``ncurses``,
+and ``pkgconf`` packages getting staged and built on the runner chosen by the
+``spack-k8s`` tag.  In this example, we assume the runner is a Docker executor
+type runner, and thus certain jobs will be run in the ``centos7`` container,
+and others in the ``ubuntu-18.04`` container.  The resulting ``.gitlab-ci.yml``
+will contain 6 jobs in three stages.  Once the jobs have been generated, the
+presence of a ``SPACK_CDASH_AUTH_TOKEN`` environment variable during the
+``spack ci generate`` command would result in all of the jobs being put in a
+build group on CDash called "Release Testing" (that group will be created if
+it didn't already exist).
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Optional compiler bootstrapping
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Spack pipelines also have support for bootstrapping compilers on systems that
+may not already have the desired compilers installed. The idea here is that
+you can specify a list of things to bootstrap in your ``definitions``, and
+spack will guarantee those will be installed in a phase of the pipeline before
+your release specs, so that you can rely on those packages being available in
+the binary mirror when you need them later on in the pipeline.  At the moment
+the only viable use-case for bootstrapping is to install compilers.
+
+Here's an example of what bootstrapping some compilers might look like:
+
+.. code-block:: yaml
+
+   spack:
+     definitions:
+     - compiler-pkgs:
+       - 'llvm+clang@6.0.1 os=centos7'
+       - 'gcc@6.5.0 os=centos7'
+       - 'llvm+clang@6.0.1 os=ubuntu18.04'
+       - 'gcc@6.5.0 os=ubuntu18.04'
+     - pkgs:
+       - readline@7.0
+     - compilers:
+       - '%gcc@5.5.0'
+       - '%gcc@6.5.0'
+       - '%gcc@7.3.0'
+       - '%clang@6.0.0'
+       - '%clang@6.0.1'
+     - oses:
+       - os=ubuntu18.04
+       - os=centos7
+     specs:
+     - matrix:
+       - [$pkgs]
+       - [$compilers]
+       - [$oses]
+       exclude:
+         - '%gcc@7.3.0 os=centos7'
+         - '%gcc@5.5.0 os=ubuntu18.04'
+     gitlab-ci:
+       bootstrap:
+         - name: compiler-pkgs
+           compiler-agnostic: true
+       mappings:
+         # mappings similar to the example higher up in this description
+         ...
+
+In the example above, we have added a list to the ``definitions`` called
+``compiler-pkgs`` (you can add any number of these), which lists compiler packages
+we want to be staged ahead of the full matrix of release specs (which consists
+only of readline in our example).  Then within the ``gitlab-ci`` section, we
+have added a ``bootstrap`` section, which can contain a list of items, each
+referring to a list in the ``definitions`` section.  These items can either
+be a dictionary or a string.  If you supply a dictionary, it must have a name
+key whose value must match one of the lists in definitions and it can have a
+``compiler-agnostic`` key whose value is a boolean.  If you supply a string,
+then it needs to match one of the lists provided in ``definitions``.  You can
+think of the bootstrap list as an ordered list of pipeline "phases" that will
+be staged before your actual release specs.  While this introduces another
+layer of bottleneck in the pipeline (all jobs in all stages of one phase must
+complete before any jobs in the next phase can begin), it also means you are
+guaranteed your bootstrapped compilers will be available when you need them.
+
+The ``compiler-agnostic`` key can be provided with each item in the
+bootstrap list. It tells the ``spack ci generate`` command that any jobs staged
+from that particular list should have the compiler removed from the spec, so
+that any compiler available on the runner where the job is run can be used to
+build the package.
+
+When including a bootstrapping phase as in the example above, the result is that
+the bootstrapped compiler packages will be pushed to the binary mirror (and the
+local artifacts mirror) before the actual release specs are built. In this case,
+the jobs corresponding to subsequent release specs are configured to
+``install_missing_compilers``, so that if spack is asked to install a package
+with a compiler it doesn't know about, it can be quickly installed from the
+binary mirror first.
+
+Since bootstrapping compilers is optional, those items can be left out of the
+environment/stack file, and in that case no bootstrapping will be done (only the
+specs will be staged for building) and the runners will be expected to already
+have all needed compilers installed and configured for spack to use.
+
+-------------------------------------
+Using a custom spack in your pipeline
+-------------------------------------
+
+If your runners will not have a version of spack ready to invoke, or if for some
+other reason you want to use a custom version of spack to run your pipelines,
+this can be accomplished fairly simply.  First, create CI environment variables
+containing the url and branch/tag you want to clone (calling them, for example,
+``SPACK_REPO`` and ``SPACK_REF``), use them to clone spack in your pre-ci
+``before_script``, and finally pass those same values along to the workload
+generation process via the ``spack-repo`` and ``spack-ref`` cli args.  Here's
+an example:
+
+.. code-block:: yaml
+
+   pipeline-job:
+     tags:
+       - <some-other-tag>
+   before_script:
+     - git clone ${SPACK_REPO} --branch ${SPACK_REF}
+     - . ./spack/share/spack/setup-env.sh
+   script:
+     - spack ci start --spack-repo ${SPACK_REPO} --spack-ref ${SPACK_REF} <...args>
+   after_script:
+     - rm -rf ./spack
+
+If the ``spack ci start`` command receives those extra command line arguments,
+then it adds similar ``before_script`` and ``after_script`` sections for each of
+the ``spack ci rebuild`` jobs it generates (cloning and sourcing a custom
+spack in the ``before_script`` and removing it again in the ``after_script``).
+This gives you control over the version of spack used when the rebuild jobs
+are actually run on the gitlab runner.
+
+.. _ci_environment_variables:
+
+--------------------------------------------------
+Environment variables affecting pipeline operation
+--------------------------------------------------
+
+Certain secrets and some other information should be provided to the pipeline
+infrastructure via environment variables, usually for reasons of security, but
+in some cases to support other pipeline use cases such as PR testing.  The
+environment variables used by the pipeline infrastructure are described here.
+
+^^^^^^^^^^^^^^^^^
+AWS_ACCESS_KEY_ID
+^^^^^^^^^^^^^^^^^
+
+Needed when binary mirror is an S3 bucket.
+
+^^^^^^^^^^^^^^^^^^^^^
+AWS_SECRET_ACCESS_KEY
+^^^^^^^^^^^^^^^^^^^^^
+
+Needed when binary mirror is an S3 bucket.
+
+^^^^^^^^^^^^^^^
+S3_ENDPOINT_URL
+^^^^^^^^^^^^^^^
+
+Needed when binary mirror is an S3 bucket that is *not* on AWS.
+
+^^^^^^^^^^^^^^^^^
+CDASH_AUTH_TOKEN
+^^^^^^^^^^^^^^^^^
+
+Needed in order to report build groups to CDash.
+
+^^^^^^^^^^^^^^^^^
+SPACK_SIGNING_KEY
+^^^^^^^^^^^^^^^^^
+
+Needed to sign/verify binary packages from the remote binary mirror.
+
+^^^^^^^^^^^^^^^^^^
+DOWNSTREAM_CI_REPO
+^^^^^^^^^^^^^^^^^^
+
+Needed until Gitlab CI supports dynamic job generation.  Can contain connection
+credentials, and could be the same repository or a different one.

lib/spack/docs/repositories.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/requirements.txt

@@ -1,7 +1,7 @@
 # These dependencies should be installed using pip in order
 # to build the documentation.
 
-sphinx==2.0.1
-sphinxcontrib-programoutput==0.14
-sphinx-rtd-theme==0.4.3
+sphinx
+sphinxcontrib-programoutput
+sphinx-rtd-theme
 python-levenshtein

lib/spack/docs/spack.yaml

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/docs/workflows.rst

@@ -1,4 +1,4 @@
-.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+.. Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
    Spack Project Developers. See the top-level COPYRIGHT file for details.
 
    SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -253,14 +253,14 @@ However, other more powerful methods are generally preferred for user
 environments.
 
 
-^^^^^^^^^^^^^^^^^^^^^^^
-Spack-Generated Modules
-^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Using ``spack load`` to Manage the User Environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Suppose that Spack has been used to install a set of command-line
 programs, which users now wish to use.  One can in principle put a
 number of ``spack load`` commands into ``.bashrc``, for example, to
-load a set of Spack-generated modules:
+load a set of Spack packages:
 
 .. code-block:: sh
 
@@ -273,7 +273,7 @@ load a set of Spack-generated modules:
 Although simple load scripts like this are useful in many cases, they
 have some drawbacks:
 
-1. The set of modules loaded by them will in general not be
+1. The set of packages loaded by them will in general not be
    consistent.  They are a decent way to load commands to be called
    from command shells.  See below for better ways to assemble a
    consistent set of packages for building application programs.
@@ -285,19 +285,24 @@ have some drawbacks:
    other hand, are not very smart: if the user-supplied spec matches
    more than one installed package, then ``spack module tcl loads`` will
    fail. This may change in the future.  For now, the workaround is to
-   be more specific on any ``spack module tcl loads`` lines that fail.
+   be more specific on any ``spack load`` commands that fail.
 
 
 """"""""""""""""""""""
 Generated Load Scripts
 """"""""""""""""""""""
 
-Another problem with using `spack load` is, it is slow; a typical user
-environment could take several seconds to load, and would not be
-appropriate to put into ``.bashrc`` directly.  It is preferable to use
-a series of ``spack module tcl loads`` commands to pre-compute which
-modules to load.  These can be put in a script that is run whenever
-installed Spack packages change.  For example:
+Another problem with using `spack load` is, it can be slow; a typical
+user environment could take several seconds to load, and would not be
+appropriate to put into ``.bashrc`` directly.  This is because it
+requires the full start-up overhead of python/Spack for each command.
+In some circumstances it is preferable to use a series of ``spack
+module tcl loads`` (or ``spack module lmod loads``) commands to
+pre-compute which modules to load.  This will generate the modulenames
+to load the packages using environment modules, rather than Spack's
+built-in support for environment modifications. These can be put in a
+script that is run whenever installed Spack packages change.  For
+example:
 
 .. code-block:: sh
 
@@ -634,7 +639,7 @@ Global Activations
 Python (and similar systems) packages directly or creating a view.
 If extensions are globally activated, then ``spack load python`` will
 also load all the extensions activated for the given ``python``.
-This reduces the need for users to load a large number of modules.
+This reduces the need for users to load a large number of packages.
 
 However, Spack global activations have two potential drawbacks:
 
@@ -1090,6 +1095,248 @@ or filesystem views.  However, it has some drawbacks:
    integrate Spack explicitly in their workflow.  Not all users are
    willing to do this.
 
+-------------------------------------
+Using Spack to Replace Homebrew/Conda
+-------------------------------------
+
+Spack is an incredibly powerful package manager, designed for supercomputers
+where users have diverse installation needs. But Spack can also be used to
+handle simple single-user installations on your laptop. Most macOS users are
+already familiar with package managers like Homebrew and Conda, where all
+installed packages are symlinked to a single central location like ``/usr/local``.
+In this section, we will show you how to emulate the behavior of Homebrew/Conda
+using :ref:`environments`!
+
+^^^^^
+Setup
+^^^^^
+
+First, let's create a new environment. We'll assume that Spack is already set up
+correctly, and that you've already sourced the setup script for your shell.
+To create a new environment, simply run:
+
+.. code-block:: console
+
+   $ spack env create myenv
+   ==> Updating view at /Users/me/spack/var/spack/environments/myenv/.spack-env/view
+   ==> Created environment 'myenv' in /Users/me/spack/var/spack/environments/myenv
+   $ spack env activate myenv
+
+Here, *myenv* can be anything you want to name your environment. Next, we can add
+a list of packages we would like to install into our environment. Let's say we
+want a newer version of Bash than the one that comes with macOS, and we want a
+few Python libraries. We can run:
+
+.. code-block:: console
+
+   $ spack add bash
+   ==> Adding bash to environment myenv
+   ==> Updating view at /Users/me/spack/var/spack/environments/myenv/.spack-env/view
+   $ spack add python@3:
+   ==> Adding python@3: to environment myenv
+   ==> Updating view at /Users/me/spack/var/spack/environments/myenv/.spack-env/view
+   $ spack add py-numpy py-scipy py-matplotlib
+   ==> Adding py-numpy to environment myenv
+   ==> Adding py-scipy to environment myenv
+   ==> Adding py-matplotlib to environment myenv
+   ==> Updating view at /Users/me/spack/var/spack/environments/myenv/.spack-env/view
+
+Each package can be listed on a separate line, or combined into a single line.
+Notice that we're explicitly asking for Python 3 here. You can use any spec
+you would normally use on the command line with other Spack commands.
+
+Next, we want to manually configure a couple of things. In the ``myenv``
+directory, we can find the ``spack.yaml`` that actually defines our environment.
+
+.. code-block:: console
+
+   $ vim ~/spack/var/spack/environments/myenv/spack.yaml
+
+.. code-block:: yaml
+
+   # This is a Spack Environment file.
+   #
+   # It describes a set of packages to be installed, along with
+   # configuration settings.
+   spack:
+     # add package specs to the `specs` list
+     specs: [bash, 'python@3:', py-numpy, py-scipy, py-matplotlib]
+     view:
+       default:
+         root: /Users/me/spack/var/spack/environments/myenv/.spack-env/view
+         projections: {}
+     config: {}
+     mirrors: {}
+     modules:
+       enable: []
+     packages: {}
+     repos: []
+     upstreams: {}
+     definitions: []
+     concretization: separately
+
+You can see the packages we added earlier in the ``specs:`` section. If you
+ever want to add more packages, you can either use ``spack add`` or manually
+edit this file.
+
+We also need to change the ``concretization:`` option. By default, Spack
+concretizes each spec *separately*, allowing multiple versions of the same
+package to coexist. Since we want a single consistent environment, we want to
+concretize all of the specs *together*.
+
+Here is what your ``spack.yaml`` looks like with these new settings, and with
+some of the sections we don't plan on using removed:
+
+.. code-block:: diff
+
+   spack:
+   -  specs: [bash, 'python@3:', py-numpy, py-scipy, py-matplotlib]
+   +  specs:
+   +  - bash
+   +  - 'python@3:'
+   +  - py-numpy
+   +  - py-scipy
+   +  - py-matplotlib
+   -  view:
+   -    default:
+   -      root: /Users/me/spack/var/spack/environments/myenv/.spack-env/view
+   -      projections: {}
+   +  view: /Users/me/spack/var/spack/environments/myenv/.spack-env/view
+   -  config: {}
+   -  mirrors: {}
+   -  modules:
+   -    enable: []
+   -  packages: {}
+   -  repos: []
+   -  upstreams: {}
+   -  definitions: []
+   +  concretization: together
+   -  concretization: separately
+
+""""""""""""""""
+Symlink location
+""""""""""""""""
+
+In the ``spack.yaml`` file above, you'll notice that by default, Spack symlinks
+all installations to ``/Users/me/spack/var/spack/environments/myenv/.spack-env/view``.
+You can actually change this to any directory you want. For example, Homebrew
+uses ``/usr/local``, while Conda uses ``/Users/me/anaconda``. In order to access
+files in these locations, you need to update ``PATH`` and other environment variables
+to point to them. Activating the Spack environment does this automatically, but
+you can also manually set them in your ``.bashrc``.
+
+.. warning::
+
+   There are several reasons why you shouldn't use ``/usr/local``:
+
+   1. If you are on macOS 10.11+ (El Capitan and newer), Apple makes it hard
+      for you. You may notice permissions issues on ``/usr/local`` due to their
+      `System Integrity Protection <https://support.apple.com/en-us/HT204899>`_.
+      By default, users don't have permissions to install anything in ``/usr/local``,
+      and you can't even change this using ``sudo chown`` or ``sudo chmod``.
+   2. Other package managers like Homebrew will try to install things to the
+      same directory. If you plan on using Homebrew in conjunction with Spack,
+      don't symlink things to ``/usr/local``.
+   3. If you are on a shared workstation, or don't have sudo priveleges, you
+      can't do this.
+
+   If you still want to do this anyway, there are several ways around SIP.
+   You could disable SIP by booting into recovery mode and running
+   ``csrutil disable``, but this is not recommended, as it can open up your OS
+   to security vulnerabilities. Another technique is to run ``spack concretize``
+   and ``spack install`` using ``sudo``. This is also not recommended.
+
+   The safest way I've found is to create your installation directories using
+   sudo, then change ownership back to the user like so:
+
+   .. code-block:: bash
+
+      for directory in .spack bin contrib include lib man share
+      do
+          sudo mkdir -p /usr/local/$directory
+          sudo chown $(id -un):$(id -gn) /usr/local/$directory
+      done
+
+   Depending on the packages you install in your environment, the exact list of
+   directories you need to create may vary. You may also find some packages
+   like Java libraries that install a single file to the installation prefix
+   instead of in a subdirectory. In this case, the action is the same, just replace
+   ``mkdir -p`` with ``touch`` in the for-loop above.
+
+   But again, it's safer just to use the default symlink location.
+
+
+^^^^^^^^^^^^
+Installation
+^^^^^^^^^^^^
+
+To actually concretize the environment, run:
+
+.. code-block:: console
+
+   $ spack concretize
+
+This will tell you which if any packages are already installed, and alert you
+to any conflicting specs.
+
+To actually install these packages and symlink them to your ``view:``
+directory, simply run:
+
+.. code-block:: console
+
+   $ spack install
+
+Now, when you type ``which python3``, it should find the one you just installed.
+
+In order to change the default shell to our newer Bash installation, we first
+need to add it to this list of acceptable shells. Run:
+
+.. code-block:: console
+
+   $ sudo vim /etc/shells
+
+and add the absolute path to your bash executable. Then run:
+
+.. code-block:: console
+
+   $ chsh -s /path/to/bash
+
+Now, when you log out and log back in, ``echo $SHELL`` should point to the
+newer version of Bash.
+
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Updating Installed Packages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Let's say you upgraded to a new version of macOS, or a new version of Python
+was released, and you want to rebuild your entire software stack. To do this,
+simply run the following commands:
+
+.. code-block:: console
+
+   $ spack env activate myenv
+   $ spack concretize --force
+   $ spack install
+
+The ``--force`` flag tells Spack to overwrite its previous concretization
+decisions, allowing you to choose a new version of Python. If any of the new
+packages like Bash are already installed, ``spack install`` won't re-install
+them, it will keep the symlinks in place.
+
+^^^^^^^^^^^^^^
+Uninstallation
+^^^^^^^^^^^^^^
+
+If you decide that Spack isn't right for you, uninstallation is simple.
+Just run:
+
+.. code-block:: console
+
+   $ spack env activate myenv
+   $ spack uninstall --all
+
+This will uninstall all packages in your environment and remove the symlinks.
+
 ------------------------
 Using Spack on Travis-CI
 ------------------------
@@ -1254,7 +1501,7 @@ In order to build and run the image, execute:
    RUN        spack install tar \
               && spack clean -a
 
-   # need the modules already during image build?
+   # need the executables from a package already during image build?
    #RUN        /bin/bash -l -c ' \
    #                spack load tar \
    #                && which tar'

lib/spack/env/cc

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/external/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -82,14 +82,6 @@
   ini-parsing, io, code, and log facilities.
 * Version: 1.4.34 (last version supporting Python 2.6)
 
-pyqver
-------
-
-* Homepage: https://github.com/ghewgill/pyqver
-* Usage: External script to query required python version of
-  python source code. Used for ensuring 2.6 compatibility.
-* Version: Unversioned
-
 pytest
 ------
 

lib/spack/external/_pytest/pytester.py

@@ -569,7 +569,7 @@ def syspathinsert(self, path=None):
     def _possibly_invalidate_import_caches(self):
         # invalidate caches if we can (py33 and above)
         try:
-            import importlib  # nopyqver
+            import importlib
         except ImportError:
             pass
         else:

lib/spack/external/ordereddict_backport.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/external/pyqver2.py

@@ -1,344 +0,0 @@
-#!/usr/bin/env python
-#
-# pyqver2.py
-# by Greg Hewgill
-# https://github.com/ghewgill/pyqver
-#
-# This software is provided 'as-is', without any express or implied
-# warranty.  In no event will the author be held liable for any damages
-# arising from the use of this software.
-#
-# Permission is granted to anyone to use this software for any purpose,
-# including commercial applications, and to alter it and redistribute it
-# freely, subject to the following restrictions:
-#
-# 1. The origin of this software must not be misrepresented; you must not
-#    claim that you wrote the original software. If you use this software
-#    in a product, an acknowledgment in the product documentation would be
-#    appreciated but is not required.
-# 2. Altered source versions must be plainly marked as such, and must not be
-#    misrepresented as being the original software.
-# 3. This notice may not be removed or altered from any source distribution.
-#
-# Copyright (c) 2009-2013 Greg Hewgill http://hewgill.com
-#
-
-import compiler
-import platform
-import sys
-
-StandardModules = {
-    "__future__":       (2, 1),
-    "abc":              (2, 6),
-# skip argparse now that it's in lib/spack/external
-#    "argparse":         (2, 7),
-    "ast":              (2, 6),
-    "atexit":           (2, 0),
-    "bz2":              (2, 3),
-    "cgitb":            (2, 2),
-    "collections":      (2, 4),
-    "contextlib":       (2, 5),
-    "cookielib":        (2, 4),
-    "cProfile":         (2, 5),
-    "csv":              (2, 3),
-    "ctypes":           (2, 5),
-    "datetime":         (2, 3),
-    "decimal":          (2, 4),
-    "difflib":          (2, 1),
-    "DocXMLRPCServer":  (2, 3),
-    "dummy_thread":     (2, 3),
-    "dummy_threading":  (2, 3),
-    "email":            (2, 2),
-    "fractions":        (2, 6),
-    "functools":        (2, 5),
-    "future_builtins":  (2, 6),
-    "hashlib":          (2, 5),
-    "heapq":            (2, 3),
-    "hmac":             (2, 2),
-    "hotshot":          (2, 2),
-    "HTMLParser":       (2, 2),
-    "importlib":        (2, 7),
-    "inspect":          (2, 1),
-    "io":               (2, 6),
-    "itertools":        (2, 3),
-    "json":             (2, 6),
-    "logging":          (2, 3),
-    "modulefinder":     (2, 3),
-    "msilib":           (2, 5),
-    "multiprocessing":  (2, 6),
-    "netrc":            (1, 5, 2),
-    "numbers":          (2, 6),
-    "optparse":         (2, 3),
-    "ossaudiodev":      (2, 3),
-    "pickletools":      (2, 3),
-    "pkgutil":          (2, 3),
-    "platform":         (2, 3),
-    "pydoc":            (2, 1),
-    "runpy":            (2, 5),
-    "sets":             (2, 3),
-    "shlex":            (1, 5, 2),
-    "SimpleXMLRPCServer": (2, 2),
-    "spwd":             (2, 5),
-    "sqlite3":          (2, 5),
-    "ssl":              (2, 6),
-    "stringprep":       (2, 3),
-    "subprocess":       (2, 4),
-    "sysconfig":        (2, 7),
-    "tarfile":          (2, 3),
-    "textwrap":         (2, 3),
-    "timeit":           (2, 3),
-    "unittest":         (2, 1),
-    "uuid":             (2, 5),
-    "warnings":         (2, 1),
-    "weakref":          (2, 1),
-    "winsound":         (1, 5, 2),
-    "wsgiref":          (2, 5),
-    "xml.dom":          (2, 0),
-    "xml.dom.minidom":  (2, 0),
-    "xml.dom.pulldom":  (2, 0),
-    "xml.etree.ElementTree": (2, 5),
-    "xml.parsers.expat":(2, 0),
-    "xml.sax":          (2, 0),
-    "xml.sax.handler":  (2, 0),
-    "xml.sax.saxutils": (2, 0),
-    "xml.sax.xmlreader":(2, 0),
-    "xmlrpclib":        (2, 2),
-    "zipfile":          (1, 6),
-    "zipimport":        (2, 3),
-    "_ast":             (2, 5),
-    "_winreg":          (2, 0),
-}
-
-Functions = {
-    "all":                      (2, 5),
-    "any":                      (2, 5),
-    "collections.Counter":      (2, 7),
-    "collections.defaultdict":  (2, 5),
-    "collections.OrderedDict":  (2, 7),
-    "functools.total_ordering": (2, 7),
-    "enumerate":                (2, 3),
-    "frozenset":                (2, 4),
-    "itertools.compress":       (2, 7),
-    "math.erf":                 (2, 7),
-    "math.erfc":                (2, 7),
-    "math.expm1":               (2, 7),
-    "math.gamma":               (2, 7),
-    "math.lgamma":              (2, 7),
-    "memoryview":               (2, 7),
-    "next":                     (2, 6),
-    "os.getresgid":             (2, 7),
-    "os.getresuid":             (2, 7),
-    "os.initgroups":            (2, 7),
-    "os.setresgid":             (2, 7),
-    "os.setresuid":             (2, 7),
-    "reversed":                 (2, 4),
-    "set":                      (2, 4),
-    "subprocess.check_call":    (2, 5),
-    "subprocess.check_output":  (2, 7),
-    "sum":                      (2, 3),
-    "symtable.is_declared_global": (2, 7),
-    "weakref.WeakSet":          (2, 7),
-}
-
-Identifiers = {
-    "False":        (2, 2),
-    "True":         (2, 2),
-}
-
-def uniq(a):
-    if len(a) == 0:
-        return []
-    else:
-        return [a[0]] + uniq([x for x in a if x != a[0]])
-
-class NodeChecker(object):
-    def __init__(self):
-        self.vers = dict()
-        self.vers[(2,0)] = []
-    def add(self, node, ver, msg):
-        if ver not in self.vers:
-            self.vers[ver] = []
-        self.vers[ver].append((node.lineno, msg))
-    def default(self, node):
-        for child in node.getChildNodes():
-            self.visit(child)
-    def visitCallFunc(self, node):
-        def rollup(n):
-            if isinstance(n, compiler.ast.Name):
-                return n.name
-            elif isinstance(n, compiler.ast.Const):
-                return type(n.value).__name__
-            elif isinstance(n, compiler.ast.Getattr):
-                r = rollup(n.expr)
-                if r:
-                    return r + "." + n.attrname
-        name = rollup(node.node)
-        if name:
-            # Special handling for empty format strings, which aren't
-            # allowed in Python 2.6
-            if name in ('unicode.format', 'str.format'):
-                n = node.node
-                if isinstance(n, compiler.ast.Getattr):
-                    n = n.expr
-                    if isinstance(n, compiler.ast.Const):
-                        if '{}' in n.value:
-                            self.add(node, (2,7), name + ' with {} format string')
-
-            v = Functions.get(name)
-            if v is not None:
-                self.add(node, v, name)
-        self.default(node)
-    def visitClass(self, node):
-        if node.bases:
-            self.add(node, (2,2), "new-style class")
-        if node.decorators:
-            self.add(node, (2,6), "class decorator")
-        self.default(node)
-    def visitDictComp(self, node):
-        self.add(node, (2,7), "dictionary comprehension")
-        self.default(node)
-    def visitFloorDiv(self, node):
-        self.add(node, (2,2), "// operator")
-        self.default(node)
-    def visitFrom(self, node):
-        v = StandardModules.get(node.modname)
-        if v is not None:
-            self.add(node, v, node.modname)
-        for n in node.names:
-            name = node.modname + "." + n[0]
-            v = Functions.get(name)
-            if v is not None:
-                self.add(node, v, name)
-    def visitFunction(self, node):
-        if node.decorators:
-            self.add(node, (2,4), "function decorator")
-        self.default(node)
-    def visitGenExpr(self, node):
-        self.add(node, (2,4), "generator expression")
-        self.default(node)
-    def visitGetattr(self, node):
-        if (isinstance(node.expr, compiler.ast.Const)
-            and isinstance(node.expr.value, str)
-            and node.attrname == "format"):
-            self.add(node, (2,6), "string literal .format()")
-        self.default(node)
-    def visitIfExp(self, node):
-        self.add(node, (2,5), "inline if expression")
-        self.default(node)
-    def visitImport(self, node):
-        for n in node.names:
-            v = StandardModules.get(n[0])
-            if v is not None:
-                self.add(node, v, n[0])
-        self.default(node)
-    def visitName(self, node):
-        v = Identifiers.get(node.name)
-        if v is not None:
-            self.add(node, v, node.name)
-        self.default(node)
-    def visitSet(self, node):
-        self.add(node, (2,7), "set literal")
-        self.default(node)
-    def visitSetComp(self, node):
-        self.add(node, (2,7), "set comprehension")
-        self.default(node)
-    def visitTryFinally(self, node):
-        # try/finally with a suite generates a Stmt node as the body,
-        # but try/except/finally generates a TryExcept as the body
-        if isinstance(node.body, compiler.ast.TryExcept):
-            self.add(node, (2,5), "try/except/finally")
-        self.default(node)
-    def visitWith(self, node):
-        if isinstance(node.body, compiler.ast.With):
-            self.add(node, (2,7), "with statement with multiple contexts")
-        else:
-            self.add(node, (2,5), "with statement")
-        self.default(node)
-    def visitYield(self, node):
-        self.add(node, (2,2), "yield expression")
-        self.default(node)
-
-def get_versions(source, filename=None):
-    """Return information about the Python versions required for specific features.
-
-    The return value is a dictionary with keys as a version number as a tuple
-    (for example Python 2.6 is (2,6)) and the value are a list of features that
-    require the indicated Python version.
-    """
-    tree = compiler.parse(source)
-    checker = compiler.walk(tree, NodeChecker())
-    return checker.vers
-
-def v27(source):
-    if sys.version_info >= (2, 7):
-        return qver(source)
-    else:
-        print >>sys.stderr, "Not all features tested, run --test with Python 2.7"
-        return (2, 7)
-
-def qver(source):
-    """Return the minimum Python version required to run a particular bit of code.
-
-    >>> qver('print "hello world"')
-    (2, 0)
-    >>> qver('class test(object): pass')
-    (2, 2)
-    >>> qver('yield 1')
-    (2, 2)
-    >>> qver('a // b')
-    (2, 2)
-    >>> qver('True')
-    (2, 2)
-    >>> qver('enumerate(a)')
-    (2, 3)
-    >>> qver('total = sum')
-    (2, 0)
-    >>> qver('sum(a)')
-    (2, 3)
-    >>> qver('(x*x for x in range(5))')
-    (2, 4)
-    >>> qver('class C:\\n @classmethod\\n def m(): pass')
-    (2, 4)
-    >>> qver('y if x else z')
-    (2, 5)
-    >>> qver('import hashlib')
-    (2, 5)
-    >>> qver('from hashlib import md5')
-    (2, 5)
-    >>> qver('import xml.etree.ElementTree')
-    (2, 5)
-    >>> qver('try:\\n try: pass;\\n except: pass;\\nfinally: pass')
-    (2, 0)
-    >>> qver('try: pass;\\nexcept: pass;\\nfinally: pass')
-    (2, 5)
-    >>> qver('from __future__ import with_statement\\nwith x: pass')
-    (2, 5)
-    >>> qver('collections.defaultdict(list)')
-    (2, 5)
-    >>> qver('from collections import defaultdict')
-    (2, 5)
-    >>> qver('"{0}".format(0)')
-    (2, 6)
-    >>> qver('memoryview(x)')
-    (2, 7)
-    >>> v27('{1, 2, 3}')
-    (2, 7)
-    >>> v27('{x for x in s}')
-    (2, 7)
-    >>> v27('{x: y for x in s}')
-    (2, 7)
-    >>> qver('from __future__ import with_statement\\nwith x:\\n with y: pass')
-    (2, 5)
-    >>> v27('from __future__ import with_statement\\nwith x, y: pass')
-    (2, 7)
-    >>> qver('@decorator\\ndef f(): pass')
-    (2, 4)
-    >>> qver('@decorator\\nclass test:\\n pass')
-    (2, 6)
-
-    #>>> qver('0o0')
-    #(2, 6)
-    #>>> qver('@foo\\nclass C: pass')
-    #(2, 6)
-    """
-    return max(get_versions(source).keys())

lib/spack/external/pyqver3.py

@@ -1,248 +0,0 @@
-#!/usr/bin/env python3
-#
-# pyqver3.py
-# by Greg Hewgill
-# https://github.com/ghewgill/pyqver
-#
-# This software is provided 'as-is', without any express or implied
-# warranty.  In no event will the author be held liable for any damages
-# arising from the use of this software.
-#
-# Permission is granted to anyone to use this software for any purpose,
-# including commercial applications, and to alter it and redistribute it
-# freely, subject to the following restrictions:
-#
-# 1. The origin of this software must not be misrepresented; you must not
-#    claim that you wrote the original software. If you use this software
-#    in a product, an acknowledgment in the product documentation would be
-#    appreciated but is not required.
-# 2. Altered source versions must be plainly marked as such, and must not be
-#    misrepresented as being the original software.
-# 3. This notice may not be removed or altered from any source distribution.
-#
-# Copyright (c) 2009-2013 Greg Hewgill http://hewgill.com
-#
-import ast
-import platform
-import sys
-
-StandardModules = {
-# skip argparse now that it's in lib/spack/external
-#    "argparse":         (3, 2),
-    "faulthandler":     (3, 3),
-    "importlib":        (3, 1),
-    "ipaddress":        (3, 3),
-    "lzma":             (3, 3),
-    "tkinter.ttk":      (3, 1),
-    "unittest.mock":    (3, 3),
-    "venv":             (3, 3),
-}
-
-Functions = {
-    "bytearray.maketrans":                      (3, 1),
-    "bytes.maketrans":                          (3, 1),
-    "bz2.open":                                 (3, 3),
-    "collections.Counter":                      (3, 1),
-    "collections.OrderedDict":                  (3, 1),
-    "crypt.mksalt":                             (3, 3),
-    "email.generator.BytesGenerator":           (3, 2),
-    "email.message_from_binary_file":           (3, 2),
-    "email.message_from_bytes":                 (3, 2),
-    "functools.lru_cache":                      (3, 2),
-    "gzip.compress":                            (3, 2),
-    "gzip.decompress":                          (3, 2),
-    "inspect.getclosurevars":                   (3, 3),
-    "inspect.getgeneratorlocals":               (3, 3),
-    "inspect.getgeneratorstate":                (3, 2),
-    "itertools.combinations_with_replacement":  (3, 1),
-    "itertools.compress":                       (3, 1),
-    "logging.config.dictConfig":                (3, 2),
-    "logging.NullHandler":                      (3, 1),
-    "math.erf":                                 (3, 2),
-    "math.erfc":                                (3, 2),
-    "math.expm1":                               (3, 2),
-    "math.gamma":                               (3, 2),
-    "math.isfinite":                            (3, 2),
-    "math.lgamma":                              (3, 2),
-    "math.log2":                                (3, 3),
-    "os.environb":                              (3, 2),
-    "os.fsdecode":                              (3, 2),
-    "os.fsencode":                              (3, 2),
-    "os.fwalk":                                 (3, 3),
-    "os.getenvb":                               (3, 2),
-    "os.get_exec_path":                         (3, 2),
-    "os.getgrouplist":                          (3, 3),
-    "os.getpriority":                           (3, 3),
-    "os.getresgid":                             (3, 2),
-    "os.getresuid":                             (3, 2),
-    "os.get_terminal_size":                     (3, 3),
-    "os.getxattr":                              (3, 3),
-    "os.initgroups":                            (3, 2),
-    "os.listxattr":                             (3, 3),
-    "os.lockf":                                 (3, 3),
-    "os.pipe2":                                 (3, 3),
-    "os.posix_fadvise":                         (3, 3),
-    "os.posix_fallocate":                       (3, 3),
-    "os.pread":                                 (3, 3),
-    "os.pwrite":                                (3, 3),
-    "os.readv":                                 (3, 3),
-    "os.removexattr":                           (3, 3),
-    "os.replace":                               (3, 3),
-    "os.sched_get_priority_max":                (3, 3),
-    "os.sched_get_priority_min":                (3, 3),
-    "os.sched_getaffinity":                     (3, 3),
-    "os.sched_getparam":                        (3, 3),
-    "os.sched_getscheduler":                    (3, 3),
-    "os.sched_rr_get_interval":                 (3, 3),
-    "os.sched_setaffinity":                     (3, 3),
-    "os.sched_setparam":                        (3, 3),
-    "os.sched_setscheduler":                    (3, 3),
-    "os.sched_yield":                           (3, 3),
-    "os.sendfile":                              (3, 3),
-    "os.setpriority":                           (3, 3),
-    "os.setresgid":                             (3, 2),
-    "os.setresuid":                             (3, 2),
-    "os.setxattr":                              (3, 3),
-    "os.sync":                                  (3, 3),
-    "os.truncate":                              (3, 3),
-    "os.waitid":                                (3, 3),
-    "os.writev":                                (3, 3),
-    "shutil.chown":                             (3, 3),
-    "shutil.disk_usage":                        (3, 3),
-    "shutil.get_archive_formats":               (3, 3),
-    "shutil.get_terminal_size":                 (3, 3),
-    "shutil.get_unpack_formats":                (3, 3),
-    "shutil.make_archive":                      (3, 3),
-    "shutil.register_archive_format":           (3, 3),
-    "shutil.register_unpack_format":            (3, 3),
-    "shutil.unpack_archive":                    (3, 3),
-    "shutil.unregister_archive_format":         (3, 3),
-    "shutil.unregister_unpack_format":          (3, 3),
-    "shutil.which":                             (3, 3),
-    "signal.pthread_kill":                      (3, 3),
-    "signal.pthread_sigmask":                   (3, 3),
-    "signal.sigpending":                        (3, 3),
-    "signal.sigtimedwait":                      (3, 3),
-    "signal.sigwait":                           (3, 3),
-    "signal.sigwaitinfo":                       (3, 3),
-    "socket.CMSG_LEN":                          (3, 3),
-    "socket.CMSG_SPACE":                        (3, 3),
-    "socket.fromshare":                         (3, 3),
-    "socket.if_indextoname":                    (3, 3),
-    "socket.if_nameindex":                      (3, 3),
-    "socket.if_nametoindex":                    (3, 3),
-    "socket.sethostname":                       (3, 3),
-    "ssl.match_hostname":                       (3, 2),
-    "ssl.RAND_bytes":                           (3, 3),
-    "ssl.RAND_pseudo_bytes":                    (3, 3),
-    "ssl.SSLContext":                           (3, 2),
-    "ssl.SSLEOFError":                          (3, 3),
-    "ssl.SSLSyscallError":                      (3, 3),
-    "ssl.SSLWantReadError":                     (3, 3),
-    "ssl.SSLWantWriteError":                    (3, 3),
-    "ssl.SSLZeroReturnError":                   (3, 3),
-    "stat.filemode":                            (3, 3),
-    "textwrap.indent":                          (3, 3),
-    "threading.get_ident":                      (3, 3),
-    "time.clock_getres":                        (3, 3),
-    "time.clock_gettime":                       (3, 3),
-    "time.clock_settime":                       (3, 3),
-    "time.get_clock_info":                      (3, 3),
-    "time.monotonic":                           (3, 3),
-    "time.perf_counter":                        (3, 3),
-    "time.process_time":                        (3, 3),
-    "types.new_class":                          (3, 3),
-    "types.prepare_class":                      (3, 3),
-}
-
-def uniq(a):
-    if len(a) == 0:
-        return []
-    else:
-        return [a[0]] + uniq([x for x in a if x != a[0]])
-
-class NodeChecker(ast.NodeVisitor):
-    def __init__(self):
-        self.vers = dict()
-        self.vers[(3,0)] = []
-    def add(self, node, ver, msg):
-        if ver not in self.vers:
-            self.vers[ver] = []
-        self.vers[ver].append((node.lineno, msg))
-    def visit_Call(self, node):
-        def rollup(n):
-            if isinstance(n, ast.Name):
-                return n.id
-            elif isinstance(n, ast.Attribute):
-                r = rollup(n.value)
-                if r:
-                    return r + "." + n.attr
-        name = rollup(node.func)
-        if name:
-            v = Functions.get(name)
-            if v is not None:
-                self.add(node, v, name)
-        self.generic_visit(node)
-    def visit_Import(self, node):
-        for n in node.names:
-            v = StandardModules.get(n.name)
-            if v is not None:
-                self.add(node, v, n.name)
-        self.generic_visit(node)
-    def visit_ImportFrom(self, node):
-        v = StandardModules.get(node.module)
-        if v is not None:
-            self.add(node, v, node.module)
-        for n in node.names:
-            name = node.module + "." + n.name
-            v = Functions.get(name)
-            if v is not None:
-                self.add(node, v, name)
-    def visit_Raise(self, node):
-        if isinstance(node.cause, ast.Name) and node.cause.id == "None":
-            self.add(node, (3,3), "raise ... from None")
-    def visit_YieldFrom(self, node):
-        self.add(node, (3,3), "yield from")
-
-def get_versions(source, filename=None):
-    """Return information about the Python versions required for specific features.
-
-    The return value is a dictionary with keys as a version number as a tuple
-    (for example Python 3.1 is (3,1)) and the value are a list of features that
-    require the indicated Python version.
-    """
-    tree = ast.parse(source, filename=filename)
-    checker = NodeChecker()
-    checker.visit(tree)
-    return checker.vers
-
-def v33(source):
-    if sys.version_info >= (3, 3):
-        return qver(source)
-    else:
-        print("Not all features tested, run --test with Python 3.3", file=sys.stderr)
-        return (3, 3)
-
-def qver(source):
-    """Return the minimum Python version required to run a particular bit of code.
-
-    >>> qver('print("hello world")')
-    (3, 0)
-    >>> qver("import importlib")
-    (3, 1)
-    >>> qver("from importlib import x")
-    (3, 1)
-    >>> qver("import tkinter.ttk")
-    (3, 1)
-    >>> qver("from collections import Counter")
-    (3, 1)
-    >>> qver("collections.OrderedDict()")
-    (3, 1)
-    >>> qver("import functools\\n@functools.lru_cache()\\ndef f(x): x*x")
-    (3, 2)
-    >>> v33("yield from x")
-    (3, 3)
-    >>> v33("raise x from None")
-    (3, 3)
-    """
-    return max(get_versions(source).keys())

lib/spack/external/ruamel/yaml/compat.py

@@ -12,7 +12,7 @@
     from ruamel.ordereddict import ordereddict
 except:
     try:
-        from collections import OrderedDict  # nopyqver
+        from collections import OrderedDict
     except ImportError:
         from ordereddict import OrderedDict
     # to get the right name import ... as ordereddict doesn't do that

lib/spack/llnl/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/argparsewriter.py

@@ -1,136 +1,165 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 from __future__ import print_function
+
 import re
 import argparse
 import errno
 import sys
 
+from six import StringIO
 
-class ArgparseWriter(object):
+
+class Command(object):
+    """Parsed representation of a command from argparse.
+
+    This is a single command from an argparse parser. ``ArgparseWriter``
+    creates these and returns them from ``parse()``, and it passes one of
+    these to each call to ``format()`` so that we can take an action for
+    a single command.
+
+    Parts of a Command:
+      - prog: command name (str)
+      - description: command description (str)
+      - usage: command usage (str)
+      - positionals: list of positional arguments (list)
+      - optionals: list of optional arguments (list)
+      - subcommands: list of subcommand parsers (list)
+    """
+    def __init__(self, prog, description, usage,
+                 positionals, optionals, subcommands):
+        self.prog = prog
+        self.description = description
+        self.usage = usage
+        self.positionals = positionals
+        self.optionals = optionals
+        self.subcommands = subcommands
+
+
+# NOTE: The only reason we subclass argparse.HelpFormatter is to get access
+# to self._expand_help(), ArgparseWriter is not intended to be used as a
+# formatter_class.
+class ArgparseWriter(argparse.HelpFormatter):
     """Analyzes an argparse ArgumentParser for easy generation of help."""
-    def __init__(self, out=sys.stdout):
+
+    def __init__(self, prog, out=sys.stdout, aliases=False):
+        """Initializes a new ArgparseWriter instance.
+
+        Parameters:
+            prog (str): the program name
+            out (file object): the file to write to
+            aliases (bool): whether or not to include subparsers for aliases
+        """
+        super(ArgparseWriter, self).__init__(prog)
         self.level = 0
+        self.prog = prog
         self.out = out
+        self.aliases = aliases
 
-    def _write(self, parser, root=True, level=0):
+    def parse(self, parser, prog):
+        """Parses the parser object and returns the relavent components.
+
+        Parameters:
+            parser (argparse.ArgumentParser): the parser
+            prog (str): the command name
+
+        Returns:
+            (Command) information about the command from the parser
+        """
         self.parser = parser
-        self.level = level
+
+        split_prog = parser.prog.split(' ')
+        split_prog[-1] = prog
+        prog = ' '.join(split_prog)
+        description = parser.description
+
+        fmt = parser._get_formatter()
         actions = parser._actions
+        groups = parser._mutually_exclusive_groups
+        usage = fmt._format_usage(None, actions, groups, '').strip()
 
-        # allow root level to be flattened with rest of commands
-        if type(root) == int:
-            self.level = root
-            root = True
-
-        # go through actions and split them into optionals, positionals,
+        # Go through actions and split them into optionals, positionals,
         # and subcommands
         optionals = []
         positionals = []
         subcommands = []
         for action in actions:
             if action.option_strings:
-                optionals.append(action)
+                flags = action.option_strings
+                dest_flags = fmt._format_action_invocation(action)
+                help = self._expand_help(action) if action.help else ''
+                help = help.replace('\n', ' ')
+                optionals.append((flags, dest_flags, help))
             elif isinstance(action, argparse._SubParsersAction):
                 for subaction in action._choices_actions:
                     subparser = action._name_parser_map[subaction.dest]
-                    subcommands.append(subparser)
+                    subcommands.append((subparser, subaction.dest))
+
+                    # Look for aliases of the form 'name (alias, ...)'
+                    if self.aliases:
+                        match = re.match(r'(.*) \((.*)\)', subaction.metavar)
+                        if match:
+                            aliases = match.group(2).split(', ')
+                            for alias in aliases:
+                                subparser = action._name_parser_map[alias]
+                                subcommands.append((subparser, alias))
             else:
-                positionals.append(action)
+                args = fmt._format_action_invocation(action)
+                help = self._expand_help(action) if action.help else ''
+                help = help.replace('\n', ' ')
+                positionals.append((args, help))
 
-        groups = parser._mutually_exclusive_groups
-        fmt = parser._get_formatter()
-        description = parser.description
+        return Command(
+            prog, description, usage, positionals, optionals, subcommands)
 
-        def action_group(function, actions):
-            for action in actions:
-                arg = fmt._format_action_invocation(action)
-                help = action.help if action.help else ''
-                function(arg, re.sub('\n', ' ', help))
+    def format(self, cmd):
+        """Returns the string representation of a single node in the
+        parser tree.
 
-        if root:
-            self.begin_command(parser.prog)
+        Override this in subclasses to define how each subcommand
+        should be displayed.
 
-            if description:
-                self.description(parser.description)
+        Parameters:
+            (Command): parsed information about a command or subcommand
 
-            usage = fmt._format_usage(None, actions, groups, '').strip()
-            self.usage(usage)
+        Returns:
+            str: the string representation of this subcommand
+        """
+        raise NotImplementedError
 
-            if positionals:
-                self.begin_positionals()
-                action_group(self.positional, positionals)
-                self.end_positionals()
+    def _write(self, parser, prog, level=0):
+        """Recursively writes a parser.
 
-            if optionals:
-                self.begin_optionals()
-                action_group(self.optional, optionals)
-                self.end_optionals()
+        Parameters:
+            parser (argparse.ArgumentParser): the parser
+            prog (str): the command name
+            level (int): the current level
+        """
+        self.level = level
 
-        if subcommands:
-            self.begin_subcommands(subcommands)
-            for subparser in subcommands:
-                self._write(subparser, root=True, level=level + 1)
-            self.end_subcommands(subcommands)
+        cmd = self.parse(parser, prog)
+        self.out.write(self.format(cmd))
 
-        if root:
-            self.end_command(parser.prog)
+        for subparser, prog in cmd.subcommands:
+            self._write(subparser, prog, level=level + 1)
 
-    def write(self, parser, root=True):
+    def write(self, parser):
         """Write out details about an ArgumentParser.
 
         Args:
-            parser (ArgumentParser): an ``argparse`` parser
-            root (bool or int): if bool, whether to include the root parser;
-                or ``1`` to flatten the root parser with first-level
-                subcommands
+            parser (argparse.ArgumentParser): the parser
         """
         try:
-            self._write(parser, root, level=0)
+            self._write(parser, self.prog)
         except IOError as e:
-            # swallow pipe errors
+            # Swallow pipe errors
+            # Raises IOError in Python 2 and BrokenPipeError in Python 3
             if e.errno != errno.EPIPE:
                 raise
 
-    def begin_command(self, prog):
-        pass
-
-    def end_command(self, prog):
-        pass
-
-    def description(self, description):
-        pass
-
-    def usage(self, usage):
-        pass
-
-    def begin_positionals(self):
-        pass
-
-    def positional(self, name, help):
-        pass
-
-    def end_positionals(self):
-        pass
-
-    def begin_optionals(self):
-        pass
-
-    def optional(self, option, help):
-        pass
-
-    def end_optionals(self):
-        pass
-
-    def begin_subcommands(self, subcommands):
-        pass
-
-    def end_subcommands(self, subcommands):
-        pass
-
 
 _rst_levels = ['=', '-', '^', '~', ':', '`']
 
@@ -138,66 +167,213 @@ def end_subcommands(self, subcommands):
 class ArgparseRstWriter(ArgparseWriter):
     """Write argparse output as rst sections."""
 
-    def __init__(self, out=sys.stdout, rst_levels=_rst_levels,
-                 strip_root_prog=True):
+    def __init__(self, prog, out=sys.stdout, aliases=False,
+                 rst_levels=_rst_levels):
         """Create a new ArgparseRstWriter.
 
-        Args:
+        Parameters:
+            prog (str): program name
             out (file object): file to write to
+            aliases (bool): whether or not to include subparsers for aliases
             rst_levels (list of str): list of characters
                 for rst section headings
-            strip_root_prog (bool): if ``True``, strip the base command name
-                from subcommands in output
         """
-        super(ArgparseRstWriter, self).__init__(out)
+        super(ArgparseRstWriter, self).__init__(prog, out, aliases)
         self.rst_levels = rst_levels
-        self.strip_root_prog = strip_root_prog
 
-    def line(self, string=''):
-        self.out.write('%s\n' % string)
+    def format(self, cmd):
+        string = StringIO()
+        string.write(self.begin_command(cmd.prog))
+
+        if cmd.description:
+            string.write(self.description(cmd.description))
+
+        string.write(self.usage(cmd.usage))
+
+        if cmd.positionals:
+            string.write(self.begin_positionals())
+            for args, help in cmd.positionals:
+                string.write(self.positional(args, help))
+            string.write(self.end_positionals())
+
+        if cmd.optionals:
+            string.write(self.begin_optionals())
+            for flags, dest_flags, help in cmd.optionals:
+                string.write(self.optional(dest_flags, help))
+            string.write(self.end_optionals())
+
+        if cmd.subcommands:
+            string.write(self.begin_subcommands(cmd.subcommands))
+
+        return string.getvalue()
 
     def begin_command(self, prog):
-        self.line()
-        self.line('----')
-        self.line()
-        self.line('.. _%s:\n' % prog.replace(' ', '-'))
-        self.line('%s' % prog)
-        self.line(self.rst_levels[self.level] * len(prog) + '\n')
+        return """
+----
+
+.. _{0}:
+
+{1}
+{2}
+
+""".format(prog.replace(' ', '-'), prog,
+           self.rst_levels[self.level] * len(prog))
 
     def description(self, description):
-        self.line('%s\n' % description)
+        return description + '\n\n'
 
     def usage(self, usage):
-        self.line('.. code-block:: console\n')
-        self.line('    %s\n' % usage)
+        return """\
+.. code-block:: console
+
+    {0}
+
+""".format(usage)
 
     def begin_positionals(self):
-        self.line()
-        self.line('**Positional arguments**\n')
+        return '\n**Positional arguments**\n\n'
 
     def positional(self, name, help):
-        self.line(name)
-        self.line('  %s\n' % help)
+        return """\
+{0}
+  {1}
+
+""".format(name, help)
+
+    def end_positionals(self):
+        return ''
 
     def begin_optionals(self):
-        self.line()
-        self.line('**Optional arguments**\n')
+        return '\n**Optional arguments**\n\n'
 
     def optional(self, opts, help):
-        self.line('``%s``' % opts)
-        self.line('  %s\n' % help)
+        return """\
+``{0}``
+  {1}
+
+""".format(opts, help)
+
+    def end_optionals(self):
+        return ''
 
     def begin_subcommands(self, subcommands):
-        self.line()
-        self.line('**Subcommands**\n')
-        self.line('.. hlist::')
-        self.line('   :columns: 4\n')
+        string = """
+**Subcommands**
 
-        for cmd in subcommands:
-            prog = cmd.prog
-            if self.strip_root_prog:
-                prog = re.sub(r'^[^ ]* ', '', prog)
+.. hlist::
+   :columns: 4
 
-            self.line('   * :ref:`%s <%s>`'
-                      % (prog, cmd.prog.replace(' ', '-')))
-        self.line()
+"""
+
+        for cmd, _ in subcommands:
+            prog = re.sub(r'^[^ ]* ', '', cmd.prog)
+            string += '   * :ref:`{0} <{1}>`\n'.format(
+                prog, cmd.prog.replace(' ', '-'))
+
+        return string + '\n'
+
+
+class ArgparseCompletionWriter(ArgparseWriter):
+    """Write argparse output as shell programmable tab completion functions."""
+
+    def format(self, cmd):
+        """Returns the string representation of a single node in the
+        parser tree.
+
+        Override this in subclasses to define how each subcommand
+        should be displayed.
+
+        Parameters:
+            (Command): parsed information about a command or subcommand
+
+        Returns:
+            str: the string representation of this subcommand
+        """
+
+        assert cmd.optionals  # we should always at least have -h, --help
+        assert not (cmd.positionals and cmd.subcommands)  # one or the other
+
+        # We only care about the arguments/flags, not the help messages
+        positionals = []
+        if cmd.positionals:
+            positionals, _ = zip(*cmd.positionals)
+        optionals, _, _ = zip(*cmd.optionals)
+        subcommands = []
+        if cmd.subcommands:
+            _, subcommands = zip(*cmd.subcommands)
+
+        # Flatten lists of lists
+        optionals = [x for xx in optionals for x in xx]
+
+        return (self.start_function(cmd.prog) +
+                self.body(positionals, optionals, subcommands) +
+                self.end_function(cmd.prog))
+
+    def start_function(self, prog):
+        """Returns the syntax needed to begin a function definition.
+
+        Parameters:
+            prog (str): the command name
+
+        Returns:
+            str: the function definition beginning
+        """
+        name = prog.replace('-', '_').replace(' ', '_')
+        return '\n_{0}() {{'.format(name)
+
+    def end_function(self, prog=None):
+        """Returns the syntax needed to end a function definition.
+
+        Parameters:
+            prog (str, optional): the command name
+
+        Returns:
+            str: the function definition ending
+        """
+        return '}\n'
+
+    def body(self, positionals, optionals, subcommands):
+        """Returns the body of the function.
+
+        Parameters:
+            positionals (list): list of positional arguments
+            optionals (list): list of optional arguments
+            subcommands (list): list of subcommand parsers
+
+        Returns:
+            str: the function body
+        """
+        return ''
+
+    def positionals(self, positionals):
+        """Returns the syntax for reporting positional arguments.
+
+        Parameters:
+            positionals (list): list of positional arguments
+
+        Returns:
+            str: the syntax for positional arguments
+        """
+        return ''
+
+    def optionals(self, optionals):
+        """Returns the syntax for reporting optional flags.
+
+        Parameters:
+            optionals (list): list of optional arguments
+
+        Returns:
+            str: the syntax for optional flags
+        """
+        return ''
+
+    def subcommands(self, subcommands):
+        """Returns the syntax for reporting subcommands.
+
+        Parameters:
+            subcommands (list): list of subcommand parsers
+
+        Returns:
+            str: the syntax for subcommand parsers
+        """
+        return ''

lib/spack/llnl/util/cpu/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/cpu/alias.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/cpu/detect.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -13,6 +13,7 @@
 import six
 
 from .microarchitecture import generic_microarchitecture, targets
+from .schema import targets_json
 
 #: Mapping from operating systems to chain of commands
 #: to obtain a dictionary of raw info on the current cpu
@@ -108,23 +109,39 @@ def sysctl(*args):
         'model': sysctl('-n', 'machdep.cpu.model'),
         'model name': sysctl('-n', 'machdep.cpu.brand_string')
     }
-
-    # Super hacky way to deal with slight representation differences
-    # Would be better to somehow consider these "identical"
-    if 'sse4.1' in info['flags']:
-        info['flags'] += ' sse4_1'
-    if 'sse4.2' in info['flags']:
-        info['flags'] += ' sse4_2'
-    if 'avx1.0' in info['flags']:
-        info['flags'] += ' avx'
-    if 'clfsopt' in info['flags']:
-        info['flags'] += ' clflushopt'
-    if 'xsave' in info['flags']:
-        info['flags'] += ' xsavec xsaveopt'
-
     return info
 
 
+def adjust_raw_flags(info):
+    """Adjust the flags detected on the system to homogenize
+    slightly different representations.
+    """
+    # Flags detected on Darwin turned to their linux counterpart
+    flags = info.get('flags', [])
+    d2l = targets_json['conversions']['darwin_flags']
+    for darwin_flag, linux_flag in d2l.items():
+        if darwin_flag in flags:
+            info['flags'] += ' ' + linux_flag
+
+
+def adjust_raw_vendor(info):
+    """Adjust the vendor field to make it human readable"""
+    if 'CPU implementer' not in info:
+        return
+
+    # Mapping numeric codes to vendor (ARM). This list is a merge from
+    # different sources:
+    #
+    # https://github.com/karelzak/util-linux/blob/master/sys-utils/lscpu-arm.c
+    # https://developer.arm.com/docs/ddi0487/latest/arm-architecture-reference-manual-armv8-for-armv8-a-architecture-profile
+    # https://github.com/gcc-mirror/gcc/blob/master/gcc/config/aarch64/aarch64-cores.def
+    # https://patchwork.kernel.org/patch/10524949/
+    arm_vendors = targets_json['conversions']['arm_vendors']
+    arm_code = info['CPU implementer']
+    if arm_code in arm_vendors:
+        info['CPU implementer'] = arm_vendors[arm_code]
+
+
 def raw_info_dictionary():
     """Returns a dictionary with information on the cpu of the current host.
 
@@ -139,6 +156,8 @@ def raw_info_dictionary():
             warnings.warn(str(e))
 
         if info:
+            adjust_raw_flags(info)
+            adjust_raw_vendor(info)
             break
 
     return info
@@ -223,3 +242,15 @@ def compatibility_check_for_x86_64(info, target):
     return (target == arch_root or arch_root in target.ancestors) \
         and (target.vendor == vendor or target.vendor == 'generic') \
         and target.features.issubset(features)
+
+
+@compatibility_check(architecture_family='aarch64')
+def compatibility_check_for_aarch64(info, target):
+    basename = 'aarch64'
+    features = set(info.get('Features', '').split())
+    vendor = info.get('CPU implementer', 'generic')
+
+    arch_root = targets[basename]
+    return (target == arch_root or arch_root in target.ancestors) \
+        and (target.vendor == vendor or target.vendor == 'generic') \
+        and target.features.issubset(features)

lib/spack/llnl/util/cpu/microarchitecture.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -8,7 +8,7 @@
 import warnings
 
 try:
-    from collections.abc import Sequence
+    from collections.abc import Sequence  # novm
 except ImportError:
     from collections import Sequence
 

lib/spack/llnl/util/cpu/microarchitectures.json

@@ -64,13 +64,13 @@
         "clang": [
           {
             "versions": "0.0.0-apple:",
-            "family": "x86-64",
-            "flags": "-march={family}"
+            "name": "x86-64",
+            "flags": "-march={name}"
           },
           {
             "versions": ":",
-            "family": "x86-64",
-            "flags": "-march={family} -mcpu=generic"
+            "name": "x86-64",
+            "flags": "-march={name} -mtune=generic"
           }
         ],
         "intel": {
@@ -96,8 +96,7 @@
         },
         "clang": {
           "versions": "3.9:",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "16.0:",
@@ -122,8 +121,7 @@
         },
         "clang": {
           "versions": "3.9:",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "16.0:",
@@ -157,8 +155,7 @@
         ],
         "clang": {
           "versions": "3.9:",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "16.0:",
@@ -188,8 +185,7 @@
         },
         "clang": {
           "versions": "3.9:",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "16.0:",
@@ -227,8 +223,7 @@
         ],
         "clang": {
           "versions": "3.9:",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": [
           {
@@ -274,8 +269,7 @@
         ],
         "clang": {
           "versions": "3.9:",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": [
           {
@@ -326,8 +320,7 @@
         ],
         "clang": {
           "versions": "3.9:",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": [
           {
@@ -373,8 +366,7 @@
         },
         "clang": {
           "versions": "3.9:",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "18.0:",
@@ -416,8 +408,7 @@
         },
         "clang": {
           "versions": "3.9:",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "18.0:",
@@ -463,8 +454,7 @@
         "clang": {
           "versions": "3.9:",
           "name": "knl",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "18.0:",
@@ -515,8 +505,7 @@
         "clang": {
           "versions": "3.9:",
           "name": "skylake-avx512",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "18.0:",
@@ -568,8 +557,7 @@
         },
         "clang": {
           "versions": "3.9:",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "18.0:",
@@ -609,7 +597,7 @@
         "avx512bw",
         "avx512dq",
         "avx512cd",
-        "avx512vnni"
+        "avx512_vnni"
       ],
       "compilers": {
         "gcc": {
@@ -618,8 +606,7 @@
         },
         "clang": {
           "versions": "8.0:",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "19.0:",
@@ -685,13 +672,11 @@
           {
             "versions": "7.0:",
             "name": "icelake-client",
-            "family": "x86-64",
-            "flags": "-march={family} -mcpu={name}"
+            "flags": "-march={name} -mtune={name}"
           },
           {
             "versions": "6.0:6.9",
-            "family": "x86-64",
-            "flags": "-march={family} -mcpu={name}"
+            "flags": "-march={name} -mtune={name}"
           }
         ],
         "intel": {
@@ -723,8 +708,7 @@
         "clang": {
           "versions": "3.9:",
           "name": "amdfam10",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "16.0:",
@@ -761,8 +745,7 @@
         "clang": {
           "versions": "3.9:",
           "name": "bdver1",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "16.0:",
@@ -803,8 +786,7 @@
         "clang": {
           "versions": "3.9:",
           "name": "bdver2",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "16.0:",
@@ -846,8 +828,7 @@
         "clang": {
           "versions": "3.9:",
           "name": "bdver3",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "16.0:",
@@ -892,8 +873,7 @@
         "clang": {
           "versions": "3.9:",
           "name": "bdver4",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "16.0:",
@@ -942,8 +922,7 @@
         "clang": {
           "versions": "4.0:",
           "name": "znver1",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "16.0:",
@@ -993,8 +972,7 @@
         "clang": {
           "versions": "9.0:",
           "name": "znver2",
-          "family": "x86-64",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-march={name} -mtune={name}"
         },
         "intel": {
           "versions": "16.0:",
@@ -1016,8 +994,7 @@
         },
         "clang": {
           "versions": ":",
-          "family": "ppc64",
-          "flags": "-march={family} -mcpu=generic"
+          "flags": "-mcpu={name} -mtune={name}"
         }
       }
     },
@@ -1033,9 +1010,7 @@
         },
         "clang": {
           "versions": "3.9:",
-          "family": "ppc64",
-          "name": "pwr7",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-mcpu={name} -mtune={name}"
         }
       }
     },
@@ -1058,9 +1033,7 @@
         ],
         "clang": {
           "versions": "3.9:",
-          "family": "ppc64",
-          "name": "pwr8",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-mcpu={name} -mtune={name}"
         }
       }
     },
@@ -1076,9 +1049,7 @@
         },
         "clang": {
           "versions": "3.9:",
-          "family": "ppc64",
-          "name": "pwr9",
-          "flags": "-march={family} -mcpu={name}"
+          "flags": "-mcpu={name} -mtune={name}"
         }
       }
     },
@@ -1094,8 +1065,7 @@
         },
         "clang": {
           "versions": ":",
-          "family": "ppc64le",
-          "flags": "-march={family} -mcpu=generic"
+          "flags": "-mcpu={name} -mtune={name}"
         }
       }
     },
@@ -1121,8 +1091,8 @@
         "clang": {
           "versions": "3.9:",
           "family": "ppc64le",
-          "name": "pwr8",
-          "flags": "-march={family} -mcpu={name}"
+          "name": "power8",
+          "flags": "-mcpu={name} -mtune={name}"
         }
       }
     },
@@ -1140,8 +1110,8 @@
         "clang": {
           "versions": "3.9:",
           "family": "ppc64le",
-          "name": "pwr9",
-          "flags": "-march={family} -mcpu={name}"
+          "name": "power9",
+          "flags": "-mcpu={name} -mtune={name}"
         }
       }
     },
@@ -1156,11 +1126,113 @@
         },
         "clang": {
           "versions": ":",
-          "family": "aarch64",
-          "flags": "-march={family} -mcpu=generic"
+          "flags": "-march=armv8-a -mtune=generic"
         }
       }
     },
+    "thunderx2": {
+      "from": "aarch64",
+      "vendor": "Cavium",
+      "features": [
+        "fp",
+        "asimd",
+        "evtstrm",
+        "aes",
+        "pmull",
+        "sha1",
+        "sha2",
+        "crc32",
+        "atomics",
+        "cpuid",
+        "asimdrdm"
+      ],
+      "compilers": {
+        "gcc": [
+          {
+            "versions": "4.8:4.8.9",
+            "flags": "-march=armv8-a"
+          },
+          {
+            "versions": "4.9:5.9",
+            "flags": "-march=armv8-a+crc+crypto"
+          },
+          {
+            "versions": "6:6.9",
+            "flags": "-march=armv8.1-a+crc+crypto"
+          },
+          {
+            "versions": "7:",
+            "flags": "-mcpu=thunderx2t99"
+          }
+        ],
+        "clang": [
+          {
+            "versions": "3.9:4.9",
+            "flags": "-march=armv8.1-a+crc+crypto"
+          },
+          {
+            "versions": "5:",
+            "flags": "-mcpu=thunderx2t99"
+          }
+        ]
+      }
+    },
+    "a64fx": {
+      "from": "aarch64",
+      "vendor": "Fujitsu",
+      "features": [
+        "fp",
+        "asimd",
+        "evtstrm",
+        "aes",
+        "pmull",
+        "sha1",
+        "sha2",
+        "crc32",
+        "atomics",
+        "cpuid",
+        "asimdrdm",
+        "fphp",
+        "asimdhp",
+        "fcma",
+        "dcpop",
+        "sve"
+      ],
+      "compilers": {
+        "gcc": [
+          {
+            "versions": "4.8:4.8.9",
+            "flags": "-march=armv8-a"
+          },
+          {
+            "versions": "4.9:5.9",
+            "flags": "-march=armv8-a+crc+crypto"
+          },
+          {
+            "versions": "6:6.9",
+            "flags": "-march=armv8.1-a+crc+crypto"
+          },
+          {
+            "versions": "7:7.9",
+            "flags": "-march=armv8.2-a+crc+crypto+fp16"
+          },
+          {
+            "versions": "8:",
+            "flags": "-march=armv8.2-a+crc+aes+sha2+fp16+sve -msve-vector-bits=512"
+          }
+        ],
+        "clang": [
+          {
+            "versions": "3.9:4.9",
+            "flags": "-march=armv8.2-a+crc+crypto+fp16"
+          },
+          {
+            "versions": "5:",
+            "flags": "-march=armv8.2-a+crc+crypto+fp16+sve"
+          }
+        ]
+      }
+    },
     "arm": {
       "from": null,
       "vendor": "generic",
@@ -1226,6 +1298,20 @@
         "ppc64"
       ]
     },
+    "vsx": {
+      "reason": "VSX alitvec extensions are supported by PowerISA from v2.06 (Power7+), but might not be listed in features",
+      "families": [
+        "ppc64le",
+        "ppc64"
+      ]
+    },
+    "fma": {
+      "reason": "FMA has been supported by PowerISA since Power1, but might not be listed in features",
+      "families": [
+        "ppc64le",
+        "ppc64"
+      ]
+    },
     "sse4.1": {
       "reason": "permits to refer to sse4_1 also as sse4.1",
       "any_of": [
@@ -1244,5 +1330,34 @@
         "aarch64"
       ]
     }
+  },
+  "conversions": {
+    "description": "Conversions that map some platform specific values to canonical values",
+    "arm_vendors": {
+      "0x41": "ARM",
+      "0x42": "Broadcom",
+      "0x43": "Cavium",
+      "0x44": "DEC",
+      "0x46": "Fujitsu",
+      "0x48": "HiSilicon",
+      "0x49": "Infineon Technologies AG",
+      "0x4d": "Motorola",
+      "0x4e": "Nvidia",
+      "0x50": "APM",
+      "0x51": "Qualcomm",
+      "0x53": "Samsung",
+      "0x56": "Marvell",
+      "0x61": "Apple",
+      "0x66": "Faraday",
+      "0x68": "HXT",
+      "0x69": "Intel"
+    },
+    "darwin_flags": {
+      "sse4.1": "sse4_1",
+      "sse4.2": "sse4_2",
+      "avx1.0": "avx",
+      "clfsopt": "clflushopt",
+      "xsave": "xsavec xsaveopt"
+    }
   }
 }

lib/spack/llnl/util/cpu/schema.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -6,7 +6,7 @@
 import os.path
 
 try:
-    from collections.abc import MutableMapping
+    from collections.abc import MutableMapping  # novm
 except ImportError:
     from collections import MutableMapping
 
@@ -72,7 +72,21 @@
                 'additionalProperties': False
             }
         },
-
+    },
+    'conversions': {
+        'type': 'object',
+        'properties': {
+            'description': {
+                'type': 'string'
+            },
+            'arm_vendors': {
+                'type': 'object',
+            },
+            'darwin_flags': {
+                'type': 'object'
+            }
+        },
+        'additionalProperties': False
     }
 }
 

lib/spack/llnl/util/filesystem.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -201,7 +201,6 @@ def groupid_to_group(x):
                         output_file.writelines(input_file.readlines())
 
         except BaseException:
-            os.remove(tmp_filename)
             # clean up the original file on failure.
             shutil.move(backup_filename, filename)
             raise
@@ -457,7 +456,7 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
             if os.path.isdir(s):
                 mkdirp(d)
             else:
-                shutil.copyfile(s, d)
+                shutil.copy2(s, d)
 
         if _permissions:
             set_install_permissions(d)
@@ -917,10 +916,8 @@ def remove_if_dead_link(path):
     Parameters:
         path (str): The potential dead link
     """
-    if os.path.islink(path):
-        real_path = os.path.realpath(path)
-        if not os.path.exists(real_path):
-            os.unlink(path)
+    if os.path.islink(path) and not os.path.exists(path):
+        os.unlink(path)
 
 
 def remove_linked_tree(path):
@@ -1156,7 +1153,9 @@ class HeaderList(FileList):
 
     # Make sure to only match complete words, otherwise path components such
     # as "xinclude" will cause false matches.
-    include_regex = re.compile(r'(.*)(\binclude\b)(.*)')
+    # Avoid matching paths such as <prefix>/include/something/detail/include,
+    # e.g. in the CUDA Toolkit which ships internal libc++ headers.
+    include_regex = re.compile(r'(.*?)(\binclude\b)(.*)')
 
     def __init__(self, files):
         super(HeaderList, self).__init__(files)

lib/spack/llnl/util/lang.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -19,12 +19,6 @@
 ignore_modules = [r'^\.#', '~$']
 
 
-class classproperty(property):
-    """classproperty decorator: like property but for classmethods."""
-    def __get__(self, cls, owner):
-        return self.fget.__get__(None, owner)()
-
-
 def index_by(objects, *funcs):
     """Create a hierarchy of dictionaries by splitting the supplied
        set of objects on unique values of the supplied functions.
@@ -612,12 +606,14 @@ def load_module_from_file(module_name, module_path):
     """
     if sys.version_info[0] == 3 and sys.version_info[1] >= 5:
         import importlib.util
-        spec = importlib.util.spec_from_file_location(module_name, module_path)
-        module = importlib.util.module_from_spec(spec)
+        spec = importlib.util.spec_from_file_location(  # novm
+            module_name, module_path)
+        module = importlib.util.module_from_spec(spec)  # novm
         spec.loader.exec_module(module)
     elif sys.version_info[0] == 3 and sys.version_info[1] < 5:
         import importlib.machinery
-        loader = importlib.machinery.SourceFileLoader(module_name, module_path)
+        loader = importlib.machinery.SourceFileLoader(  # novm
+            module_name, module_path)
         module = loader.load_module()
     elif sys.version_info[0] == 2:
         import imp

lib/spack/llnl/util/link_tree.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/lock.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -8,14 +8,32 @@
 import errno
 import time
 import socket
+from datetime import datetime
 
 import llnl.util.tty as tty
+import spack.util.string
 
 
 __all__ = ['Lock', 'LockTransaction', 'WriteTransaction', 'ReadTransaction',
            'LockError', 'LockTimeoutError',
            'LockPermissionError', 'LockROFileError', 'CantCreateLockError']
 
+#: Mapping of supported locks to description
+lock_type = {fcntl.LOCK_SH: 'read', fcntl.LOCK_EX: 'write'}
+
+#: A useful replacement for functions that should return True when not provided
+#: for example.
+true_fn = lambda: True
+
+
+def _attempts_str(wait_time, nattempts):
+    # Don't print anything if we succeeded on the first try
+    if nattempts <= 1:
+        return ''
+
+    attempts = spack.util.string.plural(nattempts, 'attempt')
+    return ' after {0:0.2f}s and {1}'.format(wait_time, attempts)
+
 
 class Lock(object):
     """This is an implementation of a filesystem lock using Python's lockf.
@@ -31,8 +49,8 @@ class Lock(object):
     maintain multiple locks on the same file.
     """
 
-    def __init__(self, path, start=0, length=0, debug=False,
-                 default_timeout=None):
+    def __init__(self, path, start=0, length=0, default_timeout=None,
+                 debug=False, desc=''):
         """Construct a new lock on the file at ``path``.
 
         By default, the lock applies to the whole file.  Optionally,
@@ -43,6 +61,16 @@ def __init__(self, path, start=0, length=0, debug=False,
         not currently expose the ``whence`` parameter -- ``whence`` is
         always ``os.SEEK_SET`` and ``start`` is always evaluated from the
         beginning of the file.
+
+        Args:
+            path (str): path to the lock
+            start (int): optional byte offset at which the lock starts
+            length (int): optional number of bytes to lock
+            default_timeout (int): number of seconds to wait for lock attempts,
+                where None means to wait indefinitely
+            debug (bool): debug mode specific to locking
+            desc (str): optional debug message lock description, which is
+                helpful for distinguishing between different Spack locks.
         """
         self.path = path
         self._file = None
@@ -56,6 +84,9 @@ def __init__(self, path, start=0, length=0, debug=False,
         # enable debug mode
         self.debug = debug
 
+        # optional debug description
+        self.desc = ' ({0})'.format(desc) if desc else ''
+
         # If the user doesn't set a default timeout, or if they choose
         # None, 0, etc. then lock attempts will not time out (unless the
         # user sets a timeout for each attempt)
@@ -89,22 +120,33 @@ def _poll_interval_generator(_wait_times=None):
             num_requests += 1
             yield wait_time
 
+    def __repr__(self):
+        """Formal representation of the lock."""
+        rep = '{0}('.format(self.__class__.__name__)
+        for attr, value in self.__dict__.items():
+            rep += '{0}={1}, '.format(attr, value.__repr__())
+        return '{0})'.format(rep.strip(', '))
+
+    def __str__(self):
+        """Readable string (with key fields) of the lock."""
+        location = '{0}[{1}:{2}]'.format(self.path, self._start, self._length)
+        timeout = 'timeout={0}'.format(self.default_timeout)
+        activity = '#reads={0}, #writes={1}'.format(self._reads, self._writes)
+        return '({0}, {1}, {2})'.format(location, timeout, activity)
+
     def _lock(self, op, timeout=None):
         """This takes a lock using POSIX locks (``fcntl.lockf``).
 
         The lock is implemented as a spin lock using a nonblocking call
         to ``lockf()``.
 
-        On acquiring an exclusive lock, the lock writes this process's
-        pid and host to the lock file, in case the holding process needs
-        to be killed later.
-
         If the lock times out, it raises a ``LockError``. If the lock is
         successfully acquired, the total wait time and the number of attempts
         is returned.
         """
-        assert op in (fcntl.LOCK_SH, fcntl.LOCK_EX)
+        assert op in lock_type
 
+        self._log_acquiring('{0} LOCK'.format(lock_type[op].upper()))
         timeout = timeout or self.default_timeout
 
         # Create file and parent directories if they don't exist.
@@ -132,6 +174,9 @@ def _lock(self, op, timeout=None):
             # If the file were writable, we'd have opened it 'r+'
             raise LockROFileError(self.path)
 
+        tty.debug("{0} locking [{1}:{2}]: timeout {3} sec"
+                  .format(lock_type[op], self._start, self._length, timeout))
+
         poll_intervals = iter(Lock._poll_interval_generator())
         start_time = time.time()
         num_attempts = 0
@@ -143,17 +188,21 @@ def _lock(self, op, timeout=None):
 
             time.sleep(next(poll_intervals))
 
+        # TBD: Is an extra attempt after timeout needed/appropriate?
         num_attempts += 1
         if self._poll_lock(op):
             total_wait_time = time.time() - start_time
             return total_wait_time, num_attempts
 
-        raise LockTimeoutError("Timed out waiting for lock.")
+        raise LockTimeoutError("Timed out waiting for a {0} lock."
+                               .format(lock_type[op]))
 
     def _poll_lock(self, op):
         """Attempt to acquire the lock in a non-blocking manner. Return whether
         the locking attempt succeeds
         """
+        assert op in lock_type
+
         try:
             # Try to get the lock (will raise if not available.)
             fcntl.lockf(self._file, op | fcntl.LOCK_NB,
@@ -163,6 +212,9 @@ def _poll_lock(self, op):
             if self.debug:
                 # All locks read the owner PID and host
                 self._read_debug_data()
+                tty.debug('{0} locked {1} [{2}:{3}] (owner={4})'
+                          .format(lock_type[op], self.path,
+                                  self._start, self._length, self.pid))
 
                 # Exclusive locks write their PID/host
                 if op == fcntl.LOCK_EX:
@@ -171,12 +223,12 @@ def _poll_lock(self, op):
             return True
 
         except IOError as e:
-            if e.errno in (errno.EAGAIN, errno.EACCES):
-                # EAGAIN and EACCES == locked by another process
-                pass
-            else:
+            # EAGAIN and EACCES == locked by another process (so try again)
+            if e.errno not in (errno.EAGAIN, errno.EACCES):
                 raise
 
+        return False
+
     def _ensure_parent_directory(self):
         parent = os.path.dirname(self.path)
 
@@ -231,6 +283,8 @@ def _unlock(self):
                     self._length, self._start, os.SEEK_SET)
         self._file.close()
         self._file = None
+        self._reads = 0
+        self._writes = 0
 
     def acquire_read(self, timeout=None):
         """Acquires a recursive, shared lock for reading.
@@ -246,15 +300,14 @@ def acquire_read(self, timeout=None):
         timeout = timeout or self.default_timeout
 
         if self._reads == 0 and self._writes == 0:
-            self._debug(
-                'READ LOCK: {0.path}[{0._start}:{0._length}] [Acquiring]'
-                .format(self))
             # can raise LockError.
             wait_time, nattempts = self._lock(fcntl.LOCK_SH, timeout=timeout)
-            self._acquired_debug('READ LOCK', wait_time, nattempts)
             self._reads += 1
+            # Log if acquired, which includes counts when verbose
+            self._log_acquired('READ LOCK', wait_time, nattempts)
             return True
         else:
+            # Increment the read count for nested lock tracking
             self._reads += 1
             return False
 
@@ -272,23 +325,90 @@ def acquire_write(self, timeout=None):
         timeout = timeout or self.default_timeout
 
         if self._writes == 0:
-            self._debug(
-                'WRITE LOCK: {0.path}[{0._start}:{0._length}] [Acquiring]'
-                .format(self))
             # can raise LockError.
             wait_time, nattempts = self._lock(fcntl.LOCK_EX, timeout=timeout)
-            self._acquired_debug('WRITE LOCK', wait_time, nattempts)
             self._writes += 1
-            return True
+            # Log if acquired, which includes counts when verbose
+            self._log_acquired('WRITE LOCK', wait_time, nattempts)
+
+            # return True only if we weren't nested in a read lock.
+            # TODO: we may need to return two values: whether we got
+            # the write lock, and whether this is acquiring a read OR
+            # write lock for the first time. Now it returns the latter.
+            return self._reads == 0
         else:
+            # Increment the write count for nested lock tracking
             self._writes += 1
             return False
 
-    def release_read(self):
+    def is_write_locked(self):
+        """Check if the file is write locked
+
+        Return:
+            (bool): ``True`` if the path is write locked, otherwise, ``False``
+        """
+        try:
+            self.acquire_read()
+
+            # If we have a read lock then no other process has a write lock.
+            self.release_read()
+        except LockTimeoutError:
+            # Another process is holding a write lock on the file
+            return True
+
+        return False
+
+    def downgrade_write_to_read(self, timeout=None):
+        """
+        Downgrade from an exclusive write lock to a shared read.
+
+        Raises:
+            LockDowngradeError: if this is an attempt at a nested transaction
+        """
+        timeout = timeout or self.default_timeout
+
+        if self._writes == 1 and self._reads == 0:
+            self._log_downgrading()
+            # can raise LockError.
+            wait_time, nattempts = self._lock(fcntl.LOCK_SH, timeout=timeout)
+            self._reads = 1
+            self._writes = 0
+            self._log_downgraded(wait_time, nattempts)
+        else:
+            raise LockDowngradeError(self.path)
+
+    def upgrade_read_to_write(self, timeout=None):
+        """
+        Attempts to upgrade from a shared read lock to an exclusive write.
+
+        Raises:
+            LockUpgradeError: if this is an attempt at a nested transaction
+        """
+        timeout = timeout or self.default_timeout
+
+        if self._reads == 1 and self._writes == 0:
+            self._log_upgrading()
+            # can raise LockError.
+            wait_time, nattempts = self._lock(fcntl.LOCK_EX, timeout=timeout)
+            self._reads = 0
+            self._writes = 1
+            self._log_upgraded(wait_time, nattempts)
+        else:
+            raise LockUpgradeError(self.path)
+
+    def release_read(self, release_fn=None):
         """Releases a read lock.
 
-        Returns True if the last recursive lock was released, False if
-        there are still outstanding locks.
+        Arguments:
+            release_fn (callable): function to call *before* the last recursive
+                lock (read or write) is released.
+
+        If the last recursive lock will be released, then this will call
+        release_fn and return its result (if provided), or return True
+        (if release_fn was not provided).
+
+        Otherwise, we are still nested inside some other lock, so do not
+        call the release_fn and, return False.
 
         Does limited correctness checking: if a read lock is released
         when none are held, this will raise an assertion error.
@@ -296,81 +416,160 @@ def release_read(self):
         """
         assert self._reads > 0
 
+        locktype = 'READ LOCK'
         if self._reads == 1 and self._writes == 0:
-            self._debug(
-                'READ LOCK: {0.path}[{0._start}:{0._length}] [Released]'
-                .format(self))
+            self._log_releasing(locktype)
+
+            # we need to call release_fn before releasing the lock
+            release_fn = release_fn or true_fn
+            result = release_fn()
+
             self._unlock()      # can raise LockError.
-            self._reads -= 1
-            return True
+            self._reads = 0
+            self._log_released(locktype)
+            return result
         else:
             self._reads -= 1
             return False
 
-    def release_write(self):
+    def release_write(self, release_fn=None):
         """Releases a write lock.
 
-        Returns True if the last recursive lock was released, False if
-        there are still outstanding locks.
+        Arguments:
+            release_fn (callable): function to call before the last recursive
+                write is released.
+
+        If the last recursive *write* lock will be released, then this
+        will call release_fn and return its result (if provided), or
+        return True (if release_fn was not provided). Otherwise, we are
+        still nested inside some other write lock, so do not call the
+        release_fn, and return False.
 
         Does limited correctness checking: if a read lock is released
         when none are held, this will raise an assertion error.
 
         """
         assert self._writes > 0
+        release_fn = release_fn or true_fn
 
+        locktype = 'WRITE LOCK'
         if self._writes == 1 and self._reads == 0:
-            self._debug(
-                'WRITE LOCK: {0.path}[{0._start}:{0._length}] [Released]'
-                .format(self))
+            self._log_releasing(locktype)
+
+            # we need to call release_fn before releasing the lock
+            result = release_fn()
+
             self._unlock()      # can raise LockError.
-            self._writes -= 1
-            return True
+            self._writes = 0
+            self._log_released(locktype)
+            return result
         else:
             self._writes -= 1
-            return False
+
+            # when the last *write* is released, we call release_fn here
+            # instead of immediately before releasing the lock.
+            if self._writes == 0:
+                return release_fn()
+            else:
+                return False
 
     def _debug(self, *args):
         tty.debug(*args)
 
-    def _acquired_debug(self, lock_type, wait_time, nattempts):
-        attempts_format = 'attempt' if nattempts == 1 else 'attempt'
-        if nattempts > 1:
-            acquired_attempts_format = ' after {0:0.2f}s and {1:d} {2}'.format(
-                wait_time, nattempts, attempts_format)
-        else:
-            # Dont print anything if we succeeded immediately
-            acquired_attempts_format = ''
-        self._debug(
-            '{0}: {1.path}[{1._start}:{1._length}] [Acquired{2}]'
-            .format(lock_type, self, acquired_attempts_format))
+    def _get_counts_desc(self):
+        return '(reads {0}, writes {1})'.format(self._reads, self._writes) \
+            if tty.is_verbose() else ''
+
+    def _log_acquired(self, locktype, wait_time, nattempts):
+        attempts_part = _attempts_str(wait_time, nattempts)
+        now = datetime.now()
+        desc = 'Acquired at %s' % now.strftime("%H:%M:%S.%f")
+        self._debug(self._status_msg(locktype, '{0}{1}'.
+                                     format(desc, attempts_part)))
+
+    def _log_acquiring(self, locktype):
+        self._debug2(self._status_msg(locktype, 'Acquiring'))
+
+    def _log_downgraded(self, wait_time, nattempts):
+        attempts_part = _attempts_str(wait_time, nattempts)
+        now = datetime.now()
+        desc = 'Downgraded at %s' % now.strftime("%H:%M:%S.%f")
+        self._debug(self._status_msg('READ LOCK', '{0}{1}'
+                                     .format(desc, attempts_part)))
+
+    def _log_downgrading(self):
+        self._debug2(self._status_msg('WRITE LOCK', 'Downgrading'))
+
+    def _log_released(self, locktype):
+        now = datetime.now()
+        desc = 'Released at %s' % now.strftime("%H:%M:%S.%f")
+        self._debug(self._status_msg(locktype, desc))
+
+    def _log_releasing(self, locktype):
+        self._debug2(self._status_msg(locktype, 'Releasing'))
+
+    def _log_upgraded(self, wait_time, nattempts):
+        attempts_part = _attempts_str(wait_time, nattempts)
+        now = datetime.now()
+        desc = 'Upgraded at %s' % now.strftime("%H:%M:%S.%f")
+        self._debug(self._status_msg('WRITE LOCK', '{0}{1}'.
+                                     format(desc, attempts_part)))
+
+    def _log_upgrading(self):
+        self._debug2(self._status_msg('READ LOCK', 'Upgrading'))
+
+    def _status_msg(self, locktype, status):
+        status_desc = '[{0}] {1}'.format(status, self._get_counts_desc())
+        return '{0}{1.desc}: {1.path}[{1._start}:{1._length}] {2}'.format(
+            locktype, self, status_desc)
+
+    def _debug2(self, *args):
+        # TODO: Easy place to make a single, temporary change to the
+        # TODO:   debug level associated with the more detailed messages.
+        # TODO:
+        # TODO:   Someday it would be great if we could switch this to
+        # TODO:   another level, perhaps _between_ debug and verbose, or
+        # TODO:   some other form of filtering so the first level of
+        # TODO:   debugging doesn't have to generate these messages.  Using
+        # TODO:   verbose here did not work as expected because tests like
+        # TODO:   test_spec_json will write the verbose messages to the
+        # TODO:   output that is used to check test correctness.
+        tty.debug(*args)
 
 
 class LockTransaction(object):
     """Simple nested transaction context manager that uses a file lock.
 
-    This class can trigger actions when the lock is acquired for the
-    first time and released for the last.
+    Arguments:
+        lock (Lock): underlying lock for this transaction to be accquired on
+            enter and released on exit
+        acquire (callable or contextmanager): function to be called after lock
+            is acquired, or contextmanager to enter after acquire and leave
+            before release.
+        release (callable): function to be called before release. If
+            ``acquire`` is a contextmanager, this will be called *after*
+            exiting the nexted context and before the lock is released.
+        timeout (float): number of seconds to set for the timeout when
+            accquiring the lock (default no timeout)
 
     If the ``acquire_fn`` returns a value, it is used as the return value for
     ``__enter__``, allowing it to be passed as the ``as`` argument of a
     ``with`` statement.
 
     If ``acquire_fn`` returns a context manager, *its* ``__enter__`` function
-    will be called in ``__enter__`` after ``acquire_fn``, and its ``__exit__``
-    funciton will be called before ``release_fn`` in ``__exit__``, allowing you
-    to nest a context manager to be used along with the lock.
+    will be called after the lock is acquired, and its ``__exit__`` funciton
+    will be called before ``release_fn`` in ``__exit__``, allowing you to
+    nest a context manager inside this one.
 
     Timeout for lock is customizable.
 
     """
 
-    def __init__(self, lock, acquire_fn=None, release_fn=None,
-                 timeout=None):
+    def __init__(self, lock, acquire=None, release=None, timeout=None):
         self._lock = lock
         self._timeout = timeout
-        self._acquire_fn = acquire_fn
-        self._release_fn = release_fn
+        self._acquire_fn = acquire
+        self._release_fn = release
         self._as = None
 
     def __enter__(self):
@@ -383,13 +582,18 @@ def __enter__(self):
 
     def __exit__(self, type, value, traceback):
         suppress = False
-        if self._exit():
-            if self._as and hasattr(self._as, '__exit__'):
-                if self._as.__exit__(type, value, traceback):
-                    suppress = True
-            if self._release_fn:
-                if self._release_fn(type, value, traceback):
-                    suppress = True
+
+        def release_fn():
+            if self._release_fn is not None:
+                return self._release_fn(type, value, traceback)
+
+        if self._as and hasattr(self._as, '__exit__'):
+            if self._as.__exit__(type, value, traceback):
+                suppress = True
+
+        if self._exit(release_fn):
+            suppress = True
+
         return suppress
 
 
@@ -398,8 +602,8 @@ class ReadTransaction(LockTransaction):
     def _enter(self):
         return self._lock.acquire_read(self._timeout)
 
-    def _exit(self):
-        return self._lock.release_read()
+    def _exit(self, release_fn):
+        return self._lock.release_read(release_fn)
 
 
 class WriteTransaction(LockTransaction):
@@ -407,18 +611,36 @@ class WriteTransaction(LockTransaction):
     def _enter(self):
         return self._lock.acquire_write(self._timeout)
 
-    def _exit(self):
-        return self._lock.release_write()
+    def _exit(self, release_fn):
+        return self._lock.release_write(release_fn)
 
 
 class LockError(Exception):
     """Raised for any errors related to locks."""
 
 
+class LockDowngradeError(LockError):
+    """Raised when unable to downgrade from a write to a read lock."""
+    def __init__(self, path):
+        msg = "Cannot downgrade lock from write to read on file: %s" % path
+        super(LockDowngradeError, self).__init__(msg)
+
+
+class LockLimitError(LockError):
+    """Raised when exceed maximum attempts to acquire a lock."""
+
+
 class LockTimeoutError(LockError):
     """Raised when an attempt to acquire a lock times out."""
 
 
+class LockUpgradeError(LockError):
+    """Raised when unable to upgrade from a read to a write lock."""
+    def __init__(self, path):
+        msg = "Cannot upgrade lock from read to write on file: %s" % path
+        super(LockUpgradeError, self).__init__(msg)
+
+
 class LockPermissionError(LockError):
     """Raised when there are permission issues with a lock."""
 

lib/spack/llnl/util/multiproc.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/tty/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -135,7 +135,9 @@ def process_stacktrace(countback):
 def get_timestamp(force=False):
     """Get a string timestamp"""
     if _debug or _timestamp or force:
-        return datetime.now().strftime("[%Y-%m-%d-%H:%M:%S.%f] ")
+        # Note inclusion of the PID is useful for parallel builds.
+        return '[{0}, {1}] '.format(
+            datetime.now().strftime("%Y-%m-%d-%H:%M:%S.%f"), os.getpid())
     else:
         return ''
 

lib/spack/llnl/util/tty/colify.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -199,10 +199,16 @@ def colify(elts, **options):
 def colify_table(table, **options):
     """Version of ``colify()`` for data expressed in rows, (list of lists).
 
-       Same as regular colify but takes a list of lists, where each
-       sub-list must be the same length, and each is interpreted as a
-       row in a table.  Regular colify displays a sequential list of
-       values in columns.
+       Same as regular colify but:
+
+       1. This takes a list of lists, where each sub-list must be the
+          same length, and each is interpreted as a row in a table.
+          Regular colify displays a sequential list of values in columns.
+
+       2. Regular colify will always print with 1 column when the output
+          is not a tty.  This will always print with same dimensions of
+          the table argument.
+
     """
     if table is None:
         raise TypeError("Can't call colify_table on NoneType")
@@ -220,6 +226,9 @@ def transpose():
         raise ValueError("Cannot override columsn in colify_table.")
     options['cols'] = columns
 
+    # don't reduce to 1 column for non-tty
+    options['tty'] = True
+
     colify(transpose(), **options)
 
 

lib/spack/llnl/util/tty/color.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/llnl/util/tty/log.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -13,12 +13,18 @@
 import select
 import sys
 import traceback
+import signal
 from contextlib import contextmanager
 from six import string_types
 from six import StringIO
 
 import llnl.util.tty as tty
 
+try:
+    import termios
+except ImportError:
+    termios = None
+
 # Use this to strip escape sequences
 _escape = re.compile(r'\x1b[^m]*m|\x1b\[?1034h')
 
@@ -31,12 +37,26 @@
 control = re.compile('(\x11\n|\x13\n)')
 
 
+@contextmanager
+def background_safe():
+    signal.signal(signal.SIGTTOU, signal.SIG_IGN)
+    yield
+    signal.signal(signal.SIGTTOU, signal.SIG_DFL)
+
+
+def _is_background_tty():
+    """Return True iff this process is backgrounded and stdout is a tty"""
+    if sys.stdout.isatty():
+        return os.getpgrp() != os.tcgetpgrp(sys.stdout.fileno())
+    return False  # not writing to tty, not background
+
+
 def _strip(line):
     """Strip color and control characters from a line."""
     return _escape.sub('', line)
 
 
-class keyboard_input(object):
+class _keyboard_input(object):
     """Context manager to disable line editing and echoing.
 
     Use this with ``sys.stdin`` for keyboard input, e.g.::
@@ -81,32 +101,30 @@ def __enter__(self):
         if not self.stream or not self.stream.isatty():
             return
 
-        try:
-            # If this fails, self.old_cfg will remain None
-            import termios
-
+        # If this fails, self.old_cfg will remain None
+        if termios and not _is_background_tty():
             # save old termios settings
-            fd = self.stream.fileno()
-            self.old_cfg = termios.tcgetattr(fd)
+            old_cfg = termios.tcgetattr(self.stream)
 
-            # create new settings with canonical input and echo
-            # disabled, so keypresses are immediate & don't echo.
-            self.new_cfg = termios.tcgetattr(fd)
-            self.new_cfg[3] &= ~termios.ICANON
-            self.new_cfg[3] &= ~termios.ECHO
+            try:
+                # create new settings with canonical input and echo
+                # disabled, so keypresses are immediate & don't echo.
+                self.new_cfg = termios.tcgetattr(self.stream)
+                self.new_cfg[3] &= ~termios.ICANON
+                self.new_cfg[3] &= ~termios.ECHO
 
-            # Apply new settings for terminal
-            termios.tcsetattr(fd, termios.TCSADRAIN, self.new_cfg)
+                # Apply new settings for terminal
+                termios.tcsetattr(self.stream, termios.TCSADRAIN, self.new_cfg)
+                self.old_cfg = old_cfg
 
-        except Exception:
-            pass  # some OS's do not support termios, so ignore
+            except Exception:
+                pass  # some OS's do not support termios, so ignore
 
     def __exit__(self, exc_type, exception, traceback):
         """If termios was avaialble, restore old settings."""
         if self.old_cfg:
-            import termios
-            termios.tcsetattr(
-                self.stream.fileno(), termios.TCSADRAIN, self.old_cfg)
+            with background_safe():  # change it back even if backgrounded now
+                termios.tcsetattr(self.stream, termios.TCSADRAIN, self.old_cfg)
 
 
 class Unbuffered(object):
@@ -426,45 +444,63 @@ def _writer_daemon(self, stdin):
         istreams = [in_pipe, stdin] if stdin else [in_pipe]
 
         log_file = self.log_file
+
+        def handle_write(force_echo):
+            # Handle output from the with block process.
+            # If we arrive here it means that in_pipe was
+            # ready for reading : it should never happen that
+            # line is false-ish
+            line = in_pipe.readline()
+            if not line:
+                return (True, force_echo)  # break while loop
+
+            # find control characters and strip them.
+            controls = control.findall(line)
+            line = re.sub(control, '', line)
+
+            # Echo to stdout if requested or forced
+            if echo or force_echo:
+                try:
+                    if termios:
+                        conf = termios.tcgetattr(sys.stdout)
+                        tostop = conf[3] & termios.TOSTOP
+                    else:
+                        tostop = True
+                except Exception:
+                    tostop = True
+                if not (tostop and _is_background_tty()):
+                    sys.stdout.write(line)
+                    sys.stdout.flush()
+
+            # Stripped output to log file.
+            log_file.write(_strip(line))
+            log_file.flush()
+
+            if xon in controls:
+                force_echo = True
+            if xoff in controls:
+                force_echo = False
+            return (False, force_echo)
+
         try:
-            with keyboard_input(stdin):
+            with _keyboard_input(stdin):
                 while True:
                     # No need to set any timeout for select.select
                     # Wait until a key press or an event on in_pipe.
                     rlist, _, _ = select.select(istreams, [], [])
-
                     # Allow user to toggle echo with 'v' key.
                     # Currently ignores other chars.
-                    if stdin in rlist:
+                    # only read stdin if we're in the foreground
+                    if stdin in rlist and not _is_background_tty():
                         if stdin.read(1) == 'v':
                             echo = not echo
 
-                    # Handle output from the with block process.
                     if in_pipe in rlist:
-                        # If we arrive here it means that in_pipe was
-                        # ready for reading : it should never happen that
-                        # line is false-ish
-                        line = in_pipe.readline()
-                        if not line:
-                            break  # EOF
+                        br, fe = handle_write(force_echo)
+                        force_echo = fe
+                        if br:
+                            break
 
-                        # find control characters and strip them.
-                        controls = control.findall(line)
-                        line = re.sub(control, '', line)
-
-                        # Echo to stdout if requested or forced
-                        if echo or force_echo:
-                            sys.stdout.write(line)
-                            sys.stdout.flush()
-
-                        # Stripped output to log file.
-                        log_file.write(_strip(line))
-                        log_file.flush()
-
-                        if xon in controls:
-                            force_echo = True
-                        if xoff in controls:
-                            force_echo = False
         except BaseException:
             tty.error("Exception occurred in writer daemon!")
             traceback.print_exc()

lib/spack/spack/__init__.py

@@ -1,11 +1,11 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 
 #: major, minor, patch version for Spack, in a tuple
-spack_version_info = (0, 13, 2)
+spack_version_info = (0, 14, 1)
 
 #: String containing Spack version joined with .'s
 spack_version = '.'.join(str(v) for v in spack_version_info)

lib/spack/spack/abi.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/architecture.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -69,6 +69,7 @@
 import spack.compiler
 import spack.paths
 import spack.error as serr
+import spack.util.executable
 import spack.version
 from spack.util.naming import mod_to_class
 from spack.util.spack_yaml import syaml_dict
@@ -214,7 +215,11 @@ def optimization_flags(self, compiler):
             import spack.spec
             if isinstance(compiler, spack.spec.CompilerSpec):
                 compiler = spack.compilers.compilers_for_spec(compiler).pop()
-            compiler_version = compiler.cc_version(compiler.cc)
+            try:
+                compiler_version = compiler.cc_version(compiler.cc)
+            except spack.util.executable.ProcessError as e:
+                # log this and just return compiler.version instead
+                tty.debug(str(e))
 
         return self.microarchitecture.optimization_flags(
             compiler.name, str(compiler_version)
@@ -351,10 +356,10 @@ def _cmp_key(self):
         return (self.name, self.version)
 
     def to_dict(self):
-        return {
-            'name': self.name,
-            'version': self.version
-        }
+        return syaml_dict([
+            ('name', self.name),
+            ('version', self.version)
+        ])
 
 
 @key_ordering
@@ -436,6 +441,7 @@ def from_dict(d):
         return arch_for_spec(spec)
 
 
+@memoized
 def get_platform(platform_name):
     """Returns a platform object that corresponds to the given name."""
     platform_list = all_platforms()

lib/spack/spack/binary_distribution.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -18,12 +18,12 @@
 from six.moves.urllib.error import URLError
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import mkdirp, install_tree
+from llnl.util.filesystem import mkdirp
 
 import spack.cmd
 import spack.config as config
 import spack.fetch_strategy as fs
-import spack.util.gpg as gpg_util
+import spack.util.gpg
 import spack.relocate as relocate
 import spack.util.spack_yaml as syaml
 import spack.mirror
@@ -33,7 +33,7 @@
 from spack.spec import Spec
 from spack.stage import Stage
 from spack.util.gpg import Gpg
-from spack.util.executable import ProcessError
+import spack.architecture as architecture
 
 _build_cache_relative_path = 'build_cache'
 
@@ -110,14 +110,6 @@ class NewLayoutException(spack.error.SpackError):
     pass
 
 
-def has_gnupg2():
-    try:
-        gpg_util.Gpg.gpg()('--version', output=os.devnull)
-        return True
-    except ProcessError:
-        return False
-
-
 def build_cache_relative_path():
     return _build_cache_relative_path
 
@@ -243,27 +235,31 @@ def checksum_tarball(file):
 
 def sign_tarball(key, force, specfile_path):
     # Sign the packages if keys available
-    if not has_gnupg2():
+    if spack.util.gpg.Gpg.gpg() is None:
         raise NoGpgException(
             "gpg2 is not available in $PATH .\n"
             "Use spack install gnupg and spack load gnupg.")
-    else:
-        if key is None:
-            keys = Gpg.signing_keys()
-            if len(keys) == 1:
-                key = keys[0]
-            if len(keys) > 1:
-                raise PickKeyException(str(keys))
-            if len(keys) == 0:
-                msg = "No default key available for signing.\n"
-                msg += "Use spack gpg init and spack gpg create"
-                msg += " to create a default key."
-                raise NoKeyException(msg)
+
+    if key is None:
+        keys = Gpg.signing_keys()
+        if len(keys) == 1:
+            key = keys[0]
+
+        if len(keys) > 1:
+            raise PickKeyException(str(keys))
+
+        if len(keys) == 0:
+            msg = "No default key available for signing.\n"
+            msg += "Use spack gpg init and spack gpg create"
+            msg += " to create a default key."
+            raise NoKeyException(msg)
+
     if os.path.exists('%s.asc' % specfile_path):
         if force:
             os.remove('%s.asc' % specfile_path)
         else:
             raise NoOverwriteException('%s.asc' % specfile_path)
+
     Gpg.sign(key, specfile_path, '%s.asc' % specfile_path)
 
 
@@ -272,7 +268,7 @@ def generate_package_index(cache_prefix):
 
     Creates (or replaces) the "index.html" page at the location given in
     cache_prefix.  This page contains a link for each binary package (*.yaml)
-    and signing key (*.key) under cache_prefix.
+    and public key (*.key) under cache_prefix.
     """
     tmpdir = tempfile.mkdtemp()
     try:
@@ -312,7 +308,7 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
     tmpdir = tempfile.mkdtemp()
     cache_prefix = build_cache_prefix(tmpdir)
 
-    tarfile_name = tarball_name(spec, '.tar.gz')
+    tarfile_name = tarball_name(spec, '.tar.bz2')
     tarfile_dir = os.path.join(cache_prefix, tarball_directory_name(spec))
     tarfile_path = os.path.join(tarfile_dir, tarfile_name)
     spackfile_path = os.path.join(
@@ -346,8 +342,18 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
             raise NoOverwriteException(url_util.format(remote_specfile_path))
 
     # make a copy of the install directory to work with
-    workdir = os.path.join(tempfile.mkdtemp(), os.path.basename(spec.prefix))
-    install_tree(spec.prefix, workdir, symlinks=True)
+    workdir = os.path.join(tmpdir, os.path.basename(spec.prefix))
+    # install_tree copies hardlinks
+    # create a temporary tarfile from prefix and exract it to workdir
+    # tarfile preserves hardlinks
+    temp_tarfile_name = tarball_name(spec, '.tar')
+    temp_tarfile_path = os.path.join(tarfile_dir, temp_tarfile_name)
+    with closing(tarfile.open(temp_tarfile_path, 'w')) as tar:
+        tar.add(name='%s' % spec.prefix,
+                arcname='.')
+    with closing(tarfile.open(temp_tarfile_path, 'r')) as tar:
+        tar.extractall(workdir)
+    os.remove(temp_tarfile_path)
 
     # create info for later relocation and create tar
     write_buildinfo_file(spec.prefix, workdir, rel=rel)
@@ -372,7 +378,7 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
             tty.die(e)
 
     # create compressed tarball of the install prefix
-    with closing(tarfile.open(tarfile_path, 'w:gz')) as tar:
+    with closing(tarfile.open(tarfile_path, 'w:bz2')) as tar:
         tar.add(name='%s' % workdir,
                 arcname='%s' % os.path.basename(spec.prefix))
     # remove copy of install directory
@@ -411,8 +417,8 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
         sign_tarball(key, force, specfile_path)
     # put tarball, spec and signature files in .spack archive
     with closing(tarfile.open(spackfile_path, 'w')) as tar:
-        tar.add(name='%s' % tarfile_path, arcname='%s' % tarfile_name)
-        tar.add(name='%s' % specfile_path, arcname='%s' % specfile_name)
+        tar.add(name=tarfile_path, arcname='%s' % tarfile_name)
+        tar.add(name=specfile_path, arcname='%s' % specfile_name)
         if not unsigned:
             tar.add(name='%s.asc' % specfile_path,
                     arcname='%s.asc' % specfile_name)
@@ -583,13 +589,17 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
     stagepath = os.path.dirname(filename)
     spackfile_name = tarball_name(spec, '.spack')
     spackfile_path = os.path.join(stagepath, spackfile_name)
-    tarfile_name = tarball_name(spec, '.tar.gz')
+    tarfile_name = tarball_name(spec, '.tar.bz2')
     tarfile_path = os.path.join(tmpdir, tarfile_name)
     specfile_name = tarball_name(spec, '.spec.yaml')
     specfile_path = os.path.join(tmpdir, specfile_name)
 
     with closing(tarfile.open(spackfile_path, 'r')) as tar:
         tar.extractall(tmpdir)
+    # older buildcache tarfiles use gzip compression
+    if not os.path.exists(tarfile_path):
+        tarfile_name = tarball_name(spec, '.tar.gz')
+        tarfile_path = os.path.join(tmpdir, tarfile_name)
     if not unsigned:
         if os.path.exists('%s.asc' % specfile_path):
             try:
@@ -642,7 +652,17 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
     # so the pathname should be the same now that the directory layout
     # is confirmed
     workdir = os.path.join(tmpdir, os.path.basename(spec.prefix))
-    install_tree(workdir, spec.prefix, symlinks=True)
+    # install_tree copies hardlinks
+    # create a temporary tarfile from prefix and exract it to workdir
+    # tarfile preserves hardlinks
+    temp_tarfile_name = tarball_name(spec, '.tar')
+    temp_tarfile_path = os.path.join(tmpdir, temp_tarfile_name)
+    with closing(tarfile.open(temp_tarfile_path, 'w')) as tar:
+        tar.add(name='%s' % workdir,
+                arcname='.')
+    with closing(tarfile.open(temp_tarfile_path, 'r')) as tar:
+        tar.extractall(spec.prefix)
+    os.remove(temp_tarfile_path)
 
     # cleanup
     os.remove(tarfile_path)
@@ -664,48 +684,17 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
         shutil.rmtree(tmpdir)
 
 
-#: Internal cache for get_specs
-_cached_specs = None
+# Internal cache for downloaded specs
+_cached_specs = set()
 
 
-def get_specs(force=False):
-    """
-    Get spec.yaml's for build caches available on mirror
-    """
+def try_download_specs(urls=None, force=False):
+    '''
+    Try to download the urls and cache them
+    '''
     global _cached_specs
-
-    if _cached_specs:
-        tty.debug("Using previously-retrieved specs")
-        return _cached_specs
-
-    if not spack.mirror.MirrorCollection():
-        tty.warn("No Spack mirrors are currently configured")
+    if urls is None:
         return {}
-
-    urls = set()
-    for mirror in spack.mirror.MirrorCollection().values():
-        fetch_url_build_cache = url_util.join(
-            mirror.fetch_url, _build_cache_relative_path)
-
-        mirror_dir = url_util.local_file_path(fetch_url_build_cache)
-        if mirror_dir:
-            tty.msg("Finding buildcaches in %s" % mirror_dir)
-            if os.path.exists(mirror_dir):
-                files = os.listdir(mirror_dir)
-                for file in files:
-                    if re.search('spec.yaml', file):
-                        link = url_util.join(fetch_url_build_cache, file)
-                        urls.add(link)
-        else:
-            tty.msg("Finding buildcaches at %s" %
-                    url_util.format(fetch_url_build_cache))
-            p, links = web_util.spider(
-                url_util.join(fetch_url_build_cache, 'index.html'))
-            for link in links:
-                if re.search("spec.yaml", link):
-                    urls.add(link)
-
-    _cached_specs = []
     for link in urls:
         with Stage(link, name="build_cache", keep=True) as stage:
             if force and os.path.exists(stage.save_filename):
@@ -721,11 +710,92 @@ def get_specs(force=False):
                 # we need to mark this spec concrete on read-in.
                 spec = Spec.from_yaml(f)
                 spec._mark_concrete()
-                _cached_specs.append(spec)
+                _cached_specs.add(spec)
 
     return _cached_specs
 
 
+def get_spec(spec=None, force=False):
+    """
+    Check if spec.yaml exists on mirrors and return it if it does
+    """
+    global _cached_specs
+    urls = set()
+    if spec is None:
+        return {}
+    specfile_name = tarball_name(spec, '.spec.yaml')
+
+    if not spack.mirror.MirrorCollection():
+        tty.debug("No Spack mirrors are currently configured")
+        return {}
+
+    if spec in _cached_specs:
+        return _cached_specs
+
+    for mirror in spack.mirror.MirrorCollection().values():
+        fetch_url_build_cache = url_util.join(
+            mirror.fetch_url, _build_cache_relative_path)
+
+        mirror_dir = url_util.local_file_path(fetch_url_build_cache)
+        if mirror_dir:
+            tty.msg("Finding buildcaches in %s" % mirror_dir)
+            link = url_util.join(fetch_url_build_cache, specfile_name)
+            urls.add(link)
+
+        else:
+            tty.msg("Finding buildcaches at %s" %
+                    url_util.format(fetch_url_build_cache))
+            link = url_util.join(fetch_url_build_cache, specfile_name)
+            urls.add(link)
+
+    return try_download_specs(urls=urls, force=force)
+
+
+def get_specs(force=False, allarch=False):
+    """
+    Get spec.yaml's for build caches available on mirror
+    """
+    arch = architecture.Arch(architecture.platform(),
+                             'default_os', 'default_target')
+    arch_pattern = ('([^-]*-[^-]*-[^-]*)')
+    if not allarch:
+        arch_pattern = '(%s-%s-[^-]*)' % (arch.platform, arch.os)
+
+    regex_pattern = '%s(.*)(spec.yaml$)' % (arch_pattern)
+    arch_re = re.compile(regex_pattern)
+
+    if not spack.mirror.MirrorCollection():
+        tty.debug("No Spack mirrors are currently configured")
+        return {}
+
+    urls = set()
+    for mirror in spack.mirror.MirrorCollection().values():
+        fetch_url_build_cache = url_util.join(
+            mirror.fetch_url, _build_cache_relative_path)
+
+        mirror_dir = url_util.local_file_path(fetch_url_build_cache)
+        if mirror_dir:
+            tty.msg("Finding buildcaches in %s" % mirror_dir)
+            if os.path.exists(mirror_dir):
+                files = os.listdir(mirror_dir)
+                for file in files:
+                    m = arch_re.search(file)
+                    if m:
+                        link = url_util.join(fetch_url_build_cache, file)
+                        urls.add(link)
+        else:
+            tty.msg("Finding buildcaches at %s" %
+                    url_util.format(fetch_url_build_cache))
+            p, links = web_util.spider(
+                url_util.join(fetch_url_build_cache, 'index.html'))
+            for link in links:
+                m = arch_re.search(link)
+                if m:
+                    urls.add(link)
+
+    return try_download_specs(urls=urls, force=force)
+
+
 def get_keys(install=False, trust=False, force=False):
     """
     Get pgp public keys available on mirror
@@ -803,10 +873,10 @@ def needs_rebuild(spec, mirror_url, rebuild_on_errors=False):
     try:
         _, _, yaml_file = web_util.read_from_url(file_path)
         yaml_contents = codecs.getreader('utf-8')(yaml_file).read()
-    except URLError as url_err:
+    except (URLError, web_util.SpackWebError) as url_err:
         err_msg = [
             'Unable to determine whether {0} needs rebuilding,',
-            ' caught URLError attempting to read from {1}.',
+            ' caught exception attempting to read from {1}.',
         ]
         tty.error(''.join(err_msg).format(spec.short_spec, file_path))
         tty.debug(url_err)
@@ -908,11 +978,16 @@ def _download_buildcache_entry(mirror_root, descriptions):
     return True
 
 
-def download_buildcache_entry(file_descriptions):
-    if not spack.mirror.MirrorCollection():
-        tty.die("Please add a spack mirror to allow " +
+def download_buildcache_entry(file_descriptions, mirror_url=None):
+    if not mirror_url and not spack.mirror.MirrorCollection():
+        tty.die("Please provide or add a spack mirror to allow " +
                 "download of buildcache entries.")
 
+    if mirror_url:
+        mirror_root = os.path.join(
+            mirror_url, _build_cache_relative_path)
+        return _download_buildcache_entry(mirror_root, file_descriptions)
+
     for mirror in spack.mirror.MirrorCollection().values():
         mirror_root = os.path.join(
             mirror.fetch_url,

lib/spack/spack/build_environment.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -39,7 +39,6 @@
 import sys
 import traceback
 import types
-from six import iteritems
 from six import StringIO
 
 import llnl.util.tty as tty
@@ -52,6 +51,7 @@
 import spack.config
 import spack.main
 import spack.paths
+import spack.schema.environment
 import spack.store
 from spack.util.string import plural
 from spack.util.environment import (
@@ -342,21 +342,7 @@ def set_build_environment_variables(pkg, env, dirty):
     # Set environment variables if specified for
     # the given compiler
     compiler = pkg.compiler
-    environment = compiler.environment
-
-    for command, variable in iteritems(environment):
-        if command == 'set':
-            for name, value in iteritems(variable):
-                env.set(name, value)
-        elif command == 'unset':
-            for name, _ in iteritems(variable):
-                env.unset(name)
-        elif command == 'prepend-path':
-            for name, value in iteritems(variable):
-                env.prepend_path(name, value)
-        elif command == 'append-path':
-            for name, value in iteritems(variable):
-                env.append_path(name, value)
+    env.extend(spack.schema.environment.parse(compiler.environment))
 
     if compiler.extra_rpaths:
         extra_rpaths = ':'.join(compiler.extra_rpaths)

lib/spack/spack/build_systems/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/build_systems/aspell_dict.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/build_systems/autotools.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/build_systems/cmake.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)

lib/spack/spack/build_systems/cuda.py

@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -61,12 +61,13 @@ def cuda_flags(arch_list):
     conflicts('%gcc@6:', when='+cuda ^cuda@:8' + arch_platform)
     conflicts('%gcc@7:', when='+cuda ^cuda@:9.1' + arch_platform)
     conflicts('%gcc@8:', when='+cuda ^cuda@:10.0.130' + arch_platform)
-    conflicts('%gcc@9:', when='+cuda ^cuda@:10.1.243' + arch_platform)
+    conflicts('%gcc@9:', when='+cuda ^cuda@:10.2.89' + arch_platform)
     conflicts('%pgi@:14.8', when='+cuda ^cuda@:7.0.27' + arch_platform)
     conflicts('%pgi@:15.3,15.5:', when='+cuda ^cuda@7.5' + arch_platform)
     conflicts('%pgi@:16.2,16.0:16.3', when='+cuda ^cuda@8' + arch_platform)
     conflicts('%pgi@:15,18:', when='+cuda ^cuda@9.0:9.1' + arch_platform)
     conflicts('%pgi@:16', when='+cuda ^cuda@9.2.88:10' + arch_platform)
+    conflicts('%pgi@:17', when='+cuda ^cuda@10.2.89' + arch_platform)
     conflicts('%clang@:3.4', when='+cuda ^cuda@:7.5' + arch_platform)
     conflicts('%clang@:3.7,4:',
               when='+cuda ^cuda@8.0:9.0' + arch_platform)
@@ -74,6 +75,10 @@ def cuda_flags(arch_list):
               when='+cuda ^cuda@9.1' + arch_platform)
     conflicts('%clang@:3.7,5.1:', when='+cuda ^cuda@9.2' + arch_platform)
     conflicts('%clang@:3.7,6.1:', when='+cuda ^cuda@10.0.130' + arch_platform)
+    conflicts('%clang@:3.7,7.1:', when='+cuda ^cuda@10.1.105' + arch_platform)
+    conflicts('%clang@:3.7,8.1:',
+              when='+cuda ^cuda@10.1.105:10.1.243' + arch_platform)
+    conflicts('%clang@:3.2,9.0:', when='+cuda ^cuda@10.2.89' + arch_platform)
 
     # x86_64 vs. ppc64le differ according to NVidia docs
     # Linux ppc64le compiler conflicts from Table from the docs below:
@@ -95,6 +100,8 @@ def cuda_flags(arch_list):
     conflicts('%clang@5:', when='+cuda ^cuda@:9.1' + arch_platform)
     conflicts('%clang@6:', when='+cuda ^cuda@:9.2' + arch_platform)
     conflicts('%clang@7:', when='+cuda ^cuda@10.0.130' + arch_platform)
+    conflicts('%clang@7.1:', when='+cuda ^cuda@:10.1.105' + arch_platform)
+    conflicts('%clang@8.1:', when='+cuda ^cuda@:10.2.89' + arch_platform)
 
     # Intel is mostly relevant for x86_64 Linux, even though it also
     # exists for Mac OS X. No information prior to CUDA 3.2 or Intel 11.1
@@ -107,12 +114,12 @@ def cuda_flags(arch_list):
     conflicts('%intel@16.0:', when='+cuda ^cuda@:8.0.43')
     conflicts('%intel@17.0:', when='+cuda ^cuda@:8.0.60')
     conflicts('%intel@18.0:', when='+cuda ^cuda@:9.9')
-    conflicts('%intel@19.0:', when='+cuda ^cuda@:10.0')
+    conflicts('%intel@19.0:', when='+cuda ^cuda@:10.2.89')
 
     # XL is mostly relevant for ppc64le Linux
     conflicts('%xl@:12,14:', when='+cuda ^cuda@:9.1')
     conflicts('%xl@:12,14:15,17:', when='+cuda ^cuda@9.2')
-    conflicts('%xl@17:', when='+cuda ^cuda@10.0.130')
+    conflicts('%xl@17:', when='+cuda ^cuda@10.0.130:10.2.89')
 
     # Mac OS X
     # platform = ' platform=darwin'