Separate integration tests; simplify test scripts (#4006)

* Separate build integration tests; simplify test scripts

- Move build tests out of the regular Travis unit tests, add more smoke
  test packages to build.

- Run all test scripts with bash -e, which fails on error.

- Factor coverage out into a Travis environment variable, so it's more
  obvious from .travis.yml which tests contribute to coverage and which
  don't.

- Factor dependency checking and much of the front-matter in tests
  scripts into a setup.sh script, which is sourced by all the test
  scripts.  Extra cruft in each tests script now reduced to 2 lines at
  the beginning.

.travis.yml

@@ -21,7 +21,11 @@ matrix:
     - python: '2.7'
       os: linux
       language: python
-      env: TEST_SUITE=unit
+      env: [ TEST_SUITE=unit, COVERAGE=true ]
+    - python: '2.7'
+      os: linux
+      language: python
+      env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=hypre^mpich' ]
     - python: '3.3'
       os: linux
       language: python
@@ -37,7 +41,11 @@ matrix:
     - python: '3.6'
       os: linux
       language: python
-      env: TEST_SUITE=unit
+      env: [ TEST_SUITE=unit, COVERAGE=true ]
+    - python: '3.6'
+      os: linux
+      language: python
+      env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=hypre^mpich' ]
     - python: '2.7'
       os: linux
       language: python
@@ -48,7 +56,7 @@ matrix:
       env: TEST_SUITE=doc
     - os: osx
       language: generic
-      env: [ TEST_SUITE=unit, PYTHON_VERSION=2.7 ]
+      env: [ TEST_SUITE=unit, PYTHON_VERSION=2.7, COVERAGE=true ]
 
 #=============================================================================
 # Environment

share/spack/qa/check_dependencies

@@ -1,96 +0,0 @@
-#!/usr/bin/env bash
-#
-# Description:
-#     Check to see if dependencies are installed.
-#     If not, warn the user and tell them how to
-#     install these dependencies.
-#
-# Usage:
-#     check-deps <dep> ...
-#
-# Options:
-#     One or more dependencies. Must use name of binary.
-
-for dep in "$@"; do
-    if ! which $dep &> /dev/null; then
-        # Map binary name to package name
-        case $dep in
-            sphinx-apidoc|sphinx-build)
-                spack_package=py-sphinx
-                pip_package=sphinx
-                ;;
-            coverage)
-                spack_package=py-coverage
-                pip_package=coverage
-                ;;
-            flake8)
-                spack_package=py-flake8
-                pip_package=flake8
-                ;;
-            dot)
-                spack_package=graphviz
-                ;;
-            git)
-                spack_package=git
-                ;;
-            hg)
-                spack_package=mercurial
-                pip_package=mercurial
-                ;;
-            svn)
-                spack_package=subversion
-                ;;
-            *)
-                spack_package=$dep
-                pip_package=$dep
-                ;;
-        esac
-
-        echo "ERROR: $dep is required to run this script."
-        echo
-
-        if [[ $spack_package ]]; then
-            echo "To install with Spack, run:"
-            echo "    $ spack install $spack_package"
-        fi
-
-        if [[ $pip_package ]]; then
-            echo "To install with pip, run:"
-            echo "    $ pip install $pip_package"
-        fi
-
-        if [[ $spack_package || $pip_package ]]; then
-            echo "Then add the bin directory to your PATH."
-        fi
-
-        exit 1
-    fi
-
-    # Flake8 and Sphinx require setuptools in order to run.
-    # Otherwise, they print out this error message:
-    #
-    #   Traceback (most recent call last):
-    #     File: "/usr/bin/flake8", line 5, in <module>
-    #       from pkg_resources import load_entry_point
-    #   ImportError: No module named pkg_resources
-    #
-    # Print a more useful error message if setuptools not found.
-    if [[ $dep == flake8 || $dep == sphinx* ]]; then
-        # Find which Python is being run
-        # Spack-installed packages have a hard-coded shebang
-        python_cmd=$(head -n 1 $(which $dep) | cut -c 3-)
-        # May not have a shebang
-        if [[ $python_cmd != *python* ]]; then
-            python_cmd=python
-        fi
-        # Check if setuptools is in the PYTHONPATH
-        if ! $python_cmd -c "import setuptools" 2> /dev/null; then
-            echo "ERROR: setuptools is required to run $dep."
-            echo "Please add it to your PYTHONPATH."
-
-            exit 1
-        fi
-    fi
-done
-
-echo "Dependencies found."

share/spack/qa/run-build-tests

@@ -0,0 +1,29 @@
+#!/bin/bash -e
+#
+# Description:
+#     Runs Spack build smoke tests.  This installs a few packages that
+#     cover different parts of the build system.  It is not an exhaustive
+#     test of Spack's packages.
+#
+# Usage:
+#     run-build-tests
+#
+. "$(dirname $0)/setup.sh"
+check_dependencies ${coverage} git hg svn
+
+# Move to root directory of Spack
+# Allows script to be run from anywhere
+cd "$SPACK_ROOT"
+
+# Make sure we have a spec to build.
+if [ -z "$SPEC" ]; then
+    echo "Error: run-build-tests requires the $SPEC to build to be set."
+    exit 1
+fi
+
+# Print compiler information
+spack config get compilers
+
+# Run some build smoke tests, potentially with code coverage
+${coverage_run} bin/spack install -v ${SPEC}
+${coverage_combine}

share/spack/qa/run-doc-tests

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash -e
 #
 # Description:
 #     Builds Spack documentation and checks for
@@ -8,33 +8,12 @@
 # Usage:
 #     run-doc-tests
 #
-# Notes:
+. "$(dirname $0)/setup.sh"
-#     Requires sphinx, graphviz, git, mercurial, and subversion.
+check_dependencies sphinx-apidoc sphinx-build dot git hg svn
-#
-
-QA_DIR="$(dirname "$0")"
-SPACK_ROOT="$QA_DIR/../../.."
-DOC_DIR="$SPACK_ROOT/lib/spack/docs"
-
-# Array of dependencies
-deps=(
-    sphinx-apidoc
-    sphinx-build
-    dot
-    git
-    hg
-    svn
-)
-
-# Check for dependencies
-"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1
-
-# Add Spack to the PATH.
-export PATH="$SPACK_ROOT/bin:$PATH"
 
 # Move to documentation directory
 # Allows script to be run from anywhere
-cd "$DOC_DIR"
+cd "$SPACK_ROOT/lib/spack/docs"
 
 # Treat warnings as fatal errors
 make clean --silent

share/spack/qa/run-flake8-tests

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash -e
 #
 # Description:
 #     Runs source code style checks on Spack.
@@ -8,22 +8,7 @@
 # Usage:
 #     run-flake8-tests
 #
-# Notes:
+. "$(dirname $0)/setup.sh"
-#     Requires flake8.
+check_dependencies flake8
-#
 
-QA_DIR="$(dirname "$0")"
+spack flake8
-SPACK_ROOT="$QA_DIR/../../.."
-
-# Array of dependencies
-deps=(
-    flake8
-)
-
-# Check for dependencies
-"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1
-
-# Add Spack to the PATH.
-export PATH="$SPACK_ROOT/bin:$PATH"
-
-exec spack flake8

share/spack/qa/run-unit-tests

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash -e
 #
 # Description:
 #     Runs Spack unit tests.
@@ -10,44 +10,16 @@
 #     Optionally add one or more unit tests
 #     to only run these tests.
 #
-# Notes:
+. "$(dirname $0)/setup.sh"
-#     Requires coverage, git, mercurial, and subversion.
+check_dependencies ${coverage} git hg svn
-#
-
-QA_DIR="$(dirname "$0")"
-SPACK_ROOT="$QA_DIR/../../.."
-
-# Array of dependencies
-deps=(
-    coverage
-    git
-    hg
-    svn
-)
-
-# Check for dependencies
-"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1
-
-# Add Spack to the PATH.
-export PATH="$SPACK_ROOT/bin:$PATH"
 
 # Move to root directory of Spack
 # Allows script to be run from anywhere
 cd "$SPACK_ROOT"
 
-# Run integration tests
+# Print compiler information
-# TODO: should these be separated into a different test suite?
-source "$SPACK_ROOT/share/spack/setup-env.sh"
-spack compilers
 spack config get compilers
 
 # Run unit tests with code coverage
-py_ver=$(python -c 'import platform; print(platform.python_version())')
+${coverage_run} bin/spack test "$@"
-if [[ "$py_ver" == 2.7* || "$py_ver" == 3.6* ]];
+${coverage_combine}
-then
-    coverage run bin/spack install -v libdwarf
-    coverage run bin/spack test "$@" && coverage combine
-else
-    spack install -v libdwarf
-    spack test "$@"
-fi

share/spack/qa/setup.sh

@@ -0,0 +1,118 @@
+#!/bin/bash -e
+#
+# Description:
+#     Common setup code to be sourced by Spack's test scripts.
+#
+
+QA_DIR="$(dirname ${BASH_SOURCE[0]})"
+SPACK_ROOT="$QA_DIR/../../.."
+
+# Source the setup script
+. "$SPACK_ROOT/share/spack/setup-env.sh"
+
+# Set up some variables for running coverage tests.
+if [[ "$COVERAGE" == true ]]; then
+    coverage=coverage
+    coverage_run="coverage run"
+    coverage_combine="coverage combine"
+else
+    coverage=""
+    coverage_run=""
+    coverage_combine=""
+fi
+
+#
+# Description:
+#     Check to see if dependencies are installed.
+#     If not, warn the user and tell them how to
+#     install these dependencies.
+#
+# Usage:
+#     check-deps <dep> ...
+#
+# Options:
+#     One or more dependencies. Must use name of binary.
+check_dependencies() {
+    for dep in "$@"; do
+        if ! which $dep &> /dev/null; then
+            # Map binary name to package name
+            case $dep in
+                sphinx-apidoc|sphinx-build)
+                    spack_package=py-sphinx
+                    pip_package=sphinx
+                    ;;
+                coverage)
+                    spack_package=py-coverage
+                    pip_package=coverage
+                    ;;
+                flake8)
+                    spack_package=py-flake8
+                    pip_package=flake8
+                    ;;
+                dot)
+                    spack_package=graphviz
+                    ;;
+                git)
+                    spack_package=git
+                    ;;
+                hg)
+                    spack_package=mercurial
+                    pip_package=mercurial
+                    ;;
+                svn)
+                    spack_package=subversion
+                    ;;
+                *)
+                    spack_package=$dep
+                    pip_package=$dep
+                    ;;
+            esac
+
+            echo "ERROR: $dep is required to run this script."
+            echo
+
+            if [[ $spack_package ]]; then
+                echo "To install with Spack, run:"
+                echo "    $ spack install $spack_package"
+            fi
+
+            if [[ $pip_package ]]; then
+                echo "To install with pip, run:"
+                echo "    $ pip install $pip_package"
+            fi
+
+            if [[ $spack_package || $pip_package ]]; then
+                echo "Then add the bin directory to your PATH."
+            fi
+
+            exit 1
+        fi
+
+        # Flake8 and Sphinx require setuptools in order to run.
+        # Otherwise, they print out this error message:
+        #
+        #   Traceback (most recent call last):
+        #     File: "/usr/bin/flake8", line 5, in <module>
+        #       from pkg_resources import load_entry_point
+        #   ImportError: No module named pkg_resources
+        #
+        # Print a more useful error message if setuptools not found.
+        if [[ $dep == flake8 || $dep == sphinx* ]]; then
+            # Find which Python is being run
+            # Spack-installed packages have a hard-coded shebang
+            python_cmd=$(head -n 1 $(which $dep) | cut -c 3-)
+            # May not have a shebang
+            if [[ $python_cmd != *python* ]]; then
+                python_cmd=python
+            fi
+            # Check if setuptools is in the PYTHONPATH
+            if ! $python_cmd -c "import setuptools" 2> /dev/null; then
+                echo "ERROR: setuptools is required to run $dep."
+                echo "Please add it to your PYTHONPATH."
+
+                exit 1
+            fi
+        fi
+    done
+    echo "Dependencies found."
+}