Separate integration tests; simplify test scripts (#4006)
* Separate build integration tests; simplify test scripts - Move build tests out of the regular Travis unit tests, add more smoke test packages to build. - Run all test scripts with bash -e, which fails on error. - Factor coverage out into a Travis environment variable, so it's more obvious from .travis.yml which tests contribute to coverage and which don't. - Factor dependency checking and much of the front-matter in tests scripts into a setup.sh script, which is sourced by all the test scripts. Extra cruft in each tests script now reduced to 2 lines at the beginning.
This commit is contained in:
Todd Gamblin
GitHub
parent
a0ebce0cb3
commit
bb5a433a46
14
.travis.yml
14
.travis.yml
@ -21,7 +21,11 @@ matrix:
|
||||
- python: '2.7'
|
||||
os: linux
|
||||
language: python
|
||||
env: TEST_SUITE=unit
|
||||
env: [ TEST_SUITE=unit, COVERAGE=true ]
|
||||
- python: '2.7'
|
||||
os: linux
|
||||
language: python
|
||||
env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=hypre^mpich' ]
|
||||
- python: '3.3'
|
||||
os: linux
|
||||
language: python
|
||||
@ -37,7 +41,11 @@ matrix:
|
||||
- python: '3.6'
|
||||
os: linux
|
||||
language: python
|
||||
env: TEST_SUITE=unit
|
||||
env: [ TEST_SUITE=unit, COVERAGE=true ]
|
||||
- python: '3.6'
|
||||
os: linux
|
||||
language: python
|
||||
env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=hypre^mpich' ]
|
||||
- python: '2.7'
|
||||
os: linux
|
||||
language: python
|
||||
@ -48,7 +56,7 @@ matrix:
|
||||
env: TEST_SUITE=doc
|
||||
- os: osx
|
||||
language: generic
|
||||
env: [ TEST_SUITE=unit, PYTHON_VERSION=2.7 ]
|
||||
env: [ TEST_SUITE=unit, PYTHON_VERSION=2.7, COVERAGE=true ]
|
||||
|
||||
#=============================================================================
|
||||
# Environment
|
||||
|
||||
@ -1,96 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Description:
|
||||
# Check to see if dependencies are installed.
|
||||
# If not, warn the user and tell them how to
|
||||
# install these dependencies.
|
||||
#
|
||||
# Usage:
|
||||
# check-deps <dep> ...
|
||||
#
|
||||
# Options:
|
||||
# One or more dependencies. Must use name of binary.
|
||||
|
||||
for dep in "$@"; do
|
||||
if ! which $dep &> /dev/null; then
|
||||
# Map binary name to package name
|
||||
case $dep in
|
||||
sphinx-apidoc|sphinx-build)
|
||||
spack_package=py-sphinx
|
||||
pip_package=sphinx
|
||||
;;
|
||||
coverage)
|
||||
spack_package=py-coverage
|
||||
pip_package=coverage
|
||||
;;
|
||||
flake8)
|
||||
spack_package=py-flake8
|
||||
pip_package=flake8
|
||||
;;
|
||||
dot)
|
||||
spack_package=graphviz
|
||||
;;
|
||||
git)
|
||||
spack_package=git
|
||||
;;
|
||||
hg)
|
||||
spack_package=mercurial
|
||||
pip_package=mercurial
|
||||
;;
|
||||
svn)
|
||||
spack_package=subversion
|
||||
;;
|
||||
*)
|
||||
spack_package=$dep
|
||||
pip_package=$dep
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "ERROR: $dep is required to run this script."
|
||||
echo
|
||||
|
||||
if [[ $spack_package ]]; then
|
||||
echo "To install with Spack, run:"
|
||||
echo " $ spack install $spack_package"
|
||||
fi
|
||||
|
||||
if [[ $pip_package ]]; then
|
||||
echo "To install with pip, run:"
|
||||
echo " $ pip install $pip_package"
|
||||
fi
|
||||
|
||||
if [[ $spack_package || $pip_package ]]; then
|
||||
echo "Then add the bin directory to your PATH."
|
||||
fi
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Flake8 and Sphinx require setuptools in order to run.
|
||||
# Otherwise, they print out this error message:
|
||||
#
|
||||
# Traceback (most recent call last):
|
||||
# File: "/usr/bin/flake8", line 5, in <module>
|
||||
# from pkg_resources import load_entry_point
|
||||
# ImportError: No module named pkg_resources
|
||||
#
|
||||
# Print a more useful error message if setuptools not found.
|
||||
if [[ $dep == flake8 || $dep == sphinx* ]]; then
|
||||
# Find which Python is being run
|
||||
# Spack-installed packages have a hard-coded shebang
|
||||
python_cmd=$(head -n 1 $(which $dep) | cut -c 3-)
|
||||
# May not have a shebang
|
||||
if [[ $python_cmd != *python* ]]; then
|
||||
python_cmd=python
|
||||
fi
|
||||
# Check if setuptools is in the PYTHONPATH
|
||||
if ! $python_cmd -c "import setuptools" 2> /dev/null; then
|
||||
echo "ERROR: setuptools is required to run $dep."
|
||||
echo "Please add it to your PYTHONPATH."
|
||||
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Dependencies found."
|
||||
29
share/spack/qa/run-build-tests
Executable file
29
share/spack/qa/run-build-tests
Executable file
@ -0,0 +1,29 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Description:
|
||||
# Runs Spack build smoke tests. This installs a few packages that
|
||||
# cover different parts of the build system. It is not an exhaustive
|
||||
# test of Spack's packages.
|
||||
#
|
||||
# Usage:
|
||||
# run-build-tests
|
||||
#
|
||||
. "$(dirname $0)/setup.sh"
|
||||
check_dependencies ${coverage} git hg svn
|
||||
|
||||
# Move to root directory of Spack
|
||||
# Allows script to be run from anywhere
|
||||
cd "$SPACK_ROOT"
|
||||
|
||||
# Make sure we have a spec to build.
|
||||
if [ -z "$SPEC" ]; then
|
||||
echo "Error: run-build-tests requires the $SPEC to build to be set."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Print compiler information
|
||||
spack config get compilers
|
||||
|
||||
# Run some build smoke tests, potentially with code coverage
|
||||
${coverage_run} bin/spack install -v ${SPEC}
|
||||
${coverage_combine}
|
||||
@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Description:
|
||||
# Builds Spack documentation and checks for
|
||||
@ -8,33 +8,12 @@
|
||||
# Usage:
|
||||
# run-doc-tests
|
||||
#
|
||||
# Notes:
|
||||
# Requires sphinx, graphviz, git, mercurial, and subversion.
|
||||
#
|
||||
|
||||
QA_DIR="$(dirname "$0")"
|
||||
SPACK_ROOT="$QA_DIR/../../.."
|
||||
DOC_DIR="$SPACK_ROOT/lib/spack/docs"
|
||||
|
||||
# Array of dependencies
|
||||
deps=(
|
||||
sphinx-apidoc
|
||||
sphinx-build
|
||||
dot
|
||||
git
|
||||
hg
|
||||
svn
|
||||
)
|
||||
|
||||
# Check for dependencies
|
||||
"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1
|
||||
|
||||
# Add Spack to the PATH.
|
||||
export PATH="$SPACK_ROOT/bin:$PATH"
|
||||
. "$(dirname $0)/setup.sh"
|
||||
check_dependencies sphinx-apidoc sphinx-build dot git hg svn
|
||||
|
||||
# Move to documentation directory
|
||||
# Allows script to be run from anywhere
|
||||
cd "$DOC_DIR"
|
||||
cd "$SPACK_ROOT/lib/spack/docs"
|
||||
|
||||
# Treat warnings as fatal errors
|
||||
make clean --silent
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Description:
|
||||
# Runs source code style checks on Spack.
|
||||
@ -8,22 +8,7 @@
|
||||
# Usage:
|
||||
# run-flake8-tests
|
||||
#
|
||||
# Notes:
|
||||
# Requires flake8.
|
||||
#
|
||||
. "$(dirname $0)/setup.sh"
|
||||
check_dependencies flake8
|
||||
|
||||
QA_DIR="$(dirname "$0")"
|
||||
SPACK_ROOT="$QA_DIR/../../.."
|
||||
|
||||
# Array of dependencies
|
||||
deps=(
|
||||
flake8
|
||||
)
|
||||
|
||||
# Check for dependencies
|
||||
"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1
|
||||
|
||||
# Add Spack to the PATH.
|
||||
export PATH="$SPACK_ROOT/bin:$PATH"
|
||||
|
||||
exec spack flake8
|
||||
spack flake8
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Description:
|
||||
# Runs Spack unit tests.
|
||||
@ -10,44 +10,16 @@
|
||||
# Optionally add one or more unit tests
|
||||
# to only run these tests.
|
||||
#
|
||||
# Notes:
|
||||
# Requires coverage, git, mercurial, and subversion.
|
||||
#
|
||||
|
||||
QA_DIR="$(dirname "$0")"
|
||||
SPACK_ROOT="$QA_DIR/../../.."
|
||||
|
||||
# Array of dependencies
|
||||
deps=(
|
||||
coverage
|
||||
git
|
||||
hg
|
||||
svn
|
||||
)
|
||||
|
||||
# Check for dependencies
|
||||
"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1
|
||||
|
||||
# Add Spack to the PATH.
|
||||
export PATH="$SPACK_ROOT/bin:$PATH"
|
||||
. "$(dirname $0)/setup.sh"
|
||||
check_dependencies ${coverage} git hg svn
|
||||
|
||||
# Move to root directory of Spack
|
||||
# Allows script to be run from anywhere
|
||||
cd "$SPACK_ROOT"
|
||||
|
||||
# Run integration tests
|
||||
# TODO: should these be separated into a different test suite?
|
||||
source "$SPACK_ROOT/share/spack/setup-env.sh"
|
||||
spack compilers
|
||||
# Print compiler information
|
||||
spack config get compilers
|
||||
|
||||
# Run unit tests with code coverage
|
||||
py_ver=$(python -c 'import platform; print(platform.python_version())')
|
||||
if [[ "$py_ver" == 2.7* || "$py_ver" == 3.6* ]];
|
||||
then
|
||||
coverage run bin/spack install -v libdwarf
|
||||
coverage run bin/spack test "$@" && coverage combine
|
||||
else
|
||||
spack install -v libdwarf
|
||||
spack test "$@"
|
||||
fi
|
||||
${coverage_run} bin/spack test "$@"
|
||||
${coverage_combine}
|
||||
|
||||
118
share/spack/qa/setup.sh
Executable file
118
share/spack/qa/setup.sh
Executable file
@ -0,0 +1,118 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Description:
|
||||
# Common setup code to be sourced by Spack's test scripts.
|
||||
#
|
||||
|
||||
QA_DIR="$(dirname ${BASH_SOURCE[0]})"
|
||||
SPACK_ROOT="$QA_DIR/../../.."
|
||||
|
||||
# Source the setup script
|
||||
. "$SPACK_ROOT/share/spack/setup-env.sh"
|
||||
|
||||
# Set up some variables for running coverage tests.
|
||||
if [[ "$COVERAGE" == true ]]; then
|
||||
coverage=coverage
|
||||
coverage_run="coverage run"
|
||||
coverage_combine="coverage combine"
|
||||
else
|
||||
coverage=""
|
||||
coverage_run=""
|
||||
coverage_combine=""
|
||||
fi
|
||||
|
||||
#
|
||||
# Description:
|
||||
# Check to see if dependencies are installed.
|
||||
# If not, warn the user and tell them how to
|
||||
# install these dependencies.
|
||||
#
|
||||
# Usage:
|
||||
# check-deps <dep> ...
|
||||
#
|
||||
# Options:
|
||||
# One or more dependencies. Must use name of binary.
|
||||
check_dependencies() {
|
||||
for dep in "$@"; do
|
||||
if ! which $dep &> /dev/null; then
|
||||
# Map binary name to package name
|
||||
case $dep in
|
||||
sphinx-apidoc|sphinx-build)
|
||||
spack_package=py-sphinx
|
||||
pip_package=sphinx
|
||||
;;
|
||||
coverage)
|
||||
spack_package=py-coverage
|
||||
pip_package=coverage
|
||||
;;
|
||||
flake8)
|
||||
spack_package=py-flake8
|
||||
pip_package=flake8
|
||||
;;
|
||||
dot)
|
||||
spack_package=graphviz
|
||||
;;
|
||||
git)
|
||||
spack_package=git
|
||||
;;
|
||||
hg)
|
||||
spack_package=mercurial
|
||||
pip_package=mercurial
|
||||
;;
|
||||
svn)
|
||||
spack_package=subversion
|
||||
;;
|
||||
*)
|
||||
spack_package=$dep
|
||||
pip_package=$dep
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "ERROR: $dep is required to run this script."
|
||||
echo
|
||||
|
||||
if [[ $spack_package ]]; then
|
||||
echo "To install with Spack, run:"
|
||||
echo " $ spack install $spack_package"
|
||||
fi
|
||||
|
||||
if [[ $pip_package ]]; then
|
||||
echo "To install with pip, run:"
|
||||
echo " $ pip install $pip_package"
|
||||
fi
|
||||
|
||||
if [[ $spack_package || $pip_package ]]; then
|
||||
echo "Then add the bin directory to your PATH."
|
||||
fi
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Flake8 and Sphinx require setuptools in order to run.
|
||||
# Otherwise, they print out this error message:
|
||||
#
|
||||
# Traceback (most recent call last):
|
||||
# File: "/usr/bin/flake8", line 5, in <module>
|
||||
# from pkg_resources import load_entry_point
|
||||
# ImportError: No module named pkg_resources
|
||||
#
|
||||
# Print a more useful error message if setuptools not found.
|
||||
if [[ $dep == flake8 || $dep == sphinx* ]]; then
|
||||
# Find which Python is being run
|
||||
# Spack-installed packages have a hard-coded shebang
|
||||
python_cmd=$(head -n 1 $(which $dep) | cut -c 3-)
|
||||
# May not have a shebang
|
||||
if [[ $python_cmd != *python* ]]; then
|
||||
python_cmd=python
|
||||
fi
|
||||
# Check if setuptools is in the PYTHONPATH
|
||||
if ! $python_cmd -c "import setuptools" 2> /dev/null; then
|
||||
echo "ERROR: setuptools is required to run $dep."
|
||||
echo "Please add it to your PYTHONPATH."
|
||||
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
echo "Dependencies found."
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user