
Historically, every PR, push, etc. to Spack generates a bunch of jobs, each of which uploads its coverage report to codecov independently. This means that we get annoying partial coverage numbers when only a few of the jobs have finished, and frequently codecov is bad at understanding when to merge reports for a given PR. The numbers of the site can be weird as a result. This restructures our coverage handling so that we do all the merging ourselves and upload exactly one report per GitHub actions workflow. In practice, that means that every push to every PR will get exactly one coverage report and exactly one coverage number reported. I think this will at least partially restore peoples' faith in what codecov is telling them, and it might even make codecov handle Spack a bit better, since this reduces the report burden by ~7x. - [x] test and audit jobs now upload artifacts for coverage - [x] add a new job that downloads artifacts and merges coverage reports together - [x] set `paths` section of `pyproject.toml` so that cross-platform clone locations are merged - [x] upload to codecov once, at the end of the workflow Signed-off-by: Todd Gamblin <tgamblin@llnl.gov>
251 lines
8.6 KiB
YAML
251 lines
8.6 KiB
YAML
name: unit tests
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
workflow_call:
|
|
|
|
concurrency:
|
|
group: unit_tests-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
# Run unit tests with different configurations on linux
|
|
ubuntu:
|
|
runs-on: ${{ matrix.os }}
|
|
strategy:
|
|
matrix:
|
|
os: [ubuntu-latest]
|
|
python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
|
|
on_develop:
|
|
- ${{ github.ref == 'refs/heads/develop' }}
|
|
include:
|
|
- python-version: '3.6'
|
|
os: ubuntu-20.04
|
|
on_develop: ${{ github.ref == 'refs/heads/develop' }}
|
|
exclude:
|
|
- python-version: '3.7'
|
|
os: ubuntu-latest
|
|
on_develop: false
|
|
- python-version: '3.8'
|
|
os: ubuntu-latest
|
|
on_develop: false
|
|
- python-version: '3.9'
|
|
os: ubuntu-latest
|
|
on_develop: false
|
|
- python-version: '3.10'
|
|
os: ubuntu-latest
|
|
on_develop: false
|
|
- python-version: '3.11'
|
|
os: ubuntu-latest
|
|
on_develop: false
|
|
|
|
steps:
|
|
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
|
|
with:
|
|
fetch-depth: 0
|
|
- uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
- name: Install System packages
|
|
run: |
|
|
sudo apt-get -y update
|
|
# Needed for unit tests
|
|
sudo apt-get -y install \
|
|
coreutils cvs gfortran graphviz gnupg2 mercurial ninja-build \
|
|
cmake bison libbison-dev kcov
|
|
- name: Install Python packages
|
|
run: |
|
|
pip install --upgrade pip setuptools pytest pytest-xdist pytest-cov
|
|
pip install --upgrade flake8 "isort>=4.3.5" "mypy>=0.900" "click" "black"
|
|
- name: Setup git configuration
|
|
run: |
|
|
# Need this for the git tests to succeed.
|
|
git --version
|
|
. .github/workflows/bin/setup_git.sh
|
|
- name: Bootstrap clingo
|
|
if: ${{ matrix.concretizer == 'clingo' }}
|
|
env:
|
|
SPACK_PYTHON: python
|
|
run: |
|
|
. share/spack/setup-env.sh
|
|
spack bootstrap disable spack-install
|
|
spack bootstrap now
|
|
spack -v solve zlib
|
|
- name: Run unit tests
|
|
env:
|
|
SPACK_PYTHON: python
|
|
SPACK_TEST_PARALLEL: 2
|
|
COVERAGE: true
|
|
COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }}
|
|
UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }}
|
|
run: |
|
|
share/spack/qa/run-unit-tests
|
|
- uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
|
|
with:
|
|
name: coverage-${{ matrix.os }}-python${{ matrix.python-version }}
|
|
path: coverage
|
|
include-hidden-files: true
|
|
# Test shell integration
|
|
shell:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
|
|
with:
|
|
fetch-depth: 0
|
|
- uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
|
|
with:
|
|
python-version: '3.11'
|
|
- name: Install System packages
|
|
run: |
|
|
sudo apt-get -y update
|
|
# Needed for shell tests
|
|
sudo apt-get install -y coreutils kcov csh zsh tcsh fish dash bash
|
|
- name: Install Python packages
|
|
run: |
|
|
pip install --upgrade pip setuptools pytest coverage[toml] pytest-xdist
|
|
- name: Setup git configuration
|
|
run: |
|
|
# Need this for the git tests to succeed.
|
|
git --version
|
|
. .github/workflows/bin/setup_git.sh
|
|
- name: Run shell tests
|
|
env:
|
|
COVERAGE: true
|
|
run: |
|
|
share/spack/qa/run-shell-tests
|
|
- uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
|
|
with:
|
|
name: coverage-shell
|
|
path: coverage
|
|
include-hidden-files: true
|
|
|
|
# Test RHEL8 UBI with platform Python. This job is run
|
|
# only on PRs modifying core Spack
|
|
rhel8-platform-python:
|
|
runs-on: ubuntu-latest
|
|
container: registry.access.redhat.com/ubi8/ubi
|
|
steps:
|
|
- name: Install dependencies
|
|
run: |
|
|
dnf install -y \
|
|
bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
|
|
make patch tcl unzip which xz
|
|
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
|
|
- name: Setup repo and non-root user
|
|
run: |
|
|
git --version
|
|
git config --global --add safe.directory /__w/spack/spack
|
|
git fetch --unshallow
|
|
. .github/workflows/bin/setup_git.sh
|
|
useradd spack-test
|
|
chown -R spack-test .
|
|
- name: Run unit tests
|
|
shell: runuser -u spack-test -- bash {0}
|
|
run: |
|
|
source share/spack/setup-env.sh
|
|
spack -d bootstrap now --dev
|
|
spack unit-test -k 'not cvs and not svn and not hg' -x --verbose
|
|
# Test for the clingo based solver (using clingo-cffi)
|
|
clingo-cffi:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
|
|
with:
|
|
fetch-depth: 0
|
|
- uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
|
|
with:
|
|
python-version: '3.11'
|
|
- name: Install System packages
|
|
run: |
|
|
sudo apt-get -y update
|
|
sudo apt-get -y install coreutils cvs gfortran graphviz gnupg2 mercurial ninja-build kcov
|
|
- name: Install Python packages
|
|
run: |
|
|
pip install --upgrade pip setuptools pytest coverage[toml] pytest-cov clingo pytest-xdist
|
|
pip install --upgrade flake8 "isort>=4.3.5" "mypy>=0.900" "click" "black"
|
|
- name: Setup git configuration
|
|
run: |
|
|
# Need this for the git tests to succeed.
|
|
git --version
|
|
. .github/workflows/bin/setup_git.sh
|
|
- name: Run unit tests (full suite with coverage)
|
|
env:
|
|
COVERAGE: true
|
|
COVERAGE_FILE: coverage/.coverage-clingo-cffi
|
|
run: |
|
|
share/spack/qa/run-unit-tests
|
|
- uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
|
|
with:
|
|
name: coverage-clingo-cffi
|
|
path: coverage
|
|
include-hidden-files: true
|
|
# Run unit tests on MacOS
|
|
macos:
|
|
runs-on: ${{ matrix.os }}
|
|
strategy:
|
|
matrix:
|
|
os: [macos-13, macos-14]
|
|
python-version: ["3.11"]
|
|
steps:
|
|
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
|
|
with:
|
|
fetch-depth: 0
|
|
- uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
- name: Install Python packages
|
|
run: |
|
|
pip install --upgrade pip setuptools
|
|
pip install --upgrade pytest coverage[toml] pytest-xdist pytest-cov
|
|
- name: Setup Homebrew packages
|
|
run: |
|
|
brew install dash fish gcc gnupg2 kcov
|
|
- name: Run unit tests
|
|
env:
|
|
SPACK_TEST_PARALLEL: 4
|
|
COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }}
|
|
run: |
|
|
git --version
|
|
. .github/workflows/bin/setup_git.sh
|
|
. share/spack/setup-env.sh
|
|
$(which spack) bootstrap disable spack-install
|
|
$(which spack) solve zlib
|
|
common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
|
|
$(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
|
|
- uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
|
|
with:
|
|
name: coverage-${{ matrix.os }}-python${{ matrix.python-version }}
|
|
path: coverage
|
|
include-hidden-files: true
|
|
# Run unit tests on Windows
|
|
windows:
|
|
defaults:
|
|
run:
|
|
shell:
|
|
powershell Invoke-Expression -Command "./share/spack/qa/windows_test_setup.ps1"; {0}
|
|
runs-on: windows-latest
|
|
steps:
|
|
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
|
|
with:
|
|
fetch-depth: 0
|
|
- uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
|
|
with:
|
|
python-version: 3.9
|
|
- name: Install Python packages
|
|
run: |
|
|
python -m pip install --upgrade pip pywin32 setuptools pytest-cov clingo
|
|
- name: Create local develop
|
|
run: |
|
|
./.github/workflows/bin/setup_git.ps1
|
|
- name: Unit Test
|
|
env:
|
|
COVERAGE_FILE: coverage/.coverage-windows
|
|
run: |
|
|
spack unit-test -x --verbose --cov --cov-config=pyproject.toml
|
|
./share/spack/qa/validate_last_exit.ps1
|
|
- uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
|
|
with:
|
|
name: coverage-windows
|
|
path: coverage
|
|
include-hidden-files: true
|