coverage: only upload to codecov once (#46385)

Historically, every PR, push, etc. to Spack generates a bunch of jobs, each of which
uploads its coverage report to codecov independently. This means that we get annoying
partial coverage numbers when only a few of the jobs have finished, and frequently
codecov is bad at understanding when to merge reports for a given PR. The numbers of the
site can be weird as a result.

This restructures our coverage handling so that we do all the merging ourselves and
upload exactly one report per GitHub actions workflow. In practice, that means that
every push to every PR will get exactly one coverage report and exactly one coverage
number reported. I think this will at least partially restore peoples' faith in what
codecov is telling them, and it might even make codecov handle Spack a bit better, since
this reduces the report burden by ~7x.

- [x] test and audit jobs now upload artifacts for coverage
- [x] add a new job that downloads artifacts and merges coverage reports together
- [x] set `paths` section of `pyproject.toml` so that cross-platform clone locations are merged
- [x] upload to codecov once, at the end of the workflow

Signed-off-by: Todd Gamblin <tgamblin@llnl.gov>

.github/workflows/audit.yaml

@@ -40,6 +40,8 @@ jobs:
       run: |
         python -m pip install --upgrade pywin32
     - name: Package audits (with coverage)
+      env:
+          COVERAGE_FILE: coverage/.coverage-audits-${{ matrix.system.os }}
       if: ${{ inputs.with_coverage == 'true' && runner.os != 'Windows' }}
       run: |
           . share/spack/setup-env.sh
@@ -47,27 +49,26 @@ jobs:
           coverage run $(which spack) audit configs
           coverage run $(which spack) -d audit externals
           coverage combine
-          coverage xml
     - name: Package audits (without coverage)
       if: ${{ inputs.with_coverage == 'false' && runner.os != 'Windows' }}
       run: |
-          . share/spack/setup-env.sh          
+          . share/spack/setup-env.sh
           spack -d audit packages
           spack -d audit configs
           spack -d audit externals
     - name: Package audits (without coverage)
       if: ${{ runner.os == 'Windows' }}
       run: |
-          . share/spack/setup-env.sh          
+          . share/spack/setup-env.sh
           spack -d audit packages
           ./share/spack/qa/validate_last_exit.ps1
           spack -d audit configs
           ./share/spack/qa/validate_last_exit.ps1
           spack -d audit externals
           ./share/spack/qa/validate_last_exit.ps1
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
-      if: ${{ inputs.with_coverage == 'true' }}
+      if: ${{ inputs.with_coverage == 'true' && runner.os != 'Windows' }}
       with:
-        flags: unittests,audits
+        name: coverage-audits-${{ matrix.system.os }}
-        token: ${{ secrets.CODECOV_TOKEN }}
+        path: coverage
-        verbose: true
+        include-hidden-files: true

.github/workflows/ci.yaml

@@ -84,8 +84,30 @@ jobs:
     needs: [ prechecks, changes ]
     uses: ./.github/workflows/unit_tests.yaml
     secrets: inherit
+  upload-coverage:
+    needs: [ unit-tests, prechecks ]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      with:
+        fetch-depth: 0
+    - name: Download coverage files
+      uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
+      with:
+        pattern: coverage-*
+        path: coverage
+        merge-multiple: true
+    - run: pip install --upgrade coverage
+    - run: ls -la coverage
+    - run: coverage combine -a coverage/.coverage*
+    - run: coverage xml
+    - name: "Upload coverage"
+      uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        verbose: true
   all:
-    needs: [ unit-tests, bootstrap ]
+    needs: [ upload-coverage, bootstrap ]
     runs-on: ubuntu-latest
     steps:
     - name: Success

.github/workflows/unit_tests.yaml

@@ -76,14 +76,15 @@ jobs:
           SPACK_PYTHON: python
           SPACK_TEST_PARALLEL: 2
           COVERAGE: true
+          COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }}
           UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }}
       run: |
           share/spack/qa/run-unit-tests
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
       with:
-        flags: unittests,linux,${{ matrix.concretizer }}
+        name: coverage-${{ matrix.os }}-python${{ matrix.python-version }}
-        token: ${{ secrets.CODECOV_TOKEN }}
+        path: coverage
-        verbose: true
+        include-hidden-files: true
   # Test shell integration
   shell:
     runs-on: ubuntu-latest
@@ -112,11 +113,11 @@ jobs:
           COVERAGE: true
       run: |
           share/spack/qa/run-shell-tests
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
       with:
-        flags: shelltests,linux
+        name: coverage-shell
-        token: ${{ secrets.CODECOV_TOKEN }}
+        path: coverage
-        verbose: true
+        include-hidden-files: true
 
   # Test RHEL8 UBI with platform Python. This job is run
   # only on PRs modifying core Spack
@@ -170,13 +171,14 @@ jobs:
     - name: Run unit tests (full suite with coverage)
       env:
           COVERAGE: true
+          COVERAGE_FILE: coverage/.coverage-clingo-cffi
       run: |
           share/spack/qa/run-unit-tests
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
       with:
-        flags: unittests,linux,clingo
+        name: coverage-clingo-cffi
-        token: ${{ secrets.CODECOV_TOKEN }}
+        path: coverage
-        verbose: true
+        include-hidden-files: true
   # Run unit tests on MacOS
   macos:
     runs-on: ${{ matrix.os }}
@@ -201,6 +203,7 @@ jobs:
     - name: Run unit tests
       env:
         SPACK_TEST_PARALLEL: 4
+        COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }}
       run: |
         git --version
         . .github/workflows/bin/setup_git.sh
@@ -209,11 +212,11 @@ jobs:
         $(which spack) solve zlib
         common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
         $(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
       with:
-        flags: unittests,macos
+        name: coverage-${{ matrix.os }}-python${{ matrix.python-version }}
-        token: ${{ secrets.CODECOV_TOKEN }}
+        path: coverage
-        verbose: true
+        include-hidden-files: true
   # Run unit tests on Windows
   windows:
     defaults:
@@ -235,13 +238,13 @@ jobs:
       run: |
         ./.github/workflows/bin/setup_git.ps1
     - name: Unit Test
+      env:
+        COVERAGE_FILE: coverage/.coverage-windows
       run: |
         spack unit-test -x --verbose --cov --cov-config=pyproject.toml
         ./share/spack/qa/validate_last_exit.ps1
-        coverage combine -a
+    - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
-        coverage xml
-    - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
       with:
-        flags: unittests,windows
+        name: coverage-windows
-        token: ${{ secrets.CODECOV_TOKEN }}
+        path: coverage
-        verbose: true
+        include-hidden-files: true

pyproject.toml

@@ -212,6 +212,14 @@ exclude_lines = [
 ]
 ignore_errors = true
 
+[tool.coverage.paths]
+source = [
+    ".",
+    "/Users/runner/work/spack/spack",
+    "/System/Volumes/Data/home/runner/work/spack/spack",
+    "D:\\a\\spack\\spack",
+]
+
 [tool.coverage.html]
 directory = "htmlcov"