zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci: add automatic checksum verification check (#45063)

Browse Source 
Add a CI check to automatically verify the checksums of newly added
package versions:
    - [x] a new command, `spack ci verify-versions`
    - [x] a GitHub actions check to run the command
    - [x] tests for the new command

This also eliminates the suggestion for maintainers to manually verify added
checksums in the case of accidental version <--> checksum mismatches.

----

Signed-off-by: Todd Gamblin <tgamblin@llnl.gov>
This commit is contained in:
Alec Scott 2025-03-20 17:58:14 -04:00 committed by GitHub
parent 38d77570b4
commit c79b6207e8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 728 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.github/workflows/ci.yaml vendored
View File

@ -76,10 +76,11 @@ jobs:
prechecks:
needs: [ changes ]
uses: ./.github/workflows/valid-style.yml
uses: ./.github/workflows/prechecks.yml
secrets: inherit
with:
with_coverage: ${{ needs.changes.outputs.core }}
with_packages: ${{ needs.changes.outputs.packages }}
import-check:
needs: [ changes ]
@ -93,7 +94,7 @@ jobs:
- name: Success
run: |
if [ "${{ needs.prechecks.result }}" == "failure" ] || [ "${{ needs.prechecks.result }}" == "canceled" ]; then
echo "Unit tests failed."
echo "Unit tests failed."
exit 1
else
exit 0
@ -101,6 +102,7 @@ jobs:
coverage:
needs: [ unit-tests, prechecks ]
if: ${{ needs.changes.outputs.core }}
uses: ./.github/workflows/coverage.yml
secrets: inherit
@ -113,10 +115,10 @@ jobs:
- name: Status summary
run: |
if [ "${{ needs.unit-tests.result }}" == "failure" ] || [ "${{ needs.unit-tests.result }}" == "canceled" ]; then
echo "Unit tests failed."
echo "Unit tests failed."
exit 1
elif [ "${{ needs.bootstrap.result }}" == "failure" ] || [ "${{ needs.bootstrap.result }}" == "canceled" ]; then
echo "Bootstrap tests failed."
echo "Bootstrap tests failed."
exit 1
else
exit 0

19
.github/workflows/valid-style.yml → .github/workflows/prechecks.yml vendored
View File

@ -1,4 +1,4 @@
name: style
name: prechecks
on:
workflow_call:
@ -6,6 +6,9 @@ on:
with_coverage:
required: true
type: string
with_packages:
required: true
type: string
concurrency:
group: style-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
@ -30,6 +33,7 @@ jobs:
run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
- name: vermin (Repositories)
run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv var/spack/repos
# Run style checks on the files that have been changed
style:
runs-on: ubuntu-latest
@ -53,12 +57,25 @@ jobs:
- name: Run style tests
run: |
share/spack/qa/run-style-tests
audit:
uses: ./.github/workflows/audit.yaml
secrets: inherit
with:
with_coverage: ${{ inputs.with_coverage }}
python_version: '3.13'
verify-checksums:
if: ${{ inputs.with_packages == 'true' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
with:
fetch-depth: 2
- name: Verify Added Checksums
run: |
bin/spack ci verify-versions HEAD^1 HEAD
# Check that spack can bootstrap the development environment on Python 3.6 - RHEL8
bootstrap-dev-rhel8:
runs-on: ubuntu-latest

42
lib/spack/spack/ci/__init__.py
View File

@ -14,7 +14,7 @@
import tempfile
import zipfile
from collections import namedtuple
from typing import Callable, Dict, List, Set
from typing import Callable, Dict, List, Set, Union
from urllib.request import Request
import llnl.path
@ -42,6 +42,7 @@
from spack import traverse
from spack.error import SpackError
from spack.reporters.cdash import SPACK_CDASH_TIMEOUT
from spack.version import GitVersion, StandardVersion
from .common import (
IS_WINDOWS,
@ -80,6 +81,45 @@ def get_change_revisions():
return None, None
def get_added_versions(
checksums_version_dict: Dict[str, Union[StandardVersion, GitVersion]],
path: str,
from_ref: str = "HEAD~1",
to_ref: str = "HEAD",
) -> List[Union[StandardVersion, GitVersion]]:
"""Get a list of the versions added between `from_ref` and `to_ref`.
Args:
checksums_version_dict (Dict): all package versions keyed by known checksums.
path (str): path to the package.py
from_ref (str): oldest git ref, defaults to `HEAD~1`
to_ref (str): newer git ref, defaults to `HEAD`
Returns: list of versions added between refs
"""
git_exe = spack.util.git.git(required=True)
# Gather git diff
diff_lines = git_exe("diff", from_ref, to_ref, "--", path, output=str).split("\n")
# Store added and removed versions
# Removed versions are tracked here to determine when versions are moved in a file
# and show up as both added and removed in a git diff.
added_checksums = set()
removed_checksums = set()
# Scrape diff for modified versions and prune added versions if they show up
# as also removed (which means they've actually just moved in the file and
# we shouldn't need to rechecksum them)
for checksum in checksums_version_dict.keys():
for line in diff_lines:
if checksum in line:
if line.startswith("+"):
added_checksums.add(checksum)
if line.startswith("-"):
removed_checksums.add(checksum)
return [checksums_version_dict[c] for c in added_checksums - removed_checksums]
def get_stack_changed(env_path, rev1="HEAD^", rev2="HEAD"):
"""Given an environment manifest path and two revisions to compare, return
whether or not the stack was changed. Returns True if the environment

179
lib/spack/spack/cmd/ci.py
View File

@ -4,12 +4,15 @@
import json
import os
import re
import shutil
import sys
from typing import Dict
from urllib.parse import urlparse, urlunparse
import llnl.util.filesystem as fs
import llnl.util.tty as tty
import llnl.util.tty.color as clr
from llnl.util import tty
import spack.binary_distribution as bindist
import spack.ci as spack_ci
@ -18,12 +21,22 @@
import spack.cmd.common.arguments
import spack.config as cfg
import spack.environment as ev
import spack.error
import spack.fetch_strategy
import spack.hash_types as ht
import spack.mirrors.mirror
import spack.package_base
import spack.paths
import spack.repo
import spack.spec
import spack.stage
import spack.util.executable
import spack.util.git
import spack.util.gpg as gpg_util
import spack.util.timer as timer
import spack.util.url as url_util
import spack.util.web as web_util
import spack.version
description = "manage continuous integration pipelines"
section = "build"
@ -32,6 +45,7 @@
SPACK_COMMAND = "spack"
INSTALL_FAIL_CODE = 1
FAILED_CREATE_BUILDCACHE_CODE = 100
BUILTIN = re.compile(r"var\/spack\/repos\/builtin\/packages\/([^\/]+)\/package\.py")
def deindent(desc):
@ -191,6 +205,16 @@ def setup_parser(subparser):
reproduce.set_defaults(func=ci_reproduce)
# Verify checksums inside of ci workflows
verify_versions = subparsers.add_parser(
"verify-versions",
description=deindent(ci_verify_versions.__doc__),
help=spack.cmd.first_line(ci_verify_versions.__doc__),
)
verify_versions.add_argument("from_ref", help="git ref from which start looking at changes")
verify_versions.add_argument("to_ref", help="git ref to end looking at changes")
verify_versions.set_defaults(func=ci_verify_versions)
def ci_generate(args):
"""generate jobs file from a CI-aware spack file
@ -659,6 +683,159 @@ def _gitlab_artifacts_url(url: str) -> str:
return urlunparse(parsed._replace(path="/".join(parts), fragment="", query=""))
def validate_standard_versions(
pkg: spack.package_base.PackageBase, versions: spack.version.VersionList
) -> bool:
"""Get and test the checksum of a package version based on a tarball.
Args:
pkg spack.package_base.PackageBase: Spack package for which to validate a version checksum
versions spack.version.VersionList: list of package versions to validate
Returns: bool: result of the validation. True is valid and false is failed.
"""
url_dict: Dict[spack.version.StandardVersion, str] = {}
for version in versions:
url = pkg.find_valid_url_for_version(version)
url_dict[version] = url
version_hashes = spack.stage.get_checksums_for_versions(
url_dict, pkg.name, fetch_options=pkg.fetch_options
)
valid_checksums = True
for version, sha in version_hashes.items():
if sha != pkg.versions[version]["sha256"]:
tty.error(
f"Invalid checksum found {pkg.name}@{version}\n"
f" [package.py] {pkg.versions[version]['sha256']}\n"
f" [Downloaded] {sha}"
)
valid_checksums = False
continue
tty.info(f"Validated {pkg.name}@{version} --> {sha}")
return valid_checksums
def validate_git_versions(
pkg: spack.package_base.PackageBase, versions: spack.version.VersionList
) -> bool:
"""Get and test the commit and tag of a package version based on a git repository.
Args:
pkg spack.package_base.PackageBase: Spack package for which to validate a version
versions spack.version.VersionList: list of package versions to validate
Returns: bool: result of the validation. True is valid and false is failed.
"""
valid_commit = True
for version in versions:
fetcher = spack.fetch_strategy.for_package_version(pkg, version)
with spack.stage.Stage(fetcher) as stage:
known_commit = pkg.versions[version]["commit"]
try:
stage.fetch()
except spack.error.FetchError:
tty.error(
f"Invalid commit for {pkg.name}@{version}\n"
f" {known_commit} could not be checked out in the git repository."
)
valid_commit = False
continue
# Test if the specified tag matches the commit in the package.py
# We retrieve the commit associated with a tag and compare it to the
# commit that is located in the package.py file.
if "tag" in pkg.versions[version]:
tag = pkg.versions[version]["tag"]
try:
with fs.working_dir(stage.source_path):
found_commit = fetcher.git(
"rev-list", "-n", "1", tag, output=str, error=str
).strip()
except spack.util.executable.ProcessError:
tty.error(
f"Invalid tag for {pkg.name}@{version}\n"
f" {tag} could not be found in the git repository."
)
valid_commit = False
continue
if found_commit != known_commit:
tty.error(
f"Mismatched tag <-> commit found for {pkg.name}@{version}\n"
f" [package.py] {known_commit}\n"
f" [Downloaded] {found_commit}"
)
valid_commit = False
continue
# If we have downloaded the repository, found the commit, and compared
# the tag (if specified) we can conclude that the version is pointing
# at what we would expect.
tty.info(f"Validated {pkg.name}@{version} --> {known_commit}")
return valid_commit
def ci_verify_versions(args):
"""validate version checksum & commits between git refs
This command takes a from_ref and to_ref arguments and
then parses the git diff between the two to determine which packages
have been modified verifies the new checksums inside of them.
"""
with fs.working_dir(spack.paths.prefix):
# We use HEAD^1 explicitly on the merge commit created by
# GitHub Actions. However HEAD~1 is a safer default for the helper function.
files = spack.util.git.get_modified_files(from_ref=args.from_ref, to_ref=args.to_ref)
# Get a list of package names from the modified files.
pkgs = [(m.group(1), p) for p in files for m in [BUILTIN.search(p)] if m]
failed_version = False
for pkg_name, path in pkgs:
spec = spack.spec.Spec(pkg_name)
pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)
# Skip checking manual download packages and trust the maintainers
if pkg.manual_download:
tty.warn(f"Skipping manual download package: {pkg_name}")
continue
# Store versions checksums / commits for future loop
checksums_version_dict = {}
commits_version_dict = {}
for version in pkg.versions:
# If the package version defines a sha256 we'll use that as the high entropy
# string to detect which versions have been added between from_ref and to_ref
if "sha256" in pkg.versions[version]:
checksums_version_dict[pkg.versions[version]["sha256"]] = version
# If a package version instead defines a commit we'll use that as a
# high entropy string to detect new versions.
elif "commit" in pkg.versions[version]:
commits_version_dict[pkg.versions[version]["commit"]] = version
# TODO: enforce every version have a commit or a sha256 defined if not
# an infinite version (there are a lot of package's where this doesn't work yet.)
with fs.working_dir(spack.paths.prefix):
added_checksums = spack_ci.get_added_versions(
checksums_version_dict, path, from_ref=args.from_ref, to_ref=args.to_ref
)
added_commits = spack_ci.get_added_versions(
commits_version_dict, path, from_ref=args.from_ref, to_ref=args.to_ref
)
if added_checksums:
failed_version = not validate_standard_versions(pkg, added_checksums) or failed_version
if added_commits:
failed_version = not validate_git_versions(pkg, added_commits) or failed_version
if failed_version:
sys.exit(1)
def ci(parser, args):
if args.func:
return args.func(args)

38
lib/spack/spack/test/ci.py
View File

@ -18,6 +18,7 @@
import spack.repo as repo
import spack.util.git
from spack.test.conftest import MockHTTPResponse
from spack.version import Version
pytestmark = [pytest.mark.usefixtures("mock_packages")]
@ -30,6 +31,43 @@ def repro_dir(tmp_path):
yield result
def test_get_added_versions_new_checksum(mock_git_package_changes):
repo_path, filename, commits = mock_git_package_changes
checksum_versions = {
"3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04": Version("2.1.5"),
"a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a": Version("2.1.4"),
"6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200": Version("2.0.7"),
"86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8": Version("2.0.0"),
}
with fs.working_dir(str(repo_path)):
added_versions = ci.get_added_versions(
checksum_versions, filename, from_ref=commits[-1], to_ref=commits[-2]
)
assert len(added_versions) == 1
assert added_versions[0] == Version("2.1.5")
def test_get_added_versions_new_commit(mock_git_package_changes):
repo_path, filename, commits = mock_git_package_changes
checksum_versions = {
"74253725f884e2424a0dd8ae3f69896d5377f325": Version("2.1.6"),
"3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04": Version("2.1.5"),
"a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a": Version("2.1.4"),
"6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200": Version("2.0.7"),
"86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8": Version("2.0.0"),
}
with fs.working_dir(str(repo_path)):
added_versions = ci.get_added_versions(
checksum_versions, filename, from_ref=commits[2], to_ref=commits[1]
)
assert len(added_versions) == 1
assert added_versions[0] == Version("2.1.6")
def test_pipeline_dag(config, tmpdir):
r"""Test creation, pruning, and traversal of PipelineDAG using the
following package dependency graph:

217
lib/spack/spack/test/cmd/ci.py
View File

@ -22,7 +22,11 @@
import spack.hash_types as ht
import spack.main
import spack.paths as spack_paths
import spack.repo
import spack.spec
import spack.stage
import spack.util.spack_yaml as syaml
import spack.version
from spack.ci import gitlab as gitlab_generator
from spack.ci.common import PipelineDag, PipelineOptions, SpackCIConfig
from spack.ci.generator_registry import generator
@ -1841,3 +1845,216 @@ def test_ci_generate_alternate_target(
assert pipeline_doc.startswith("unittestpipeline")
assert "externaltest" in pipeline_doc
@pytest.fixture
def fetch_versions_match(monkeypatch):
"""Fake successful checksums returned from downloaded tarballs."""
def get_checksums_for_versions(url_by_version, package_name, **kwargs):
pkg_cls = spack.repo.PATH.get_pkg_class(package_name)
return {v: pkg_cls.versions[v]["sha256"] for v in url_by_version}
monkeypatch.setattr(spack.stage, "get_checksums_for_versions", get_checksums_for_versions)
@pytest.fixture
def fetch_versions_invalid(monkeypatch):
"""Fake successful checksums returned from downloaded tarballs."""
def get_checksums_for_versions(url_by_version, package_name, **kwargs):
return {
v: "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"
for v in url_by_version
}
monkeypatch.setattr(spack.stage, "get_checksums_for_versions", get_checksums_for_versions)
@pytest.mark.parametrize("versions", [["2.1.4"], ["2.1.4", "2.1.5"]])
def test_ci_validate_standard_versions_valid(capfd, mock_packages, fetch_versions_match, versions):
spec = spack.spec.Spec("diff-test")
pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)
version_list = [spack.version.Version(v) for v in versions]
assert spack.cmd.ci.validate_standard_versions(pkg, version_list)
out, err = capfd.readouterr()
for version in versions:
assert f"Validated diff-test@{version}" in out
@pytest.mark.parametrize("versions", [["2.1.4"], ["2.1.4", "2.1.5"]])
def test_ci_validate_standard_versions_invalid(
capfd, mock_packages, fetch_versions_invalid, versions
):
spec = spack.spec.Spec("diff-test")
pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)
version_list = [spack.version.Version(v) for v in versions]
assert spack.cmd.ci.validate_standard_versions(pkg, version_list) is False
out, err = capfd.readouterr()
for version in versions:
assert f"Invalid checksum found diff-test@{version}" in err
@pytest.mark.parametrize("versions", [[("1.0", -2)], [("1.1", -4), ("2.0", -6)]])
def test_ci_validate_git_versions_valid(
capfd, monkeypatch, mock_packages, mock_git_version_info, versions
):
spec = spack.spec.Spec("diff-test")
pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)
version_list = [spack.version.Version(v) for v, _ in versions]
repo_path, filename, commits = mock_git_version_info
version_commit_dict = {
spack.version.Version(v): {"tag": f"v{v}", "commit": commits[c]} for v, c in versions
}
pkg_class = spec.package_class
monkeypatch.setattr(pkg_class, "git", repo_path)
monkeypatch.setattr(pkg_class, "versions", version_commit_dict)
assert spack.cmd.ci.validate_git_versions(pkg, version_list)
out, err = capfd.readouterr()
for version in version_list:
assert f"Validated diff-test@{version}" in out
@pytest.mark.parametrize("versions", [[("1.0", -3)], [("1.1", -5), ("2.0", -5)]])
def test_ci_validate_git_versions_bad_tag(
capfd, monkeypatch, mock_packages, mock_git_version_info, versions
):
spec = spack.spec.Spec("diff-test")
pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)
version_list = [spack.version.Version(v) for v, _ in versions]
repo_path, filename, commits = mock_git_version_info
version_commit_dict = {
spack.version.Version(v): {"tag": f"v{v}", "commit": commits[c]} for v, c in versions
}
pkg_class = spec.package_class
monkeypatch.setattr(pkg_class, "git", repo_path)
monkeypatch.setattr(pkg_class, "versions", version_commit_dict)
assert spack.cmd.ci.validate_git_versions(pkg, version_list) is False
out, err = capfd.readouterr()
for version in version_list:
assert f"Mismatched tag <-> commit found for diff-test@{version}" in err
@pytest.mark.parametrize("versions", [[("1.0", -2)], [("1.1", -4), ("2.0", -6), ("3.0", -6)]])
def test_ci_validate_git_versions_invalid(
capfd, monkeypatch, mock_packages, mock_git_version_info, versions
):
spec = spack.spec.Spec("diff-test")
pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec)
version_list = [spack.version.Version(v) for v, _ in versions]
repo_path, filename, commits = mock_git_version_info
version_commit_dict = {
spack.version.Version(v): {
"tag": f"v{v}",
"commit": "abcdefabcdefabcdefabcdefabcdefabcdefabc",
}
for v, c in versions
}
pkg_class = spec.package_class
monkeypatch.setattr(pkg_class, "git", repo_path)
monkeypatch.setattr(pkg_class, "versions", version_commit_dict)
assert spack.cmd.ci.validate_git_versions(pkg, version_list) is False
out, err = capfd.readouterr()
for version in version_list:
assert f"Invalid commit for diff-test@{version}" in err
@pytest.fixture
def verify_standard_versions_valid(monkeypatch):
def validate_standard_versions(pkg, versions):
for version in versions:
print(f"Validated {pkg.name}@{version}")
return True
monkeypatch.setattr(spack.cmd.ci, "validate_standard_versions", validate_standard_versions)
@pytest.fixture
def verify_git_versions_valid(monkeypatch):
def validate_git_versions(pkg, versions):
for version in versions:
print(f"Validated {pkg.name}@{version}")
return True
monkeypatch.setattr(spack.cmd.ci, "validate_git_versions", validate_git_versions)
@pytest.fixture
def verify_standard_versions_invalid(monkeypatch):
def validate_standard_versions(pkg, versions):
for version in versions:
print(f"Invalid checksum found {pkg.name}@{version}")
return False
monkeypatch.setattr(spack.cmd.ci, "validate_standard_versions", validate_standard_versions)
@pytest.fixture
def verify_git_versions_invalid(monkeypatch):
def validate_git_versions(pkg, versions):
for version in versions:
print(f"Invalid commit for {pkg.name}@{version}")
return False
monkeypatch.setattr(spack.cmd.ci, "validate_git_versions", validate_git_versions)
def test_ci_verify_versions_valid(
monkeypatch,
mock_packages,
mock_git_package_changes,
verify_standard_versions_valid,
verify_git_versions_valid,
):
repo_path, _, commits = mock_git_package_changes
monkeypatch.setattr(spack.paths, "prefix", repo_path)
out = ci_cmd("verify-versions", commits[-1], commits[-3])
assert "Validated diff-test@2.1.5" in out
assert "Validated diff-test@2.1.6" in out
def test_ci_verify_versions_standard_invalid(
monkeypatch,
mock_packages,
mock_git_package_changes,
verify_standard_versions_invalid,
verify_git_versions_invalid,
):
repo_path, _, commits = mock_git_package_changes
monkeypatch.setattr(spack.paths, "prefix", repo_path)
out = ci_cmd("verify-versions", commits[-1], commits[-3], fail_on_error=False)
assert "Invalid checksum found diff-test@2.1.5" in out
assert "Invalid commit for diff-test@2.1.6" in out
def test_ci_verify_versions_manual_package(monkeypatch, mock_packages, mock_git_package_changes):
repo_path, _, commits = mock_git_package_changes
monkeypatch.setattr(spack.paths, "prefix", repo_path)
pkg_class = spack.spec.Spec("diff-test").package_class
monkeypatch.setattr(pkg_class, "manual_download", True)
out = ci_cmd("verify-versions", commits[-1], commits[-2])
assert "Skipping manual download package: diff-test" in out

80
lib/spack/spack/test/conftest.py
View File

@ -161,7 +161,7 @@ def mock_git_version_info(git, tmpdir, override_git_repos_cache_path):
version tags on multiple branches, and version order is not equal to time
order or topological order.
"""
repo_path = str(tmpdir.mkdir("git_repo"))
repo_path = str(tmpdir.mkdir("git_version_info_repo"))
filename = "file.txt"
def commit(message):
@ -242,6 +242,84 @@ def latest_commit():
yield repo_path, filename, commits
@pytest.fixture
def mock_git_package_changes(git, tmpdir, override_git_repos_cache_path):
"""Create a mock git repo with known structure of package edits
The structure of commits in this repo is as follows::
o diff-test: modification to make manual download package
|
o diff-test: add v1.2 (from a git ref)
|
o diff-test: add v1.1 (from source tarball)
|
o diff-test: new package (testing multiple added versions)
The repo consists of a single package.py file for DiffTest.
Important attributes of the repo for test coverage are: multiple package
versions are added with some coming from a tarball and some from git refs.
"""
repo_path = str(tmpdir.mkdir("git_package_changes_repo"))
filename = "var/spack/repos/builtin/packages/diff-test/package.py"
def commit(message):
global commit_counter
git(
"commit",
"--no-gpg-sign",
"--date",
"2020-01-%02d 12:0:00 +0300" % commit_counter,
"-am",
message,
)
commit_counter += 1
with working_dir(repo_path):
git("init")
git("config", "user.name", "Spack")
git("config", "user.email", "spack@spack.io")
commits = []
def latest_commit():
return git("rev-list", "-n1", "HEAD", output=str, error=str).strip()
os.makedirs(os.path.dirname(filename))
# add pkg-a as a new package to the repository
shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-0.txt", filename)
git("add", filename)
commit("diff-test: new package")
commits.append(latest_commit())
# add v2.1.5 to pkg-a
shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-1.txt", filename)
git("add", filename)
commit("diff-test: add v2.1.5")
commits.append(latest_commit())
# add v2.1.6 to pkg-a
shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-2.txt", filename)
git("add", filename)
commit("diff-test: add v2.1.6")
commits.append(latest_commit())
# convert pkg-a to a manual download package
shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-3.txt", filename)
git("add", filename)
commit("diff-test: modification to make manual download package")
commits.append(latest_commit())
# The commits are ordered with the last commit first in the list
commits = list(reversed(commits))
# Return the git directory to install, the filename used, and the commits
yield repo_path, filename, commits
@pytest.fixture(autouse=True)
def clear_recorded_monkeypatches():
yield

19
lib/spack/spack/test/data/conftest/diff-test/package-0.txt Normal file
View File

@ -0,0 +1,19 @@
# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
class DiffTest(AutotoolsPackage):
"""zlib replacement with optimizations for next generation systems."""
homepage = "https://github.com/zlib-ng/zlib-ng"
url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz"
git = "https://github.com/zlib-ng/zlib-ng.git"
license("Zlib")
version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a")
version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8")
version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200")

20
lib/spack/spack/test/data/conftest/diff-test/package-1.txt Normal file
View File

@ -0,0 +1,20 @@
# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
class DiffTest(AutotoolsPackage):
"""zlib replacement with optimizations for next generation systems."""
homepage = "https://github.com/zlib-ng/zlib-ng"
url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz"
git = "https://github.com/zlib-ng/zlib-ng.git"
license("Zlib")
version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04")
version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a")
version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200")
version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8")

21
lib/spack/spack/test/data/conftest/diff-test/package-2.txt Normal file
View File

@ -0,0 +1,21 @@
# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
class DiffTest(AutotoolsPackage):
"""zlib replacement with optimizations for next generation systems."""
homepage = "https://github.com/zlib-ng/zlib-ng"
url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz"
git = "https://github.com/zlib-ng/zlib-ng.git"
license("Zlib")
version("2.1.6", tag="2.1.6", commit="74253725f884e2424a0dd8ae3f69896d5377f325")
version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04")
version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a")
version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200")
version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8")

23
lib/spack/spack/test/data/conftest/diff-test/package-3.txt Normal file
View File

@ -0,0 +1,23 @@
# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
class DiffTest(AutotoolsPackage):
"""zlib replacement with optimizations for next generation systems."""
homepage = "https://github.com/zlib-ng/zlib-ng"
url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz"
git = "https://github.com/zlib-ng/zlib-ng.git"
license("Zlib")
manual_download = True
version("2.1.6", tag="2.1.6", commit="74253725f884e2424a0dd8ae3f69896d5377f325")
version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04")
version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a")
version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200")
version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8")

15
lib/spack/spack/test/util/git.py Normal file
View File

@ -0,0 +1,15 @@
# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from llnl.util.filesystem import working_dir
from spack.util.git import get_modified_files
def test_modified_files(mock_git_package_changes):
repo_path, filename, commits = mock_git_package_changes
with working_dir(repo_path):
files = get_modified_files(from_ref="HEAD~1", to_ref="HEAD")
assert len(files) == 1
assert files[0] == filename

16
lib/spack/spack/util/git.py
View File

@ -4,7 +4,7 @@
"""Single util module where Spack should get a git executable."""
import sys
from typing import Optional
from typing import List, Optional
import llnl.util.lang
@ -26,3 +26,17 @@ def git(required: bool = False):
git.add_default_arg("-c", "protocol.file.allow=always")
return git
def get_modified_files(from_ref: str = "HEAD~1", to_ref: str = "HEAD") -> List[str]:
"""Get a list of files modified between `from_ref` and `to_ref`
Args:
from_ref (str): oldest git ref, defaults to `HEAD~1`
to_ref (str): newer git ref, defaults to `HEAD`
Returns: list of file paths
"""
git_exe = git(required=True)
stdout = git_exe("diff", "--name-only", from_ref, to_ref, output=str)
return stdout.split()

11
share/spack/spack-completion.bash
View File

@ -687,7 +687,7 @@ _spack_ci() {
then
SPACK_COMPREPLY="-h --help"
else
SPACK_COMPREPLY="generate rebuild-index rebuild reproduce-build"
SPACK_COMPREPLY="generate rebuild-index rebuild reproduce-build verify-versions"
fi
}
@ -712,6 +712,15 @@ _spack_ci_reproduce_build() {
fi
}
_spack_ci_verify_versions() {
if $list_options
then
SPACK_COMPREPLY="-h --help"
else
SPACK_COMPREPLY=""
fi
}
_spack_clean() {
if $list_options
then

7
share/spack/spack-completion.fish
View File

@ -950,6 +950,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a generate -d 'ge
complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a rebuild-index -d 'rebuild the buildcache index for the remote mirror'
complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a rebuild -d 'rebuild a spec if it is not on the remote mirror'
complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a reproduce-build -d 'generate instructions for reproducing the spec rebuild job'
complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a verify-versions -d 'validate version checksum & commits between git refs'
complete -c spack -n '__fish_spack_using_command ci' -s h -l help -f -a help
complete -c spack -n '__fish_spack_using_command ci' -s h -l help -d 'show this help message and exit'
@ -1016,6 +1017,12 @@ complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l gpg-file
complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l gpg-url -r -f -a gpg_url
complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l gpg-url -r -d 'URL to public GPG key for validating binary cache installs'
# spack ci verify-versions
set -g __fish_spack_optspecs_spack_ci_verify_versions h/help
complete -c spack -n '__fish_spack_using_command ci verify-versions' -s h -l help -f -a help
complete -c spack -n '__fish_spack_using_command ci verify-versions' -s h -l help -d 'show this help message and exit'
# spack clean
set -g __fish_spack_optspecs_spack_clean h/help s/stage d/downloads f/failures m/misc-cache p/python-cache b/bootstrap a/all
complete -c spack -n '__fish_spack_using_command_pos_remainder 0 clean' -f -k -a '(__fish_spack_specs)'

21
var/spack/repos/builtin.mock/packages/diff-test/package.py Normal file
View File

@ -0,0 +1,21 @@
# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
class DiffTest(AutotoolsPackage):
"""zlib replacement with optimizations for next generation systems."""
homepage = "https://github.com/zlib-ng/zlib-ng"
url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz"
git = "https://github.com/zlib-ng/zlib-ng.git"
license("Zlib")
version("2.1.6", tag="2.1.6", commit="74253725f884e2424a0dd8ae3f69896d5377f325")
version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04")
version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a")
version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200")
version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8")