diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 74ad0bd4d25..50a65578925 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -76,10 +76,11 @@ jobs: prechecks: needs: [ changes ] - uses: ./.github/workflows/valid-style.yml + uses: ./.github/workflows/prechecks.yml secrets: inherit with: with_coverage: ${{ needs.changes.outputs.core }} + with_packages: ${{ needs.changes.outputs.packages }} import-check: needs: [ changes ] @@ -93,7 +94,7 @@ jobs: - name: Success run: | if [ "${{ needs.prechecks.result }}" == "failure" ] || [ "${{ needs.prechecks.result }}" == "canceled" ]; then - echo "Unit tests failed." + echo "Unit tests failed." exit 1 else exit 0 @@ -101,6 +102,7 @@ jobs: coverage: needs: [ unit-tests, prechecks ] + if: ${{ needs.changes.outputs.core }} uses: ./.github/workflows/coverage.yml secrets: inherit @@ -113,10 +115,10 @@ jobs: - name: Status summary run: | if [ "${{ needs.unit-tests.result }}" == "failure" ] || [ "${{ needs.unit-tests.result }}" == "canceled" ]; then - echo "Unit tests failed." + echo "Unit tests failed." exit 1 elif [ "${{ needs.bootstrap.result }}" == "failure" ] || [ "${{ needs.bootstrap.result }}" == "canceled" ]; then - echo "Bootstrap tests failed." + echo "Bootstrap tests failed." exit 1 else exit 0 diff --git a/.github/workflows/valid-style.yml b/.github/workflows/prechecks.yml similarity index 90% rename from .github/workflows/valid-style.yml rename to .github/workflows/prechecks.yml index 4553750e8cd..7da6a97fd3d 100644 --- a/.github/workflows/valid-style.yml +++ b/.github/workflows/prechecks.yml @@ -1,4 +1,4 @@ -name: style +name: prechecks on: workflow_call: @@ -6,6 +6,9 @@ on: with_coverage: required: true type: string + with_packages: + required: true + type: string concurrency: group: style-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} @@ -30,6 +33,7 @@ jobs: run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/ - name: vermin (Repositories) run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv var/spack/repos + # Run style checks on the files that have been changed style: runs-on: ubuntu-latest @@ -53,12 +57,25 @@ jobs: - name: Run style tests run: | share/spack/qa/run-style-tests + audit: uses: ./.github/workflows/audit.yaml secrets: inherit with: with_coverage: ${{ inputs.with_coverage }} python_version: '3.13' + + verify-checksums: + if: ${{ inputs.with_packages == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 + with: + fetch-depth: 2 + - name: Verify Added Checksums + run: | + bin/spack ci verify-versions HEAD^1 HEAD + # Check that spack can bootstrap the development environment on Python 3.6 - RHEL8 bootstrap-dev-rhel8: runs-on: ubuntu-latest diff --git a/lib/spack/spack/ci/__init__.py b/lib/spack/spack/ci/__init__.py index c5726590972..055df22882a 100644 --- a/lib/spack/spack/ci/__init__.py +++ b/lib/spack/spack/ci/__init__.py @@ -14,7 +14,7 @@ import tempfile import zipfile from collections import namedtuple -from typing import Callable, Dict, List, Set +from typing import Callable, Dict, List, Set, Union from urllib.request import Request import llnl.path @@ -42,6 +42,7 @@ from spack import traverse from spack.error import SpackError from spack.reporters.cdash import SPACK_CDASH_TIMEOUT +from spack.version import GitVersion, StandardVersion from .common import ( IS_WINDOWS, @@ -80,6 +81,45 @@ def get_change_revisions(): return None, None +def get_added_versions( + checksums_version_dict: Dict[str, Union[StandardVersion, GitVersion]], + path: str, + from_ref: str = "HEAD~1", + to_ref: str = "HEAD", +) -> List[Union[StandardVersion, GitVersion]]: + """Get a list of the versions added between `from_ref` and `to_ref`. + Args: + checksums_version_dict (Dict): all package versions keyed by known checksums. + path (str): path to the package.py + from_ref (str): oldest git ref, defaults to `HEAD~1` + to_ref (str): newer git ref, defaults to `HEAD` + Returns: list of versions added between refs + """ + git_exe = spack.util.git.git(required=True) + + # Gather git diff + diff_lines = git_exe("diff", from_ref, to_ref, "--", path, output=str).split("\n") + + # Store added and removed versions + # Removed versions are tracked here to determine when versions are moved in a file + # and show up as both added and removed in a git diff. + added_checksums = set() + removed_checksums = set() + + # Scrape diff for modified versions and prune added versions if they show up + # as also removed (which means they've actually just moved in the file and + # we shouldn't need to rechecksum them) + for checksum in checksums_version_dict.keys(): + for line in diff_lines: + if checksum in line: + if line.startswith("+"): + added_checksums.add(checksum) + if line.startswith("-"): + removed_checksums.add(checksum) + + return [checksums_version_dict[c] for c in added_checksums - removed_checksums] + + def get_stack_changed(env_path, rev1="HEAD^", rev2="HEAD"): """Given an environment manifest path and two revisions to compare, return whether or not the stack was changed. Returns True if the environment diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index 1cd4a1127d3..0b5f4f0d001 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -4,12 +4,15 @@ import json import os +import re import shutil +import sys +from typing import Dict from urllib.parse import urlparse, urlunparse import llnl.util.filesystem as fs -import llnl.util.tty as tty import llnl.util.tty.color as clr +from llnl.util import tty import spack.binary_distribution as bindist import spack.ci as spack_ci @@ -18,12 +21,22 @@ import spack.cmd.common.arguments import spack.config as cfg import spack.environment as ev +import spack.error +import spack.fetch_strategy import spack.hash_types as ht import spack.mirrors.mirror +import spack.package_base +import spack.paths +import spack.repo +import spack.spec +import spack.stage +import spack.util.executable +import spack.util.git import spack.util.gpg as gpg_util import spack.util.timer as timer import spack.util.url as url_util import spack.util.web as web_util +import spack.version description = "manage continuous integration pipelines" section = "build" @@ -32,6 +45,7 @@ SPACK_COMMAND = "spack" INSTALL_FAIL_CODE = 1 FAILED_CREATE_BUILDCACHE_CODE = 100 +BUILTIN = re.compile(r"var\/spack\/repos\/builtin\/packages\/([^\/]+)\/package\.py") def deindent(desc): @@ -191,6 +205,16 @@ def setup_parser(subparser): reproduce.set_defaults(func=ci_reproduce) + # Verify checksums inside of ci workflows + verify_versions = subparsers.add_parser( + "verify-versions", + description=deindent(ci_verify_versions.__doc__), + help=spack.cmd.first_line(ci_verify_versions.__doc__), + ) + verify_versions.add_argument("from_ref", help="git ref from which start looking at changes") + verify_versions.add_argument("to_ref", help="git ref to end looking at changes") + verify_versions.set_defaults(func=ci_verify_versions) + def ci_generate(args): """generate jobs file from a CI-aware spack file @@ -659,6 +683,159 @@ def _gitlab_artifacts_url(url: str) -> str: return urlunparse(parsed._replace(path="/".join(parts), fragment="", query="")) +def validate_standard_versions( + pkg: spack.package_base.PackageBase, versions: spack.version.VersionList +) -> bool: + """Get and test the checksum of a package version based on a tarball. + Args: + pkg spack.package_base.PackageBase: Spack package for which to validate a version checksum + versions spack.version.VersionList: list of package versions to validate + Returns: bool: result of the validation. True is valid and false is failed. + """ + url_dict: Dict[spack.version.StandardVersion, str] = {} + + for version in versions: + url = pkg.find_valid_url_for_version(version) + url_dict[version] = url + + version_hashes = spack.stage.get_checksums_for_versions( + url_dict, pkg.name, fetch_options=pkg.fetch_options + ) + + valid_checksums = True + for version, sha in version_hashes.items(): + if sha != pkg.versions[version]["sha256"]: + tty.error( + f"Invalid checksum found {pkg.name}@{version}\n" + f" [package.py] {pkg.versions[version]['sha256']}\n" + f" [Downloaded] {sha}" + ) + valid_checksums = False + continue + + tty.info(f"Validated {pkg.name}@{version} --> {sha}") + + return valid_checksums + + +def validate_git_versions( + pkg: spack.package_base.PackageBase, versions: spack.version.VersionList +) -> bool: + """Get and test the commit and tag of a package version based on a git repository. + Args: + pkg spack.package_base.PackageBase: Spack package for which to validate a version + versions spack.version.VersionList: list of package versions to validate + Returns: bool: result of the validation. True is valid and false is failed. + """ + valid_commit = True + for version in versions: + fetcher = spack.fetch_strategy.for_package_version(pkg, version) + with spack.stage.Stage(fetcher) as stage: + known_commit = pkg.versions[version]["commit"] + try: + stage.fetch() + except spack.error.FetchError: + tty.error( + f"Invalid commit for {pkg.name}@{version}\n" + f" {known_commit} could not be checked out in the git repository." + ) + valid_commit = False + continue + + # Test if the specified tag matches the commit in the package.py + # We retrieve the commit associated with a tag and compare it to the + # commit that is located in the package.py file. + if "tag" in pkg.versions[version]: + tag = pkg.versions[version]["tag"] + try: + with fs.working_dir(stage.source_path): + found_commit = fetcher.git( + "rev-list", "-n", "1", tag, output=str, error=str + ).strip() + except spack.util.executable.ProcessError: + tty.error( + f"Invalid tag for {pkg.name}@{version}\n" + f" {tag} could not be found in the git repository." + ) + valid_commit = False + continue + + if found_commit != known_commit: + tty.error( + f"Mismatched tag <-> commit found for {pkg.name}@{version}\n" + f" [package.py] {known_commit}\n" + f" [Downloaded] {found_commit}" + ) + valid_commit = False + continue + + # If we have downloaded the repository, found the commit, and compared + # the tag (if specified) we can conclude that the version is pointing + # at what we would expect. + tty.info(f"Validated {pkg.name}@{version} --> {known_commit}") + + return valid_commit + + +def ci_verify_versions(args): + """validate version checksum & commits between git refs + This command takes a from_ref and to_ref arguments and + then parses the git diff between the two to determine which packages + have been modified verifies the new checksums inside of them. + """ + with fs.working_dir(spack.paths.prefix): + # We use HEAD^1 explicitly on the merge commit created by + # GitHub Actions. However HEAD~1 is a safer default for the helper function. + files = spack.util.git.get_modified_files(from_ref=args.from_ref, to_ref=args.to_ref) + + # Get a list of package names from the modified files. + pkgs = [(m.group(1), p) for p in files for m in [BUILTIN.search(p)] if m] + + failed_version = False + for pkg_name, path in pkgs: + spec = spack.spec.Spec(pkg_name) + pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec) + + # Skip checking manual download packages and trust the maintainers + if pkg.manual_download: + tty.warn(f"Skipping manual download package: {pkg_name}") + continue + + # Store versions checksums / commits for future loop + checksums_version_dict = {} + commits_version_dict = {} + for version in pkg.versions: + # If the package version defines a sha256 we'll use that as the high entropy + # string to detect which versions have been added between from_ref and to_ref + if "sha256" in pkg.versions[version]: + checksums_version_dict[pkg.versions[version]["sha256"]] = version + + # If a package version instead defines a commit we'll use that as a + # high entropy string to detect new versions. + elif "commit" in pkg.versions[version]: + commits_version_dict[pkg.versions[version]["commit"]] = version + + # TODO: enforce every version have a commit or a sha256 defined if not + # an infinite version (there are a lot of package's where this doesn't work yet.) + + with fs.working_dir(spack.paths.prefix): + added_checksums = spack_ci.get_added_versions( + checksums_version_dict, path, from_ref=args.from_ref, to_ref=args.to_ref + ) + added_commits = spack_ci.get_added_versions( + commits_version_dict, path, from_ref=args.from_ref, to_ref=args.to_ref + ) + + if added_checksums: + failed_version = not validate_standard_versions(pkg, added_checksums) or failed_version + + if added_commits: + failed_version = not validate_git_versions(pkg, added_commits) or failed_version + + if failed_version: + sys.exit(1) + + def ci(parser, args): if args.func: return args.func(args) diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index afbe02fd79e..3412cf41736 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -18,6 +18,7 @@ import spack.repo as repo import spack.util.git from spack.test.conftest import MockHTTPResponse +from spack.version import Version pytestmark = [pytest.mark.usefixtures("mock_packages")] @@ -30,6 +31,43 @@ def repro_dir(tmp_path): yield result +def test_get_added_versions_new_checksum(mock_git_package_changes): + repo_path, filename, commits = mock_git_package_changes + + checksum_versions = { + "3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04": Version("2.1.5"), + "a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a": Version("2.1.4"), + "6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200": Version("2.0.7"), + "86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8": Version("2.0.0"), + } + + with fs.working_dir(str(repo_path)): + added_versions = ci.get_added_versions( + checksum_versions, filename, from_ref=commits[-1], to_ref=commits[-2] + ) + assert len(added_versions) == 1 + assert added_versions[0] == Version("2.1.5") + + +def test_get_added_versions_new_commit(mock_git_package_changes): + repo_path, filename, commits = mock_git_package_changes + + checksum_versions = { + "74253725f884e2424a0dd8ae3f69896d5377f325": Version("2.1.6"), + "3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04": Version("2.1.5"), + "a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a": Version("2.1.4"), + "6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200": Version("2.0.7"), + "86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8": Version("2.0.0"), + } + + with fs.working_dir(str(repo_path)): + added_versions = ci.get_added_versions( + checksum_versions, filename, from_ref=commits[2], to_ref=commits[1] + ) + assert len(added_versions) == 1 + assert added_versions[0] == Version("2.1.6") + + def test_pipeline_dag(config, tmpdir): r"""Test creation, pruning, and traversal of PipelineDAG using the following package dependency graph: diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 8afe70dc981..31ac7981d23 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -22,7 +22,11 @@ import spack.hash_types as ht import spack.main import spack.paths as spack_paths +import spack.repo +import spack.spec +import spack.stage import spack.util.spack_yaml as syaml +import spack.version from spack.ci import gitlab as gitlab_generator from spack.ci.common import PipelineDag, PipelineOptions, SpackCIConfig from spack.ci.generator_registry import generator @@ -1841,3 +1845,216 @@ def test_ci_generate_alternate_target( assert pipeline_doc.startswith("unittestpipeline") assert "externaltest" in pipeline_doc + + +@pytest.fixture +def fetch_versions_match(monkeypatch): + """Fake successful checksums returned from downloaded tarballs.""" + + def get_checksums_for_versions(url_by_version, package_name, **kwargs): + pkg_cls = spack.repo.PATH.get_pkg_class(package_name) + return {v: pkg_cls.versions[v]["sha256"] for v in url_by_version} + + monkeypatch.setattr(spack.stage, "get_checksums_for_versions", get_checksums_for_versions) + + +@pytest.fixture +def fetch_versions_invalid(monkeypatch): + """Fake successful checksums returned from downloaded tarballs.""" + + def get_checksums_for_versions(url_by_version, package_name, **kwargs): + return { + v: "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + for v in url_by_version + } + + monkeypatch.setattr(spack.stage, "get_checksums_for_versions", get_checksums_for_versions) + + +@pytest.mark.parametrize("versions", [["2.1.4"], ["2.1.4", "2.1.5"]]) +def test_ci_validate_standard_versions_valid(capfd, mock_packages, fetch_versions_match, versions): + spec = spack.spec.Spec("diff-test") + pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec) + version_list = [spack.version.Version(v) for v in versions] + + assert spack.cmd.ci.validate_standard_versions(pkg, version_list) + + out, err = capfd.readouterr() + for version in versions: + assert f"Validated diff-test@{version}" in out + + +@pytest.mark.parametrize("versions", [["2.1.4"], ["2.1.4", "2.1.5"]]) +def test_ci_validate_standard_versions_invalid( + capfd, mock_packages, fetch_versions_invalid, versions +): + spec = spack.spec.Spec("diff-test") + pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec) + version_list = [spack.version.Version(v) for v in versions] + + assert spack.cmd.ci.validate_standard_versions(pkg, version_list) is False + + out, err = capfd.readouterr() + for version in versions: + assert f"Invalid checksum found diff-test@{version}" in err + + +@pytest.mark.parametrize("versions", [[("1.0", -2)], [("1.1", -4), ("2.0", -6)]]) +def test_ci_validate_git_versions_valid( + capfd, monkeypatch, mock_packages, mock_git_version_info, versions +): + spec = spack.spec.Spec("diff-test") + pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec) + version_list = [spack.version.Version(v) for v, _ in versions] + + repo_path, filename, commits = mock_git_version_info + version_commit_dict = { + spack.version.Version(v): {"tag": f"v{v}", "commit": commits[c]} for v, c in versions + } + + pkg_class = spec.package_class + + monkeypatch.setattr(pkg_class, "git", repo_path) + monkeypatch.setattr(pkg_class, "versions", version_commit_dict) + + assert spack.cmd.ci.validate_git_versions(pkg, version_list) + + out, err = capfd.readouterr() + for version in version_list: + assert f"Validated diff-test@{version}" in out + + +@pytest.mark.parametrize("versions", [[("1.0", -3)], [("1.1", -5), ("2.0", -5)]]) +def test_ci_validate_git_versions_bad_tag( + capfd, monkeypatch, mock_packages, mock_git_version_info, versions +): + spec = spack.spec.Spec("diff-test") + pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec) + version_list = [spack.version.Version(v) for v, _ in versions] + + repo_path, filename, commits = mock_git_version_info + version_commit_dict = { + spack.version.Version(v): {"tag": f"v{v}", "commit": commits[c]} for v, c in versions + } + + pkg_class = spec.package_class + + monkeypatch.setattr(pkg_class, "git", repo_path) + monkeypatch.setattr(pkg_class, "versions", version_commit_dict) + + assert spack.cmd.ci.validate_git_versions(pkg, version_list) is False + + out, err = capfd.readouterr() + for version in version_list: + assert f"Mismatched tag <-> commit found for diff-test@{version}" in err + + +@pytest.mark.parametrize("versions", [[("1.0", -2)], [("1.1", -4), ("2.0", -6), ("3.0", -6)]]) +def test_ci_validate_git_versions_invalid( + capfd, monkeypatch, mock_packages, mock_git_version_info, versions +): + spec = spack.spec.Spec("diff-test") + pkg = spack.repo.PATH.get_pkg_class(spec.name)(spec) + version_list = [spack.version.Version(v) for v, _ in versions] + + repo_path, filename, commits = mock_git_version_info + version_commit_dict = { + spack.version.Version(v): { + "tag": f"v{v}", + "commit": "abcdefabcdefabcdefabcdefabcdefabcdefabc", + } + for v, c in versions + } + + pkg_class = spec.package_class + + monkeypatch.setattr(pkg_class, "git", repo_path) + monkeypatch.setattr(pkg_class, "versions", version_commit_dict) + + assert spack.cmd.ci.validate_git_versions(pkg, version_list) is False + + out, err = capfd.readouterr() + for version in version_list: + assert f"Invalid commit for diff-test@{version}" in err + + +@pytest.fixture +def verify_standard_versions_valid(monkeypatch): + def validate_standard_versions(pkg, versions): + for version in versions: + print(f"Validated {pkg.name}@{version}") + return True + + monkeypatch.setattr(spack.cmd.ci, "validate_standard_versions", validate_standard_versions) + + +@pytest.fixture +def verify_git_versions_valid(monkeypatch): + def validate_git_versions(pkg, versions): + for version in versions: + print(f"Validated {pkg.name}@{version}") + return True + + monkeypatch.setattr(spack.cmd.ci, "validate_git_versions", validate_git_versions) + + +@pytest.fixture +def verify_standard_versions_invalid(monkeypatch): + def validate_standard_versions(pkg, versions): + for version in versions: + print(f"Invalid checksum found {pkg.name}@{version}") + return False + + monkeypatch.setattr(spack.cmd.ci, "validate_standard_versions", validate_standard_versions) + + +@pytest.fixture +def verify_git_versions_invalid(monkeypatch): + def validate_git_versions(pkg, versions): + for version in versions: + print(f"Invalid commit for {pkg.name}@{version}") + return False + + monkeypatch.setattr(spack.cmd.ci, "validate_git_versions", validate_git_versions) + + +def test_ci_verify_versions_valid( + monkeypatch, + mock_packages, + mock_git_package_changes, + verify_standard_versions_valid, + verify_git_versions_valid, +): + repo_path, _, commits = mock_git_package_changes + monkeypatch.setattr(spack.paths, "prefix", repo_path) + + out = ci_cmd("verify-versions", commits[-1], commits[-3]) + assert "Validated diff-test@2.1.5" in out + assert "Validated diff-test@2.1.6" in out + + +def test_ci_verify_versions_standard_invalid( + monkeypatch, + mock_packages, + mock_git_package_changes, + verify_standard_versions_invalid, + verify_git_versions_invalid, +): + repo_path, _, commits = mock_git_package_changes + + monkeypatch.setattr(spack.paths, "prefix", repo_path) + + out = ci_cmd("verify-versions", commits[-1], commits[-3], fail_on_error=False) + assert "Invalid checksum found diff-test@2.1.5" in out + assert "Invalid commit for diff-test@2.1.6" in out + + +def test_ci_verify_versions_manual_package(monkeypatch, mock_packages, mock_git_package_changes): + repo_path, _, commits = mock_git_package_changes + monkeypatch.setattr(spack.paths, "prefix", repo_path) + + pkg_class = spack.spec.Spec("diff-test").package_class + monkeypatch.setattr(pkg_class, "manual_download", True) + + out = ci_cmd("verify-versions", commits[-1], commits[-2]) + assert "Skipping manual download package: diff-test" in out diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 21b298a41ee..775985ccc03 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -161,7 +161,7 @@ def mock_git_version_info(git, tmpdir, override_git_repos_cache_path): version tags on multiple branches, and version order is not equal to time order or topological order. """ - repo_path = str(tmpdir.mkdir("git_repo")) + repo_path = str(tmpdir.mkdir("git_version_info_repo")) filename = "file.txt" def commit(message): @@ -242,6 +242,84 @@ def latest_commit(): yield repo_path, filename, commits +@pytest.fixture +def mock_git_package_changes(git, tmpdir, override_git_repos_cache_path): + """Create a mock git repo with known structure of package edits + + The structure of commits in this repo is as follows:: + + o diff-test: modification to make manual download package + | + o diff-test: add v1.2 (from a git ref) + | + o diff-test: add v1.1 (from source tarball) + | + o diff-test: new package (testing multiple added versions) + + The repo consists of a single package.py file for DiffTest. + + Important attributes of the repo for test coverage are: multiple package + versions are added with some coming from a tarball and some from git refs. + """ + repo_path = str(tmpdir.mkdir("git_package_changes_repo")) + filename = "var/spack/repos/builtin/packages/diff-test/package.py" + + def commit(message): + global commit_counter + git( + "commit", + "--no-gpg-sign", + "--date", + "2020-01-%02d 12:0:00 +0300" % commit_counter, + "-am", + message, + ) + commit_counter += 1 + + with working_dir(repo_path): + git("init") + + git("config", "user.name", "Spack") + git("config", "user.email", "spack@spack.io") + + commits = [] + + def latest_commit(): + return git("rev-list", "-n1", "HEAD", output=str, error=str).strip() + + os.makedirs(os.path.dirname(filename)) + + # add pkg-a as a new package to the repository + shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-0.txt", filename) + git("add", filename) + commit("diff-test: new package") + commits.append(latest_commit()) + + # add v2.1.5 to pkg-a + shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-1.txt", filename) + git("add", filename) + commit("diff-test: add v2.1.5") + commits.append(latest_commit()) + + # add v2.1.6 to pkg-a + shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-2.txt", filename) + git("add", filename) + commit("diff-test: add v2.1.6") + commits.append(latest_commit()) + + # convert pkg-a to a manual download package + shutil.copy2(f"{spack.paths.test_path}/data/conftest/diff-test/package-3.txt", filename) + git("add", filename) + commit("diff-test: modification to make manual download package") + commits.append(latest_commit()) + + # The commits are ordered with the last commit first in the list + commits = list(reversed(commits)) + + # Return the git directory to install, the filename used, and the commits + yield repo_path, filename, commits + + @pytest.fixture(autouse=True) def clear_recorded_monkeypatches(): yield diff --git a/lib/spack/spack/test/data/conftest/diff-test/package-0.txt b/lib/spack/spack/test/data/conftest/diff-test/package-0.txt new file mode 100644 index 00000000000..9c42cc031a3 --- /dev/null +++ b/lib/spack/spack/test/data/conftest/diff-test/package-0.txt @@ -0,0 +1,19 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +class DiffTest(AutotoolsPackage): + """zlib replacement with optimizations for next generation systems.""" + + homepage = "https://github.com/zlib-ng/zlib-ng" + url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz" + git = "https://github.com/zlib-ng/zlib-ng.git" + + license("Zlib") + + version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a") + version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8") + version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200") diff --git a/lib/spack/spack/test/data/conftest/diff-test/package-1.txt b/lib/spack/spack/test/data/conftest/diff-test/package-1.txt new file mode 100644 index 00000000000..43e6c749b16 --- /dev/null +++ b/lib/spack/spack/test/data/conftest/diff-test/package-1.txt @@ -0,0 +1,20 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +class DiffTest(AutotoolsPackage): + """zlib replacement with optimizations for next generation systems.""" + + homepage = "https://github.com/zlib-ng/zlib-ng" + url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz" + git = "https://github.com/zlib-ng/zlib-ng.git" + + license("Zlib") + + version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04") + version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a") + version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200") + version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8") diff --git a/lib/spack/spack/test/data/conftest/diff-test/package-2.txt b/lib/spack/spack/test/data/conftest/diff-test/package-2.txt new file mode 100644 index 00000000000..b384756eda3 --- /dev/null +++ b/lib/spack/spack/test/data/conftest/diff-test/package-2.txt @@ -0,0 +1,21 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +class DiffTest(AutotoolsPackage): + """zlib replacement with optimizations for next generation systems.""" + + homepage = "https://github.com/zlib-ng/zlib-ng" + url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz" + git = "https://github.com/zlib-ng/zlib-ng.git" + + license("Zlib") + + version("2.1.6", tag="2.1.6", commit="74253725f884e2424a0dd8ae3f69896d5377f325") + version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04") + version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a") + version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200") + version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8") diff --git a/lib/spack/spack/test/data/conftest/diff-test/package-3.txt b/lib/spack/spack/test/data/conftest/diff-test/package-3.txt new file mode 100644 index 00000000000..02acea3cb5a --- /dev/null +++ b/lib/spack/spack/test/data/conftest/diff-test/package-3.txt @@ -0,0 +1,23 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +class DiffTest(AutotoolsPackage): + """zlib replacement with optimizations for next generation systems.""" + + homepage = "https://github.com/zlib-ng/zlib-ng" + url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz" + git = "https://github.com/zlib-ng/zlib-ng.git" + + license("Zlib") + + manual_download = True + + version("2.1.6", tag="2.1.6", commit="74253725f884e2424a0dd8ae3f69896d5377f325") + version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04") + version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a") + version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200") + version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8") diff --git a/lib/spack/spack/test/util/git.py b/lib/spack/spack/test/util/git.py new file mode 100644 index 00000000000..4f8c2a0d384 --- /dev/null +++ b/lib/spack/spack/test/util/git.py @@ -0,0 +1,15 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from llnl.util.filesystem import working_dir + +from spack.util.git import get_modified_files + + +def test_modified_files(mock_git_package_changes): + repo_path, filename, commits = mock_git_package_changes + + with working_dir(repo_path): + files = get_modified_files(from_ref="HEAD~1", to_ref="HEAD") + assert len(files) == 1 + assert files[0] == filename diff --git a/lib/spack/spack/util/git.py b/lib/spack/spack/util/git.py index d93b15418dd..23b25699f8b 100644 --- a/lib/spack/spack/util/git.py +++ b/lib/spack/spack/util/git.py @@ -4,7 +4,7 @@ """Single util module where Spack should get a git executable.""" import sys -from typing import Optional +from typing import List, Optional import llnl.util.lang @@ -26,3 +26,17 @@ def git(required: bool = False): git.add_default_arg("-c", "protocol.file.allow=always") return git + + +def get_modified_files(from_ref: str = "HEAD~1", to_ref: str = "HEAD") -> List[str]: + """Get a list of files modified between `from_ref` and `to_ref` + Args: + from_ref (str): oldest git ref, defaults to `HEAD~1` + to_ref (str): newer git ref, defaults to `HEAD` + Returns: list of file paths + """ + git_exe = git(required=True) + + stdout = git_exe("diff", "--name-only", from_ref, to_ref, output=str) + + return stdout.split() diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 829cd5c17ca..33ea464d776 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -687,7 +687,7 @@ _spack_ci() { then SPACK_COMPREPLY="-h --help" else - SPACK_COMPREPLY="generate rebuild-index rebuild reproduce-build" + SPACK_COMPREPLY="generate rebuild-index rebuild reproduce-build verify-versions" fi } @@ -712,6 +712,15 @@ _spack_ci_reproduce_build() { fi } +_spack_ci_verify_versions() { + if $list_options + then + SPACK_COMPREPLY="-h --help" + else + SPACK_COMPREPLY="" + fi +} + _spack_clean() { if $list_options then diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index 29cd458b2cc..0297384f924 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -950,6 +950,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a generate -d 'ge complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a rebuild-index -d 'rebuild the buildcache index for the remote mirror' complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a rebuild -d 'rebuild a spec if it is not on the remote mirror' complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a reproduce-build -d 'generate instructions for reproducing the spec rebuild job' +complete -c spack -n '__fish_spack_using_command_pos 0 ci' -f -a verify-versions -d 'validate version checksum & commits between git refs' complete -c spack -n '__fish_spack_using_command ci' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command ci' -s h -l help -d 'show this help message and exit' @@ -1016,6 +1017,12 @@ complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l gpg-file complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l gpg-url -r -f -a gpg_url complete -c spack -n '__fish_spack_using_command ci reproduce-build' -l gpg-url -r -d 'URL to public GPG key for validating binary cache installs' +# spack ci verify-versions +set -g __fish_spack_optspecs_spack_ci_verify_versions h/help + +complete -c spack -n '__fish_spack_using_command ci verify-versions' -s h -l help -f -a help +complete -c spack -n '__fish_spack_using_command ci verify-versions' -s h -l help -d 'show this help message and exit' + # spack clean set -g __fish_spack_optspecs_spack_clean h/help s/stage d/downloads f/failures m/misc-cache p/python-cache b/bootstrap a/all complete -c spack -n '__fish_spack_using_command_pos_remainder 0 clean' -f -k -a '(__fish_spack_specs)' diff --git a/var/spack/repos/builtin.mock/packages/diff-test/package.py b/var/spack/repos/builtin.mock/packages/diff-test/package.py new file mode 100644 index 00000000000..b384756eda3 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/diff-test/package.py @@ -0,0 +1,21 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +class DiffTest(AutotoolsPackage): + """zlib replacement with optimizations for next generation systems.""" + + homepage = "https://github.com/zlib-ng/zlib-ng" + url = "https://github.com/zlib-ng/zlib-ng/archive/2.0.0.tar.gz" + git = "https://github.com/zlib-ng/zlib-ng.git" + + license("Zlib") + + version("2.1.6", tag="2.1.6", commit="74253725f884e2424a0dd8ae3f69896d5377f325") + version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04") + version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a") + version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200") + version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8")