zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ad-hoc Git commit versions: support submodules (#30037)

Browse Source 
* Allow packages to add a 'submodules' property that determines when ad-hoc Git-commit-based versions should initialize submodules

* add support for ad-hoc git-commit-based versions to instantiate submodules if the associated package has a 'submodules' property and it indicates this should happen for the associated spec

* allow Package-level submodule request to influence all explicitly-defined version() in the Package

* skip test on windows which fails because of long paths
This commit is contained in:
Peter Scheibel 2022-04-13 20:05:14 -07:00 committed by GitHub
parent b28b24ccf8
commit 89f6db21f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 90 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/spack/spack/fetch_strategy.py
View File

@ -1596,7 +1596,12 @@ def for_package_version(pkg, version):
if version.is_commit and hasattr(pkg, "git"):
# Populate the version with comparisons to other commits
version.generate_commit_lookup(pkg.name)
fetcher = GitFetchStrategy(git=pkg.git, commit=str(version))
kwargs = {
'git': pkg.git,
'commit': str(version)
}
kwargs['submodules'] = getattr(pkg, 'submodules', False)
fetcher = GitFetchStrategy(**kwargs)
return fetcher
# If it's not a known version, try to extrapolate one by URL
@ -1612,6 +1617,8 @@ def for_package_version(pkg, version):
for fetcher in all_strategies:
if fetcher.url_attr in args:
_check_version_attributes(fetcher, pkg, version)
if fetcher.url_attr == 'git' and hasattr(pkg, 'submodules'):
args.setdefault('submodules', pkg.submodules)
return fetcher(**args)
# if a version's optional attributes imply a particular fetch

48
lib/spack/spack/test/conftest.py
View File

@ -1239,12 +1239,30 @@ def get_date():
@pytest.fixture(scope='session')
def mock_git_repository(tmpdir_factory):
"""Creates a simple git repository with two branches,
two commits and two submodules. Each submodule has one commit.
"""Creates a git repository multiple commits, branches, submodules, and
a tag. Visual representation of the commit history (starting with the
earliest commit at c0)::
c3 c1 (test-branch, r1) c2 (tag-branch)
|______/_____________________/
c0 (r0)
There are two branches aside from 'master': 'test-branch' and 'tag-branch';
each has one commit; the tag-branch has a tag referring to its commit
(c2 in the diagram).
Two submodules are added as part of the very first commit on 'master'; each
of these refers to a repository with a single commit.
c0, c1, and c2 include information to define explicit versions in the
associated builtin.mock package 'git-test'. c3 is a commit in the
repository but does not have an associated explicit package version.
"""
git = spack.util.executable.which('git', required=True)
suburls = []
# Create two git repositories which will be used as submodules in the
# main repository
for submodule_count in range(2):
tmpdir = tmpdir_factory.mktemp('mock-git-repo-submodule-dir-{0}'
.format(submodule_count))
@ -1252,7 +1270,6 @@ def mock_git_repository(tmpdir_factory):
repodir = tmpdir.join(spack.stage._source_path_subdir)
suburls.append((submodule_count, 'file://' + str(repodir)))
# Initialize the repository
with repodir.as_cwd():
git('init')
git('config', 'user.name', 'Spack')
@ -1269,7 +1286,7 @@ def mock_git_repository(tmpdir_factory):
tmpdir.ensure(spack.stage._source_path_subdir, dir=True)
repodir = tmpdir.join(spack.stage._source_path_subdir)
# Initialize the repository
# Create the main repository
with repodir.as_cwd():
git('init')
git('config', 'user.name', 'Spack')
@ -1279,7 +1296,7 @@ def mock_git_repository(tmpdir_factory):
git('submodule', 'add', suburl,
'third_party/submodule{0}'.format(number))
# r0 is just the first commit
# r0 is the first commit: it consists of one file and two submodules
r0_file = 'r0_file'
repodir.ensure(r0_file)
git('add', r0_file)
@ -1293,13 +1310,13 @@ def mock_git_repository(tmpdir_factory):
tag_file = 'tag_file'
git('branch', tag_branch)
# Check out first branch
# Check out test branch and add one commit
git('checkout', branch)
repodir.ensure(branch_file)
git('add', branch_file)
git('-c', 'commit.gpgsign=false', 'commit', '-m' 'r1 test branch')
# Check out a second branch and tag it
# Check out the tag branch, add one commit, and then add a tag for it
git('checkout', tag_branch)
repodir.ensure(tag_file)
git('add', tag_file)
@ -1310,11 +1327,24 @@ def mock_git_repository(tmpdir_factory):
git('checkout', 'master')
# R1 test is the same as test for branch
r2_file = 'r2_file'
repodir.ensure(r2_file)
git('add', r2_file)
git('-c', 'commit.gpgsign=false', 'commit', '-m', 'mock-git-repo r2')
rev_hash = lambda x: git('rev-parse', x, output=str).strip()
r2 = rev_hash('master')
# Record the commit hash of the (only) commit from test-branch and
# the file added by that commit
r1 = rev_hash(branch)
r1_file = branch_file
# Map of version -> bunch. Each bunch includes; all the args
# that must be specified as part of a version() declaration (used to
# manufacture a version for the 'git-test' package); the associated
# revision for the version; a file associated with (and particular to)
# that revision/branch.
checks = {
'master': Bunch(
revision='master', file=r0_file, args={'git': url}
@ -1338,7 +1368,7 @@ def mock_git_repository(tmpdir_factory):
}
t = Bunch(checks=checks, url=url, hash=rev_hash,
path=str(repodir), git_exe=git)
path=str(repodir), git_exe=git, unversioned_commit=r2)
yield t

37
lib/spack/spack/test/git_fetch.py
View File

@ -137,6 +137,36 @@ def test_fetch(type_of_test,
assert h('HEAD') == h(t.revision)
@pytest.mark.skipif(str(spack.platforms.host()) == 'windows',
reason=('Git fails to clone because the src/dst paths'
' are too long: the name of the staging directory'
' for ad-hoc Git commit versions is longer than'
' other staged sources'))
@pytest.mark.disable_clean_stage_check
def test_adhoc_version_submodules(
mock_git_repository,
config,
mutable_mock_repo,
monkeypatch,
mock_stage):
t = mock_git_repository.checks['tag']
# Construct the package under test
pkg_class = spack.repo.path.get_pkg_class('git-test')
monkeypatch.setitem(pkg_class.versions, ver('git'), t.args)
monkeypatch.setattr(pkg_class, 'git', 'file://%s' % mock_git_repository.path,
raising=False)
spec = Spec('git-test@{0}'.format(mock_git_repository.unversioned_commit))
spec.concretize()
spec.package.do_stage()
collected_fnames = set()
for root, dirs, files in os.walk(spec.package.stage.source_path):
collected_fnames.update(files)
# The submodules generate files with the prefix "r0_file_"
assert set(['r0_file_0', 'r0_file_1']) < collected_fnames
@pytest.mark.parametrize("type_of_test", ['branch', 'commit'])
def test_debug_fetch(
mock_packages, type_of_test, mock_git_repository, config, monkeypatch
@ -227,7 +257,12 @@ def test_get_full_repo(get_full_repo, git_version, mock_git_repository,
def test_gitsubmodule(submodules, mock_git_repository, config,
mutable_mock_repo, monkeypatch):
"""
Test GitFetchStrategy behavior with submodules
Test GitFetchStrategy behavior with submodules. This package
has a `submodules` property which is always True: when a specific
version also indicates to include submodules, this should not
interfere; if the specific version explicitly requests that
submodules *not* be initialized, this should override the
Package-level request.
"""
type_of_test = 'tag-branch'
t = mock_git_repository.checks[type_of_test]

2
var/spack/repos/builtin.mock/packages/git-test/package.py
View File

@ -10,4 +10,6 @@ class GitTest(Package):
"""Mock package that uses git for fetching."""
homepage = "http://www.git-fetch-example.com"
submodules = True
version('git', git='to-be-filled-in-by-test')

5
var/spack/repos/builtin/packages/axom/package.py
View File

@ -50,6 +50,11 @@ class Axom(CachedCMakePackage, CudaPackage):
version('0.3.0', tag='v0.3.0', submodules=True)
version('0.2.9', tag='v0.2.9', submodules=True)
@property
def submodules(self):
# All git checkouts should also initialize submodules
return True
patch('scr_examples_gtest.patch', when='@0.6.0:0.6.1')
root_cmakelists_dir = 'src'