zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add spack buildcache push (alias to buildcache create) (#34861)

Browse Source 
`spack buildcache create` is a misnomer cause it's the only way to push to
an existing buildcache (and it in fact calls binary_distribution.push).

Also we have `spack buildcache update-index` but for create the flag is
`--rebuild-index`, which is confusing (and also... why "rebuild"
something if the command is "create" in the first place, that implies it
wasn't there to begin with).

So, after this PR, you can use either

```
spack buildcache create --rebuild-index
```

or

```
spack buildcache push --update-index
```

Also, alias `spack buildcache rebuild-index` to `spack buildcache
update-index`.
This commit is contained in:
Harmen Stoppels 2023-05-05 19:54:26 +02:00 committed by GitHub
parent ddea33bdc0
commit 9ef062fcca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 66 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/spack/docs/binary_caches.rst
View File

@ -36,7 +36,7 @@ Build caches are created via:
.. code-block:: console .. code-block:: console
$ spack buildcache create <path/url/mirror name> <spec> $ spack buildcache push <path/url/mirror name> <spec>
This command takes the locally installed spec and its dependencies, and This command takes the locally installed spec and its dependencies, and
creates tarballs of their install prefixes. It also generates metadata files, creates tarballs of their install prefixes. It also generates metadata files,
@ -48,7 +48,7 @@ Here is an example where a build cache is created in a local directory named
.. code-block:: console .. code-block:: console
$ spack buildcache create --allow-root ./spack-cache ninja $ spack buildcache push --allow-root ./spack-cache ninja
==> Pushing binary packages to file:///home/spackuser/spack/spack-cache/build_cache ==> Pushing binary packages to file:///home/spackuser/spack/spack-cache/build_cache
Not that ``ninja`` must be installed locally for this to work. Not that ``ninja`` must be installed locally for this to work.
@ -177,7 +177,7 @@ need to be adjusted for better re-locatability.
-------------------- --------------------
^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^
``spack buildcache create`` ``spack buildcache push``
^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^
Create tarball of installed Spack package and all dependencies. Create tarball of installed Spack package and all dependencies.

42
lib/spack/spack/cmd/buildcache.py
View File

@ -42,33 +42,28 @@ def setup_parser(subparser):
setup_parser.parser = subparser setup_parser.parser = subparser
subparsers = subparser.add_subparsers(help="buildcache sub-commands") subparsers = subparser.add_subparsers(help="buildcache sub-commands")
create = subparsers.add_parser("create", help=create_fn.__doc__) push = subparsers.add_parser("push", aliases=["create"], help=push_fn.__doc__)
# TODO: remove from Spack 0.21 # TODO: remove from Spack 0.21
create.add_argument( push.add_argument(
"-r", "-r",
"--rel", "--rel",
action="store_true", action="store_true",
help="make all rpaths relative before creating tarballs. (deprecated)", help="make all rpaths relative before creating tarballs. (deprecated)",
) )
create.add_argument( push.add_argument("-f", "--force", action="store_true", help="overwrite tarball if it exists.")
"-f", "--force", action="store_true", help="overwrite tarball if it exists." push.add_argument(
"-u", "--unsigned", action="store_true", help="push unsigned buildcache tarballs"
) )
create.add_argument( push.add_argument(
"-u",
"--unsigned",
action="store_true",
help="create unsigned buildcache tarballs for testing",
)
create.add_argument(
"-a", "-a",
"--allow-root", "--allow-root",
action="store_true", action="store_true",
help="allow install root string in binary files after RPATH substitution", help="allow install root string in binary files after RPATH substitution",
) )
create.add_argument( push.add_argument(
"-k", "--key", metavar="key", type=str, default=None, help="Key for signing." "-k", "--key", metavar="key", type=str, default=None, help="Key for signing."
) )
output = create.add_mutually_exclusive_group(required=False) output = push.add_mutually_exclusive_group(required=False)
# TODO: remove from Spack 0.21 # TODO: remove from Spack 0.21
output.add_argument( output.add_argument(
"-d", "-d",
@ -98,17 +93,18 @@ def setup_parser(subparser):
# Unfortunately we cannot add this to the mutually exclusive group above, # Unfortunately we cannot add this to the mutually exclusive group above,
# because we have further positional arguments. # because we have further positional arguments.
# TODO: require from Spack 0.21 # TODO: require from Spack 0.21
create.add_argument("mirror", type=str, help="Mirror name, path, or URL.", nargs="?") push.add_argument("mirror", type=str, help="Mirror name, path, or URL.", nargs="?")
create.add_argument( push.add_argument(
"--update-index",
"--rebuild-index", "--rebuild-index",
action="store_true", action="store_true",
default=False, default=False,
help="Regenerate buildcache index after building package(s)", help="Regenerate buildcache index after building package(s)",
) )
create.add_argument( push.add_argument(
"--spec-file", default=None, help="Create buildcache entry for spec from json or yaml file" "--spec-file", default=None, help="Create buildcache entry for spec from json or yaml file"
) )
create.add_argument( push.add_argument(
"--only", "--only",
default="package,dependencies", default="package,dependencies",
dest="things_to_install", dest="things_to_install",
@ -121,8 +117,8 @@ def setup_parser(subparser):
" or only the dependencies" " or only the dependencies"
), ),
) )
arguments.add_common_arguments(create, ["specs"]) arguments.add_common_arguments(push, ["specs"])
create.set_defaults(func=create_fn) push.set_defaults(func=push_fn)
install = subparsers.add_parser("install", help=install_fn.__doc__) install = subparsers.add_parser("install", help=install_fn.__doc__)
install.add_argument( install.add_argument(
@ -345,7 +341,9 @@ def setup_parser(subparser):
sync.set_defaults(func=sync_fn) sync.set_defaults(func=sync_fn)
# Update buildcache index without copying any additional packages # Update buildcache index without copying any additional packages
update_index = subparsers.add_parser("update-index", help=update_index_fn.__doc__) update_index = subparsers.add_parser(
"update-index", aliases=["rebuild-index"], help=update_index_fn.__doc__
)
update_index_out = update_index.add_mutually_exclusive_group(required=True) update_index_out = update_index.add_mutually_exclusive_group(required=True)
# TODO: remove in Spack 0.21 # TODO: remove in Spack 0.21
update_index_out.add_argument( update_index_out.add_argument(
@ -436,7 +434,7 @@ def _concrete_spec_from_args(args):
return Spec.from_specfile(specfile_path) return Spec.from_specfile(specfile_path)
def create_fn(args): def push_fn(args):
"""create a binary package and push it to a mirror""" """create a binary package and push it to a mirror"""
if args.mirror_flag: if args.mirror_flag:
mirror = args.mirror_flag mirror = args.mirror_flag
@ -492,7 +490,7 @@ def create_fn(args):
unsigned=args.unsigned, unsigned=args.unsigned,
allow_root=args.allow_root, allow_root=args.allow_root,
key=args.key, key=args.key,
regenerate_index=args.rebuild_index, regenerate_index=args.update_index,
), ),
) )

12
lib/spack/spack/test/bindist.py
View File

@ -204,9 +204,9 @@ def test_default_rpaths_create_install_default_layout(mirror_dir):
install_cmd("--no-cache", sy_spec.name) install_cmd("--no-cache", sy_spec.name)
# Create a buildache # Create a buildache
buildcache_cmd("create", "-au", "-d", mirror_dir, cspec.name, sy_spec.name) buildcache_cmd("push", "-au", "-d", mirror_dir, cspec.name, sy_spec.name)
# Test force overwrite create buildcache (-f option) # Test force overwrite create buildcache (-f option)
buildcache_cmd("create", "-auf", "-d", mirror_dir, cspec.name) buildcache_cmd("push", "-auf", "-d", mirror_dir, cspec.name)
# Create mirror index # Create mirror index
buildcache_cmd("update-index", "-d", mirror_dir) buildcache_cmd("update-index", "-d", mirror_dir)
@ -271,7 +271,7 @@ def test_relative_rpaths_create_default_layout(mirror_dir):
install_cmd("--no-cache", cspec.name) install_cmd("--no-cache", cspec.name)
# Create build cache with relative rpaths # Create build cache with relative rpaths
buildcache_cmd("create", "-aur", "-d", mirror_dir, cspec.name) buildcache_cmd("push", "-aur", "-d", mirror_dir, cspec.name)
# Create mirror index # Create mirror index
buildcache_cmd("update-index", "-d", mirror_dir) buildcache_cmd("update-index", "-d", mirror_dir)
@ -404,7 +404,7 @@ def test_spec_needs_rebuild(monkeypatch, tmpdir):
install_cmd(s.name) install_cmd(s.name)
# Put installed package in the buildcache # Put installed package in the buildcache
buildcache_cmd("create", "-u", "-a", "-d", mirror_dir.strpath, s.name) buildcache_cmd("push", "-u", "-a", "-d", mirror_dir.strpath, s.name)
rebuild = bindist.needs_rebuild(s, mirror_url) rebuild = bindist.needs_rebuild(s, mirror_url)
@ -433,7 +433,7 @@ def test_generate_index_missing(monkeypatch, tmpdir, mutable_config):
install_cmd("--no-cache", s.name) install_cmd("--no-cache", s.name)
# Create a buildcache and update index # Create a buildcache and update index
buildcache_cmd("create", "-uad", mirror_dir.strpath, s.name) buildcache_cmd("push", "-uad", mirror_dir.strpath, s.name)
buildcache_cmd("update-index", "-d", mirror_dir.strpath) buildcache_cmd("update-index", "-d", mirror_dir.strpath)
# Check package and dependency in buildcache # Check package and dependency in buildcache
@ -522,7 +522,7 @@ def test_update_sbang(tmpdir, test_mirror):
install_cmd("--no-cache", old_spec.name) install_cmd("--no-cache", old_spec.name)
# Create a buildcache with the installed spec. # Create a buildcache with the installed spec.
buildcache_cmd("create", "-u", "-a", "-d", mirror_dir, old_spec_hash_str) buildcache_cmd("push", "-u", "-a", "-d", mirror_dir, old_spec_hash_str)
# Need to force an update of the buildcache index # Need to force an update of the buildcache index
buildcache_cmd("update-index", "-d", mirror_dir) buildcache_cmd("update-index", "-d", mirror_dir)

16
lib/spack/spack/test/cmd/buildcache.py
View File

@ -85,7 +85,7 @@ def tests_buildcache_create(install_mockery, mock_fetch, monkeypatch, tmpdir):
pkg = "trivial-install-test-package" pkg = "trivial-install-test-package"
install(pkg) install(pkg)
buildcache("create", "-d", str(tmpdir), "--unsigned", pkg) buildcache("push", "-d", str(tmpdir), "--unsigned", pkg)
spec = Spec(pkg).concretized() spec = Spec(pkg).concretized()
tarball_path = spack.binary_distribution.tarball_path_name(spec, ".spack") tarball_path = spack.binary_distribution.tarball_path_name(spec, ".spack")
@ -105,7 +105,7 @@ def tests_buildcache_create_env(
add(pkg) add(pkg)
install() install()
buildcache("create", "-d", str(tmpdir), "--unsigned") buildcache("push", "-d", str(tmpdir), "--unsigned")
spec = Spec(pkg).concretized() spec = Spec(pkg).concretized()
tarball_path = spack.binary_distribution.tarball_path_name(spec, ".spack") tarball_path = spack.binary_distribution.tarball_path_name(spec, ".spack")
@ -118,7 +118,7 @@ def test_buildcache_create_fails_on_noargs(tmpdir):
"""Ensure that buildcache create fails when given no args or """Ensure that buildcache create fails when given no args or
environment.""" environment."""
with pytest.raises(spack.main.SpackCommandError): with pytest.raises(spack.main.SpackCommandError):
buildcache("create", "-d", str(tmpdir), "--unsigned") buildcache("push", "-d", str(tmpdir), "--unsigned")
def test_buildcache_create_fail_on_perm_denied(install_mockery, mock_fetch, monkeypatch, tmpdir): def test_buildcache_create_fail_on_perm_denied(install_mockery, mock_fetch, monkeypatch, tmpdir):
@ -127,7 +127,7 @@ def test_buildcache_create_fail_on_perm_denied(install_mockery, mock_fetch, monk
tmpdir.chmod(0) tmpdir.chmod(0)
with pytest.raises(OSError) as error: with pytest.raises(OSError) as error:
buildcache("create", "-d", str(tmpdir), "--unsigned", "trivial-install-test-package") buildcache("push", "-d", str(tmpdir), "--unsigned", "trivial-install-test-package")
assert error.value.errno == errno.EACCES assert error.value.errno == errno.EACCES
tmpdir.chmod(0o700) tmpdir.chmod(0o700)
@ -159,7 +159,7 @@ def test_update_key_index(
# Put installed package in the buildcache, which, because we're signing # Put installed package in the buildcache, which, because we're signing
# it, should result in the public key getting pushed to the buildcache # it, should result in the public key getting pushed to the buildcache
# as well. # as well.
buildcache("create", "-a", "-d", mirror_dir.strpath, s.name) buildcache("push", "-a", "-d", mirror_dir.strpath, s.name)
# Now make sure that when we pass the "--keys" argument to update-index # Now make sure that when we pass the "--keys" argument to update-index
# it causes the index to get update. # it causes the index to get update.
@ -213,13 +213,13 @@ def verify_mirror_contents():
# Install a package and put it in the buildcache # Install a package and put it in the buildcache
s = Spec(out_env_pkg).concretized() s = Spec(out_env_pkg).concretized()
install(s.name) install(s.name)
buildcache("create", "-u", "-f", "-a", "--mirror-url", src_mirror_url, s.name) buildcache("push", "-u", "-f", "-a", "--mirror-url", src_mirror_url, s.name)
env("create", "test") env("create", "test")
with ev.read("test"): with ev.read("test"):
add(in_env_pkg) add(in_env_pkg)
install() install()
buildcache("create", "-u", "-f", "-a", "--mirror-url", src_mirror_url, in_env_pkg) buildcache("push", "-u", "-f", "-a", "--mirror-url", src_mirror_url, in_env_pkg)
# Now run the spack buildcache sync command with all the various options # Now run the spack buildcache sync command with all the various options
# for specifying mirrors # for specifying mirrors
@ -260,7 +260,7 @@ def test_buildcache_create_install(
pkg = "trivial-install-test-package" pkg = "trivial-install-test-package"
install(pkg) install(pkg)
buildcache("create", "-d", str(tmpdir), "--unsigned", pkg) buildcache("push", "-d", str(tmpdir), "--unsigned", pkg)
spec = Spec(pkg).concretized() spec = Spec(pkg).concretized()
tarball_path = spack.binary_distribution.tarball_path_name(spec, ".spack") tarball_path = spack.binary_distribution.tarball_path_name(spec, ".spack")

10
lib/spack/spack/test/cmd/ci.py
View File

@ -1055,7 +1055,7 @@ def test_ci_nothing_to_rebuild(
) )
install_cmd("archive-files") install_cmd("archive-files")
buildcache_cmd("create", "-a", "-f", "-u", "--mirror-url", mirror_url, "archive-files") buildcache_cmd("push", "-a", "-f", "-u", "--mirror-url", mirror_url, "archive-files")
filename = str(tmpdir.join("spack.yaml")) filename = str(tmpdir.join("spack.yaml"))
with open(filename, "w") as f: with open(filename, "w") as f:
@ -1155,7 +1155,7 @@ def test_ci_generate_mirror_override(
second_ci_yaml = str(tmpdir.join(".gitlab-ci-2.yml")) second_ci_yaml = str(tmpdir.join(".gitlab-ci-2.yml"))
with ev.read("test"): with ev.read("test"):
install_cmd() install_cmd()
buildcache_cmd("create", "-u", "--mirror-url", mirror_url, "patchelf") buildcache_cmd("push", "-u", "--mirror-url", mirror_url, "patchelf")
buildcache_cmd("update-index", "--mirror-url", mirror_url, output=str) buildcache_cmd("update-index", "--mirror-url", mirror_url, output=str)
# This generate should not trigger a rebuild of patchelf, since it's in # This generate should not trigger a rebuild of patchelf, since it's in
@ -1613,7 +1613,7 @@ def test_ci_rebuild_index(
ypfd.write(spec_json) ypfd.write(spec_json)
install_cmd("--add", "--keep-stage", "-f", json_path) install_cmd("--add", "--keep-stage", "-f", json_path)
buildcache_cmd("create", "-u", "-a", "-f", "--mirror-url", mirror_url, "callpath") buildcache_cmd("push", "-u", "-a", "-f", "--mirror-url", mirror_url, "callpath")
ci_cmd("rebuild-index") ci_cmd("rebuild-index")
buildcache_path = os.path.join(mirror_dir.strpath, "build_cache") buildcache_path = os.path.join(mirror_dir.strpath, "build_cache")
@ -1647,7 +1647,7 @@ def test_ci_generate_bootstrap_prune_dag(
install_cmd("gcc@12.2.0%gcc@10.2.1") install_cmd("gcc@12.2.0%gcc@10.2.1")
# Put installed compiler in the buildcache # Put installed compiler in the buildcache
buildcache_cmd("create", "-u", "-a", "-f", "-d", mirror_dir.strpath, "gcc@12.2.0%gcc@10.2.1") buildcache_cmd("push", "-u", "-a", "-f", "-d", mirror_dir.strpath, "gcc@12.2.0%gcc@10.2.1")
# Now uninstall the compiler # Now uninstall the compiler
uninstall_cmd("-y", "gcc@12.2.0%gcc@10.2.1") uninstall_cmd("-y", "gcc@12.2.0%gcc@10.2.1")
@ -1662,7 +1662,7 @@ def test_ci_generate_bootstrap_prune_dag(
install_cmd("--no-check-signature", "b%gcc@12.2.0") install_cmd("--no-check-signature", "b%gcc@12.2.0")
# Put spec built with installed compiler in the buildcache # Put spec built with installed compiler in the buildcache
buildcache_cmd("create", "-u", "-a", "-f", "-d", mirror_dir.strpath, "b%gcc@12.2.0") buildcache_cmd("push", "-u", "-a", "-f", "-d", mirror_dir.strpath, "b%gcc@12.2.0")
# Now uninstall the spec # Now uninstall the spec
uninstall_cmd("-y", "b%gcc@12.2.0") uninstall_cmd("-y", "b%gcc@12.2.0")

4
lib/spack/spack/test/cmd/install.py
View File

@ -969,7 +969,7 @@ def test_compiler_bootstrap_from_binary_mirror(
install("gcc@10.2.0") install("gcc@10.2.0")
# Put installed compiler in the buildcache # Put installed compiler in the buildcache
buildcache("create", "-u", "-a", "-f", "-d", mirror_dir.strpath, "gcc@10.2.0") buildcache("push", "-u", "-a", "-f", "-d", mirror_dir.strpath, "gcc@10.2.0")
# Now uninstall the compiler # Now uninstall the compiler
uninstall("-y", "gcc@10.2.0") uninstall("-y", "gcc@10.2.0")
@ -1134,7 +1134,7 @@ def install_use_buildcache(opt):
# Populate the buildcache # Populate the buildcache
install(package_name) install(package_name)
buildcache("create", "-u", "-a", "-f", "-d", mirror_dir.strpath, package_name, dependency_name) buildcache("push", "-u", "-a", "-f", "-d", mirror_dir.strpath, package_name, dependency_name)
# Uninstall the all of the packages for clean slate # Uninstall the all of the packages for clean slate
uninstall("-y", "-a") uninstall("-y", "-a")

4
lib/spack/spack/test/cmd/mirror.py
View File

@ -235,7 +235,7 @@ def test_mirror_destroy(
# Put a binary package in a buildcache # Put a binary package in a buildcache
install("--no-cache", spec_name) install("--no-cache", spec_name)
buildcache("create", "-u", "-a", "-f", "-d", mirror_dir.strpath, spec_name) buildcache("push", "-u", "-a", "-f", "-d", mirror_dir.strpath, spec_name)
contents = os.listdir(mirror_dir.strpath) contents = os.listdir(mirror_dir.strpath)
assert "build_cache" in contents assert "build_cache" in contents
@ -245,7 +245,7 @@ def test_mirror_destroy(
assert not os.path.exists(mirror_dir.strpath) assert not os.path.exists(mirror_dir.strpath)
buildcache("create", "-u", "-a", "-f", "-d", mirror_dir.strpath, spec_name) buildcache("push", "-u", "-a", "-f", "-d", mirror_dir.strpath, spec_name)
contents = os.listdir(mirror_dir.strpath) contents = os.listdir(mirror_dir.strpath)
assert "build_cache" in contents assert "build_cache" in contents

22
share/spack/spack-completion.bash
View File

@ -491,14 +491,23 @@ _spack_buildcache() {
then then
SPACK_COMPREPLY="-h --help" SPACK_COMPREPLY="-h --help"
else else
SPACK_COMPREPLY="create install list keys preview check download get-buildcache-name save-specfile sync update-index" SPACK_COMPREPLY="push create install list keys preview check download get-buildcache-name save-specfile sync update-index rebuild-index"
fi
}
_spack_buildcache_push() {
if $list_options
then
SPACK_COMPREPLY="-h --help -r --rel -f --force -u --unsigned -a --allow-root -k --key -d --directory -m --mirror-name --mirror-url --update-index --rebuild-index --spec-file --only"
else
_mirrors
fi fi
} }
_spack_buildcache_create() { _spack_buildcache_create() {
if $list_options if $list_options
then then
SPACK_COMPREPLY="-h --help -r --rel -f --force -u --unsigned -a --allow-root -k --key -d --directory -m --mirror-name --mirror-url --rebuild-index --spec-file --only" SPACK_COMPREPLY="-h --help -r --rel -f --force -u --unsigned -a --allow-root -k --key -d --directory -m --mirror-name --mirror-url --update-index --rebuild-index --spec-file --only"
else else
_mirrors _mirrors
fi fi
@ -569,6 +578,15 @@ _spack_buildcache_update_index() {
fi fi
} }
_spack_buildcache_rebuild_index() {
if $list_options
then
SPACK_COMPREPLY="-h --help -d --directory -m --mirror-name --mirror-url -k --keys"
else
_mirrors
fi
}
_spack_cd() { _spack_cd() {
if $list_options if $list_options
then then