zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow packages to be pushed to build cache after install from source (#42423)

Browse Source 
This commit adds a property `autopush` to mirrors. When true, every source build is immediately followed by a push to the build cache. This is useful in ephemeral environments such as CI / containers.

To enable autopush on existing build caches, use `spack mirror set --autopush <name>`. The same flag can be used in `spack mirror add`.
This commit is contained in:
Radim Janalík
2024-04-12 03:43:13 +02:00
committed by GitHub
parent b76e9a887b
commit d23e06c27e
9 changed files with 167 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
lib/spack/docs/binary_caches.rst
View File

@@ -220,6 +220,40 @@ section of the configuration:
.. _binary_caches_oci:
---------------------------------
Automatic push to a build cache
---------------------------------
Sometimes it is convenient to push packages to a build cache as soon as they are installed. Spack can do this by setting autopush flag when adding a mirror:
.. code-block:: console
$ spack mirror add --autopush <name> <url or path>
Or the autopush flag can be set for an existing mirror:
.. code-block:: console
$ spack mirror set --autopush <name> # enable automatic push for an existing mirror
$ spack mirror set --no-autopush <name> # disable automatic push for an existing mirror
Then after installing a package it is automatically pushed to all mirrors with ``autopush: true``. The command
.. code-block:: console
$ spack install <package>
will have the same effect as
.. code-block:: console
$ spack install <package>
$ spack buildcache push <cache> <package> # for all caches with autopush: true
.. note::
Packages are automatically pushed to a build cache only if they are built from source.
-----------------------------------------
OCI / Docker V2 registries as build cache
-----------------------------------------

25
lib/spack/spack/cmd/mirror.py
View File

@@ -108,6 +108,11 @@ def setup_parser(subparser):
"and source use `--type binary --type source` (default)"
),
)
add_parser.add_argument(
"--autopush",
action="store_true",
help=("set mirror to push automatically after installation"),
)
add_parser_signed = add_parser.add_mutually_exclusive_group(required=False)
add_parser_signed.add_argument(
"--unsigned",
@@ -175,6 +180,21 @@ def setup_parser(subparser):
),
)
set_parser.add_argument("--url", help="url of mirror directory from 'spack mirror create'")
set_parser_autopush = set_parser.add_mutually_exclusive_group(required=False)
set_parser_autopush.add_argument(
"--autopush",
help="set mirror to push automatically after installation",
action="store_true",
default=None,
dest="autopush",
)
set_parser_autopush.add_argument(
"--no-autopush",
help="set mirror to not push automatically after installation",
action="store_false",
default=None,
dest="autopush",
)
set_parser_unsigned = set_parser.add_mutually_exclusive_group(required=False)
set_parser_unsigned.add_argument(
"--unsigned",
@@ -218,6 +238,7 @@ def mirror_add(args):
or args.type
or args.oci_username
or args.oci_password
or args.autopush
or args.signed is not None
):
connection = {"url": args.url}
@@ -234,6 +255,8 @@ def mirror_add(args):
if args.type:
connection["binary"] = "binary" in args.type
connection["source"] = "source" in args.type
if args.autopush:
connection["autopush"] = args.autopush
if args.signed is not None:
connection["signed"] = args.signed
mirror = spack.mirror.Mirror(connection, name=args.name)
@@ -270,6 +293,8 @@ def _configure_mirror(args):
changes["access_pair"] = [args.oci_username, args.oci_password]
if getattr(args, "signed", None) is not None:
changes["signed"] = args.signed
if getattr(args, "autopush", None) is not None:
changes["autopush"] = args.autopush
# argparse cannot distinguish between --binary and --no-binary when same dest :(
# notice that set-url does not have these args, so getattr

27
lib/spack/spack/hooks/autopush.py Normal file
View File

@@ -0,0 +1,27 @@
# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import llnl.util.tty as tty
import spack.binary_distribution as bindist
import spack.mirror
def post_install(spec, explicit):
# Push package to all buildcaches with autopush==True
# Do nothing if package was not installed from source
pkg = spec.package
if pkg.installed_from_binary_cache:
return
# Push the package to all autopush mirrors
for mirror in spack.mirror.MirrorCollection(binary=True, autopush=True).values():
bindist.push_or_raise(
spec,
mirror.push_url,
bindist.PushOptions(force=True, regenerate_index=False, unsigned=not mirror.signed),
)
tty.msg(f"{spec.name}: Pushed to build cache: '{mirror.name}'")

41
lib/spack/spack/mirror.py
View File

@@ -137,6 +137,12 @@ def source(self):
def signed(self) -> bool:
return isinstance(self._data, str) or self._data.get("signed", True)
@property
def autopush(self) -> bool:
if isinstance(self._data, str):
return False
return self._data.get("autopush", False)
@property
def fetch_url(self):
"""Get the valid, canonicalized fetch URL"""
@@ -150,7 +156,7 @@ def push_url(self):
def _update_connection_dict(self, current_data: dict, new_data: dict, top_level: bool):
keys = ["url", "access_pair", "access_token", "profile", "endpoint_url"]
if top_level:
keys += ["binary", "source", "signed"]
keys += ["binary", "source", "signed", "autopush"]
changed = False
for key in keys:
if key in new_data and current_data.get(key) != new_data[key]:
@@ -286,6 +292,7 @@ def __init__(
scope=None,
binary: Optional[bool] = None,
source: Optional[bool] = None,
autopush: Optional[bool] = None,
):
"""Initialize a mirror collection.
@@ -297,21 +304,27 @@ def __init__(
If None, do not filter on binary mirrors.
source: If True, only include source mirrors.
If False, omit source mirrors.
If None, do not filter on source mirrors."""
self._mirrors = {
name: Mirror(data=mirror, name=name)
for name, mirror in (
mirrors.items()
if mirrors is not None
else spack.config.get("mirrors", scope=scope).items()
)
}
If None, do not filter on source mirrors.
autopush: If True, only include mirrors that have autopush enabled.
If False, omit mirrors that have autopush enabled.
If None, do not filter on autopush."""
mirrors_data = (
mirrors.items()
if mirrors is not None
else spack.config.get("mirrors", scope=scope).items()
)
mirrors = (Mirror(data=mirror, name=name) for name, mirror in mirrors_data)
if source is not None:
self._mirrors = {k: v for k, v in self._mirrors.items() if v.source == source}
def _filter(m: Mirror):
if source is not None and m.source != source:
return False
if binary is not None and m.binary != binary:
return False
if autopush is not None and m.autopush != autopush:
return False
return True
if binary is not None:
self._mirrors = {k: v for k, v in self._mirrors.items() if v.binary == binary}
self._mirrors = {m.name: m for m in mirrors if _filter(m)}
def __eq__(self, other):
return self._mirrors == other._mirrors

1
lib/spack/spack/schema/mirrors.py
View File

@@ -46,6 +46,7 @@
"signed": {"type": "boolean"},
"fetch": fetch_and_push,
"push": fetch_and_push,
"autopush": {"type": "boolean"},
**connection, # type: ignore
},
}

19
lib/spack/spack/test/cmd/buildcache.py
View File

@@ -169,6 +169,25 @@ def test_update_key_index(
assert "index.json" in key_dir_list
def test_buildcache_autopush(tmp_path, install_mockery, mock_fetch):
"""Test buildcache with autopush"""
mirror_dir = tmp_path / "mirror"
mirror_autopush_dir = tmp_path / "mirror_autopush"
mirror("add", "--unsigned", "mirror", mirror_dir.as_uri())
mirror("add", "--autopush", "--unsigned", "mirror-autopush", mirror_autopush_dir.as_uri())
s = Spec("libdwarf").concretized()
# Install and generate build cache index
s.package.do_install()
metadata_file = spack.binary_distribution.tarball_name(s, ".spec.json")
assert not (mirror_dir / "build_cache" / metadata_file).exists()
assert (mirror_autopush_dir / "build_cache" / metadata_file).exists()
def test_buildcache_sync(
mutable_mock_env_path,
install_mockery_mutable_config,

24
lib/spack/spack/test/cmd/mirror.py
View File

@@ -407,3 +407,27 @@ def test_mirror_add_set_signed(mutable_config):
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "signed": False}
mirror("set", "--signed", "example")
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "signed": True}
def test_mirror_add_set_autopush(mutable_config):
# Add mirror without autopush
mirror("add", "example", "http://example.com")
assert spack.config.get("mirrors:example") == "http://example.com"
mirror("set", "--no-autopush", "example")
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": False}
mirror("set", "--autopush", "example")
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True}
mirror("set", "--no-autopush", "example")
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": False}
mirror("remove", "example")
# Add mirror with autopush
mirror("add", "--autopush", "example", "http://example.com")
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True}
mirror("set", "--autopush", "example")
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True}
mirror("set", "--no-autopush", "example")
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": False}
mirror("set", "--autopush", "example")
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True}
mirror("remove", "example")