Allow packages to be pushed to build cache after install from source (#42423)

This commit adds a property `autopush` to mirrors. When true, every source build is immediately followed by a push to the build cache. This is useful in ephemeral environments such as CI / containers.

To enable autopush on existing build caches, use `spack mirror set --autopush <name>`. The same flag can be used in `spack mirror add`.

lib/spack/docs/binary_caches.rst

@@ -220,6 +220,40 @@ section of the configuration:
 
 .. _binary_caches_oci:
 
+---------------------------------
+Automatic push to a build cache
+---------------------------------
+
+Sometimes it is convenient to push packages to a build cache as soon as they are installed. Spack can do this by setting autopush flag when adding a mirror:
+
+.. code-block:: console
+
+    $ spack mirror add --autopush <name> <url or path>
+
+Or the autopush flag can be set for an existing mirror:
+
+.. code-block:: console
+
+    $ spack mirror set --autopush <name>  # enable automatic push for an existing mirror
+    $ spack mirror set --no-autopush <name>  # disable automatic push for an existing mirror
+
+Then after installing a package it is automatically pushed to all mirrors with ``autopush: true``. The command
+
+.. code-block:: console
+
+    $ spack install <package>
+
+will have the same effect as
+
+.. code-block:: console
+
+    $ spack install <package>
+    $ spack buildcache push <cache> <package>  # for all caches with autopush: true
+
+.. note::
+
+    Packages are automatically pushed to a build cache only if they are built from source.
+
 -----------------------------------------
 OCI / Docker V2 registries as build cache
 -----------------------------------------

lib/spack/spack/cmd/mirror.py

@@ -108,6 +108,11 @@ def setup_parser(subparser):
             "and source use `--type binary --type source` (default)"
         ),
     )
+    add_parser.add_argument(
+        "--autopush",
+        action="store_true",
+        help=("set mirror to push automatically after installation"),
+    )
     add_parser_signed = add_parser.add_mutually_exclusive_group(required=False)
     add_parser_signed.add_argument(
         "--unsigned",
@@ -175,6 +180,21 @@ def setup_parser(subparser):
         ),
     )
     set_parser.add_argument("--url", help="url of mirror directory from 'spack mirror create'")
+    set_parser_autopush = set_parser.add_mutually_exclusive_group(required=False)
+    set_parser_autopush.add_argument(
+        "--autopush",
+        help="set mirror to push automatically after installation",
+        action="store_true",
+        default=None,
+        dest="autopush",
+    )
+    set_parser_autopush.add_argument(
+        "--no-autopush",
+        help="set mirror to not push automatically after installation",
+        action="store_false",
+        default=None,
+        dest="autopush",
+    )
     set_parser_unsigned = set_parser.add_mutually_exclusive_group(required=False)
     set_parser_unsigned.add_argument(
         "--unsigned",
@@ -218,6 +238,7 @@ def mirror_add(args):
         or args.type
         or args.oci_username
         or args.oci_password
+        or args.autopush
         or args.signed is not None
     ):
         connection = {"url": args.url}
@@ -234,6 +255,8 @@ def mirror_add(args):
         if args.type:
             connection["binary"] = "binary" in args.type
             connection["source"] = "source" in args.type
+        if args.autopush:
+            connection["autopush"] = args.autopush
         if args.signed is not None:
             connection["signed"] = args.signed
         mirror = spack.mirror.Mirror(connection, name=args.name)
@@ -270,6 +293,8 @@ def _configure_mirror(args):
         changes["access_pair"] = [args.oci_username, args.oci_password]
     if getattr(args, "signed", None) is not None:
         changes["signed"] = args.signed
+    if getattr(args, "autopush", None) is not None:
+        changes["autopush"] = args.autopush
 
     # argparse cannot distinguish between --binary and --no-binary when same dest :(
     # notice that set-url does not have these args, so getattr

lib/spack/spack/hooks/autopush.py

@@ -0,0 +1,27 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import llnl.util.tty as tty
+
+import spack.binary_distribution as bindist
+import spack.mirror
+
+
+def post_install(spec, explicit):
+    # Push package to all buildcaches with autopush==True
+
+    # Do nothing if package was not installed from source
+    pkg = spec.package
+    if pkg.installed_from_binary_cache:
+        return
+
+    # Push the package to all autopush mirrors
+    for mirror in spack.mirror.MirrorCollection(binary=True, autopush=True).values():
+        bindist.push_or_raise(
+            spec,
+            mirror.push_url,
+            bindist.PushOptions(force=True, regenerate_index=False, unsigned=not mirror.signed),
+        )
+        tty.msg(f"{spec.name}: Pushed to build cache: '{mirror.name}'")

lib/spack/spack/mirror.py

@@ -137,6 +137,12 @@ def source(self):
     def signed(self) -> bool:
         return isinstance(self._data, str) or self._data.get("signed", True)
 
+    @property
+    def autopush(self) -> bool:
+        if isinstance(self._data, str):
+            return False
+        return self._data.get("autopush", False)
+
     @property
     def fetch_url(self):
         """Get the valid, canonicalized fetch URL"""
@@ -150,7 +156,7 @@ def push_url(self):
     def _update_connection_dict(self, current_data: dict, new_data: dict, top_level: bool):
         keys = ["url", "access_pair", "access_token", "profile", "endpoint_url"]
         if top_level:
-            keys += ["binary", "source", "signed"]
+            keys += ["binary", "source", "signed", "autopush"]
         changed = False
         for key in keys:
             if key in new_data and current_data.get(key) != new_data[key]:
@@ -286,6 +292,7 @@ def __init__(
         scope=None,
         binary: Optional[bool] = None,
         source: Optional[bool] = None,
+        autopush: Optional[bool] = None,
     ):
         """Initialize a mirror collection.
 
@@ -297,21 +304,27 @@ def __init__(
                     If None, do not filter on binary mirrors.
             source: If True, only include source mirrors.
                     If False, omit source mirrors.
-                    If None, do not filter on source mirrors."""
+                    If None, do not filter on source mirrors.
-        self._mirrors = {
+            autopush: If True, only include mirrors that have autopush enabled.
-            name: Mirror(data=mirror, name=name)
+                      If False, omit mirrors that have autopush enabled.
-            for name, mirror in (
+                      If None, do not filter on autopush."""
+        mirrors_data = (
             mirrors.items()
             if mirrors is not None
             else spack.config.get("mirrors", scope=scope).items()
         )
-        }
+        mirrors = (Mirror(data=mirror, name=name) for name, mirror in mirrors_data)
 
-        if source is not None:
+        def _filter(m: Mirror):
-            self._mirrors = {k: v for k, v in self._mirrors.items() if v.source == source}
+            if source is not None and m.source != source:
+                return False
+            if binary is not None and m.binary != binary:
+                return False
+            if autopush is not None and m.autopush != autopush:
+                return False
+            return True
 
-        if binary is not None:
+        self._mirrors = {m.name: m for m in mirrors if _filter(m)}
-            self._mirrors = {k: v for k, v in self._mirrors.items() if v.binary == binary}
 
     def __eq__(self, other):
         return self._mirrors == other._mirrors

lib/spack/spack/schema/mirrors.py

@@ -46,6 +46,7 @@
         "signed": {"type": "boolean"},
         "fetch": fetch_and_push,
         "push": fetch_and_push,
+        "autopush": {"type": "boolean"},
         **connection,  # type: ignore
     },
 }

lib/spack/spack/test/cmd/buildcache.py

@@ -169,6 +169,25 @@ def test_update_key_index(
     assert "index.json" in key_dir_list
 
 
+def test_buildcache_autopush(tmp_path, install_mockery, mock_fetch):
+    """Test buildcache with autopush"""
+    mirror_dir = tmp_path / "mirror"
+    mirror_autopush_dir = tmp_path / "mirror_autopush"
+
+    mirror("add", "--unsigned", "mirror", mirror_dir.as_uri())
+    mirror("add", "--autopush", "--unsigned", "mirror-autopush", mirror_autopush_dir.as_uri())
+
+    s = Spec("libdwarf").concretized()
+
+    # Install and generate build cache index
+    s.package.do_install()
+
+    metadata_file = spack.binary_distribution.tarball_name(s, ".spec.json")
+
+    assert not (mirror_dir / "build_cache" / metadata_file).exists()
+    assert (mirror_autopush_dir / "build_cache" / metadata_file).exists()
+
+
 def test_buildcache_sync(
     mutable_mock_env_path,
     install_mockery_mutable_config,

lib/spack/spack/test/cmd/mirror.py

@@ -407,3 +407,27 @@ def test_mirror_add_set_signed(mutable_config):
     assert spack.config.get("mirrors:example") == {"url": "http://example.com", "signed": False}
     mirror("set", "--signed", "example")
     assert spack.config.get("mirrors:example") == {"url": "http://example.com", "signed": True}
+
+
+def test_mirror_add_set_autopush(mutable_config):
+    # Add mirror without autopush
+    mirror("add", "example", "http://example.com")
+    assert spack.config.get("mirrors:example") == "http://example.com"
+    mirror("set", "--no-autopush", "example")
+    assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": False}
+    mirror("set", "--autopush", "example")
+    assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True}
+    mirror("set", "--no-autopush", "example")
+    assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": False}
+    mirror("remove", "example")
+
+    # Add mirror with autopush
+    mirror("add", "--autopush", "example", "http://example.com")
+    assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True}
+    mirror("set", "--autopush", "example")
+    assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True}
+    mirror("set", "--no-autopush", "example")
+    assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": False}
+    mirror("set", "--autopush", "example")
+    assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True}
+    mirror("remove", "example")

share/spack/spack-completion.bash

@@ -1441,7 +1441,7 @@ _spack_mirror_destroy() {
 _spack_mirror_add() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --scope --type --unsigned --signed --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url --oci-username --oci-password"
+        SPACK_COMPREPLY="-h --help --scope --type --autopush --unsigned --signed --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url --oci-username --oci-password"
     else
         _mirrors
     fi
@@ -1477,7 +1477,7 @@ _spack_mirror_set_url() {
 _spack_mirror_set() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --push --fetch --type --url --unsigned --signed --scope --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url --oci-username --oci-password"
+        SPACK_COMPREPLY="-h --help --push --fetch --type --url --autopush --no-autopush --unsigned --signed --scope --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url --oci-username --oci-password"
     else
         _mirrors
     fi

share/spack/spack-completion.fish

@@ -2253,7 +2253,7 @@ complete -c spack -n '__fish_spack_using_command mirror destroy' -l mirror-url -
 complete -c spack -n '__fish_spack_using_command mirror destroy' -l mirror-url -r -d 'find mirror to destroy by url'
 
 # spack mirror add
-set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= unsigned signed s3-access-key-id= s3-access-key-secret= s3-access-token= s3-profile= s3-endpoint-url= oci-username= oci-password=
+set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsigned signed s3-access-key-id= s3-access-key-secret= s3-access-token= s3-profile= s3-endpoint-url= oci-username= oci-password=
 complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f
 complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help
 complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit'
@@ -2261,6 +2261,8 @@ complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '
 complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify'
 complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source'
 complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source` (default)'
+complete -c spack -n '__fish_spack_using_command mirror add' -l autopush -f -a autopush
+complete -c spack -n '__fish_spack_using_command mirror add' -l autopush -d 'set mirror to push automatically after installation'
 complete -c spack -n '__fish_spack_using_command mirror add' -l unsigned -f -a signed
 complete -c spack -n '__fish_spack_using_command mirror add' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache'
 complete -c spack -n '__fish_spack_using_command mirror add' -l signed -f -a signed
@@ -2323,7 +2325,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l oci-password
 complete -c spack -n '__fish_spack_using_command mirror set-url' -l oci-password -r -d 'password to use to connect to this OCI mirror'
 
 # spack mirror set
-set -g __fish_spack_optspecs_spack_mirror_set h/help push fetch type= url= unsigned signed scope= s3-access-key-id= s3-access-key-secret= s3-access-token= s3-profile= s3-endpoint-url= oci-username= oci-password=
+set -g __fish_spack_optspecs_spack_mirror_set h/help push fetch type= url= autopush no-autopush unsigned signed scope= s3-access-key-id= s3-access-key-secret= s3-access-token= s3-profile= s3-endpoint-url= oci-username= oci-password=
 complete -c spack -n '__fish_spack_using_command_pos 0 mirror set' -f -a '(__fish_spack_mirrors)'
 complete -c spack -n '__fish_spack_using_command mirror set' -s h -l help -f -a help
 complete -c spack -n '__fish_spack_using_command mirror set' -s h -l help -d 'show this help message and exit'
@@ -2335,6 +2337,10 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l type -r -f -a 'b
 complete -c spack -n '__fish_spack_using_command mirror set' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source`'
 complete -c spack -n '__fish_spack_using_command mirror set' -l url -r -f -a url
 complete -c spack -n '__fish_spack_using_command mirror set' -l url -r -d 'url of mirror directory from \'spack mirror create\''
+complete -c spack -n '__fish_spack_using_command mirror set' -l autopush -f -a autopush
+complete -c spack -n '__fish_spack_using_command mirror set' -l autopush -d 'set mirror to push automatically after installation'
+complete -c spack -n '__fish_spack_using_command mirror set' -l no-autopush -f -a autopush
+complete -c spack -n '__fish_spack_using_command mirror set' -l no-autopush -d 'set mirror to not push automatically after installation'
 complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a signed
 complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache'
 complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed