Add configurable filters to mirrors

Add the ability to filter what specs get pushed to a mirror.
This allows for a more controlled autopush for binary mirrors.

lib/spack/docs/mirrors.rst

@@ -120,6 +120,14 @@ what it looks like:
 Once this is done, you can tar up the ``spack-mirror-2014-06-24`` directory and
 copy it over to the machine you want it hosted on.
 
+Customization of the mirror contents can be done by selectively excluding
+specs using the ``--exclude-file`` or ``--exclude-specs`` flags with 
+``spack mirror create``.  You may additionally add an ``exclude`` or ``include``
+section to the ``mirrors`` configuration section. These are lists of abstract 
+or concrete specs to configure what gets pushed to your mirror.
+If overlapping inclusion and exclusions are applied then inclusion is preferred.
+
+
 ^^^^^^^^^^^^^^^^^^^
 Custom package sets
 ^^^^^^^^^^^^^^^^^^^

lib/spack/spack/binary_distribution.py

@@ -1230,6 +1230,8 @@ def push(
             force=self.force,
             tmpdir=self.tmpdir,
             executor=self.executor,
+            exclusions=self.mirror.exclusions,
+            inclusions=self.mirror.inclusions,
         )
 
         self._base_images = base_images
@@ -1280,6 +1282,8 @@ def push(
             signing_key=self.signing_key,
             tmpdir=self.tmpdir,
             executor=self.executor,
+            exclusions=self.mirror.exclusions,
+            inclusions=self.mirror.inclusions,
         )
 
 
@@ -1342,6 +1346,34 @@ def fail(self) -> None:
         tty.info(f"{self.pre}Failed to push {self.pretty_spec}")
 
 
+def _filter_specs(specs: List[spack.spec.Spec], exclude: List[str], include: List[str]):
+    """
+    Determine the intersection of include/exclude filters
+    Tie goes to keeping
+
+    skip  | keep  | outcome
+    ------------------------
+    False | False | Keep
+    True  | True  | Keep
+    False | True  | Keep
+    True  | False | Skip
+    """
+    filter = []
+    filtrate = []
+    ex_specs = [spack.spec.Spec(spec) for spec in exclude]
+    ic_specs = [spack.spec.Spec(spec) for spec in include]
+    for spec in specs:
+        skip = any([spec.satisfies(test) for test in ex_specs])
+        keep = any([spec.satisfies(test) for test in ic_specs])
+
+        if skip and not keep:
+            filtrate.append(spec)
+        else:
+            filter.append(spec)
+
+    return filter, filtrate
+
+
 def _url_push(
     specs: List[spack.spec.Spec],
     out_url: str,
@@ -1350,6 +1382,8 @@ def _url_push(
     update_index: bool,
     tmpdir: str,
     executor: concurrent.futures.Executor,
+    exclusions: List[str] = [],
+    inclusions: List[str] = [],
 ) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]:
     """Pushes to the provided build cache, and returns a list of skipped specs that were already
     present (when force=False), and a list of errors. Does not raise on error."""
@@ -1380,6 +1414,11 @@ def _url_push(
     if not specs_to_upload:
         return skipped, errors
 
+    filter, filtrate = _filter_specs(specs_to_upload, exclusions, inclusions)
+
+    skipped.extend(filtrate)
+    specs_to_upload = filter
+
     total = len(specs_to_upload)
 
     if total != len(specs):
@@ -1647,6 +1686,8 @@ def _oci_push(
     tmpdir: str,
     executor: concurrent.futures.Executor,
     force: bool = False,
+    exclusions: List[str] = [],
+    inclusions: List[str] = [],
 ) -> Tuple[
     List[spack.spec.Spec],
     Dict[str, Tuple[dict, dict]],
@@ -1683,6 +1724,11 @@ def _oci_push(
     if not blobs_to_upload:
         return skipped, base_images, checksums, []
 
+    filter, filtrate = _filter_specs(blobs_to_upload, exclusions, inclusions)
+
+    skipped.extend(filtrate)
+    blobs_to_upload = filter
+
     if len(blobs_to_upload) != len(installed_specs_with_deps):
         tty.info(
             f"{len(blobs_to_upload)} specs need to be pushed to "

lib/spack/spack/mirrors/mirror.py

@@ -5,7 +5,7 @@
 import operator
 import os
 import urllib.parse
-from typing import Any, Dict, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import llnl.util.tty as tty
 
@@ -99,6 +99,11 @@ def display(self, max_len=0):
         binary = "b" if self.binary else " "
         print(f"{self.name: <{max_len}} [{source}{binary}] {url}")
 
+    def _process_spec_filters(self, key: str) -> List[str]:
+        if isinstance(self._data, str):
+            return []
+        return self._data.get(key, [])
+
     @property
     def name(self):
         return self._name or "<unnamed>"
@@ -131,6 +136,14 @@ def push_url(self):
         """Get the valid, canonicalized fetch URL"""
         return self.get_url("push")
 
+    @property
+    def exclusions(self):
+        return self._process_spec_filters("exclude")
+
+    @property
+    def inclusions(self):
+        return self._process_spec_filters("include")
+
     def ensure_mirror_usable(self, direction: str = "push"):
         access_pair = self._get_value("access_pair", direction)
         access_token_variable = self._get_value("access_token_variable", direction)

lib/spack/spack/schema/mirrors.py

@@ -77,6 +77,8 @@
         "fetch": fetch_and_push,
         "push": fetch_and_push,
         "autopush": {"type": "boolean"},
+        "exclude": {"type": "array", "items": {"type": "string"}},
+        "include": {"type": "array", "items": {"type": "string"}},
         **connection,  # type: ignore
     },
     **connection_ext,  # type: ignore

lib/spack/spack/test/mirror.py

@@ -434,3 +434,14 @@ def test_mirror_name_or_url_dir_parsing(tmp_path):
     with working_dir(curdir):
         assert mirror_name_or_url(".").fetch_url == curdir.as_uri()
         assert mirror_name_or_url("..").fetch_url == tmp_path.as_uri()
+
+
+def test_mirror_parse_exclude_include():
+    mirror_raw = {
+        "url": "https://example.com",
+        "exclude": ["dev_path=*", "+shared"],
+        "include": ["+foo"],
+    }
+    m = spack.mirrors.mirror.Mirror(mirror_raw)
+    assert "dev_path=*" in m.exclusions
+    assert "+foo" in m.inclusions