diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index d10de8081df..edcbec63019 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -120,6 +120,14 @@ what it looks like: Once this is done, you can tar up the ``spack-mirror-2014-06-24`` directory and copy it over to the machine you want it hosted on. +Customization of the mirror contents can be done by selectively excluding +specs using the ``--exclude-file`` or ``--exclude-specs`` flags with +``spack mirror create``. You may additionally add an ``exclude`` or ``include`` +section to the ``mirrors`` configuration section. These are lists of abstract +or concrete specs to configure what gets pushed to your mirror. +If overlapping inclusion and exclusions are applied then inclusion is preferred. + + ^^^^^^^^^^^^^^^^^^^ Custom package sets ^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 167e2129a97..a13737ea697 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -1230,6 +1230,8 @@ def push( force=self.force, tmpdir=self.tmpdir, executor=self.executor, + exclusions=self.mirror.exclusions, + inclusions=self.mirror.inclusions, ) self._base_images = base_images @@ -1280,6 +1282,8 @@ def push( signing_key=self.signing_key, tmpdir=self.tmpdir, executor=self.executor, + exclusions=self.mirror.exclusions, + inclusions=self.mirror.inclusions, ) @@ -1342,6 +1346,34 @@ def fail(self) -> None: tty.info(f"{self.pre}Failed to push {self.pretty_spec}") +def _filter_specs(specs: List[spack.spec.Spec], exclude: List[str], include: List[str]): + """ + Determine the intersection of include/exclude filters + Tie goes to keeping + + skip | keep | outcome + ------------------------ + False | False | Keep + True | True | Keep + False | True | Keep + True | False | Skip + """ + filter = [] + filtrate = [] + ex_specs = [spack.spec.Spec(spec) for spec in exclude] + ic_specs = [spack.spec.Spec(spec) for spec in include] + for spec in specs: + skip = any([spec.satisfies(test) for test in ex_specs]) + keep = any([spec.satisfies(test) for test in ic_specs]) + + if skip and not keep: + filtrate.append(spec) + else: + filter.append(spec) + + return filter, filtrate + + def _url_push( specs: List[spack.spec.Spec], out_url: str, @@ -1350,6 +1382,8 @@ def _url_push( update_index: bool, tmpdir: str, executor: concurrent.futures.Executor, + exclusions: List[str] = [], + inclusions: List[str] = [], ) -> Tuple[List[spack.spec.Spec], List[Tuple[spack.spec.Spec, BaseException]]]: """Pushes to the provided build cache, and returns a list of skipped specs that were already present (when force=False), and a list of errors. Does not raise on error.""" @@ -1380,6 +1414,11 @@ def _url_push( if not specs_to_upload: return skipped, errors + filter, filtrate = _filter_specs(specs_to_upload, exclusions, inclusions) + + skipped.extend(filtrate) + specs_to_upload = filter + total = len(specs_to_upload) if total != len(specs): @@ -1647,6 +1686,8 @@ def _oci_push( tmpdir: str, executor: concurrent.futures.Executor, force: bool = False, + exclusions: List[str] = [], + inclusions: List[str] = [], ) -> Tuple[ List[spack.spec.Spec], Dict[str, Tuple[dict, dict]], @@ -1683,6 +1724,11 @@ def _oci_push( if not blobs_to_upload: return skipped, base_images, checksums, [] + filter, filtrate = _filter_specs(blobs_to_upload, exclusions, inclusions) + + skipped.extend(filtrate) + blobs_to_upload = filter + if len(blobs_to_upload) != len(installed_specs_with_deps): tty.info( f"{len(blobs_to_upload)} specs need to be pushed to " diff --git a/lib/spack/spack/mirrors/mirror.py b/lib/spack/spack/mirrors/mirror.py index 7f3527a28c2..3ca97186cb9 100644 --- a/lib/spack/spack/mirrors/mirror.py +++ b/lib/spack/spack/mirrors/mirror.py @@ -5,7 +5,7 @@ import operator import os import urllib.parse -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import llnl.util.tty as tty @@ -99,6 +99,11 @@ def display(self, max_len=0): binary = "b" if self.binary else " " print(f"{self.name: <{max_len}} [{source}{binary}] {url}") + def _process_spec_filters(self, key: str) -> List[str]: + if isinstance(self._data, str): + return [] + return self._data.get(key, []) + @property def name(self): return self._name or "" @@ -131,6 +136,14 @@ def push_url(self): """Get the valid, canonicalized fetch URL""" return self.get_url("push") + @property + def exclusions(self): + return self._process_spec_filters("exclude") + + @property + def inclusions(self): + return self._process_spec_filters("include") + def ensure_mirror_usable(self, direction: str = "push"): access_pair = self._get_value("access_pair", direction) access_token_variable = self._get_value("access_token_variable", direction) diff --git a/lib/spack/spack/schema/mirrors.py b/lib/spack/spack/schema/mirrors.py index 45e1f5adbf9..1227d715506 100644 --- a/lib/spack/spack/schema/mirrors.py +++ b/lib/spack/spack/schema/mirrors.py @@ -77,6 +77,8 @@ "fetch": fetch_and_push, "push": fetch_and_push, "autopush": {"type": "boolean"}, + "exclude": {"type": "array", "items": {"type": "string"}}, + "include": {"type": "array", "items": {"type": "string"}}, **connection, # type: ignore }, **connection_ext, # type: ignore diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index a0c6458719d..fdc0b88019f 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -434,3 +434,14 @@ def test_mirror_name_or_url_dir_parsing(tmp_path): with working_dir(curdir): assert mirror_name_or_url(".").fetch_url == curdir.as_uri() assert mirror_name_or_url("..").fetch_url == tmp_path.as_uri() + + +def test_mirror_parse_exclude_include(): + mirror_raw = { + "url": "https://example.com", + "exclude": ["dev_path=*", "+shared"], + "include": ["+foo"], + } + m = spack.mirrors.mirror.Mirror(mirror_raw) + assert "dev_path=*" in m.exclusions + assert "+foo" in m.inclusions