From 8d9af73d83235911eaa725062cf524084e4990f4 Mon Sep 17 00:00:00 2001 From: Philip Sakievich Date: Wed, 16 Apr 2025 18:09:53 -0600 Subject: [PATCH] Tests and parsing --- lib/spack/docs/mirrors.rst | 8 +++++--- lib/spack/spack/cmd/mirror.py | 19 +++++++++++++++++++ lib/spack/spack/mirrors/mirror.py | 2 +- lib/spack/spack/test/cmd/buildcache.py | 26 ++++++++++++++++++++++++++ lib/spack/spack/test/mirror.py | 9 +++++++++ 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index edcbec63019..1402de77675 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -122,9 +122,11 @@ copy it over to the machine you want it hosted on. Customization of the mirror contents can be done by selectively excluding specs using the ``--exclude-file`` or ``--exclude-specs`` flags with -``spack mirror create``. You may additionally add an ``exclude`` or ``include`` -section to the ``mirrors`` configuration section. These are lists of abstract -or concrete specs to configure what gets pushed to your mirror. +``spack mirror create``. Note that these only apply to source mirrors. + +You may additionally add an ``exclude`` or ``include`` +section to the ``mirrors`` configuration section for pushing to binary mirrors. +These are lists of abstract or concrete specs to configure what gets pushed to your mirror. If overlapping inclusion and exclusions are applied then inclusion is preferred. diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 512931e11b4..18186da5c3c 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -134,6 +134,15 @@ def setup_parser(subparser): default=None, dest="signed", ) + add_parser.add_argument( + "--exclude-file", + help="specs which Spack should not try to add to a mirror" + " (listed in a file, one per line)", + ) + add_parser.add_argument( + "--exclude-specs", + help="specs which Spack should not try to add to a mirror (specified on command line)", + ) arguments.add_connection_args(add_parser, False) # Remove remove_parser = sp.add_parser("remove", aliases=["rm"], help=mirror_remove.__doc__) @@ -368,6 +377,16 @@ def mirror_add(args): mirror = spack.mirrors.mirror.Mirror(connection, name=args.name) else: mirror = spack.mirrors.mirror.Mirror(args.url, name=args.name) + + exclude_specs = mirror.to_dict().get("exclude", []) + if args.exclude_file: + exclude_specs.extend(specs_from_text_file(args.exclude_file, concretize=False)) + if args.exclude_specs: + exclude_specs.extend(spack.cmd.parse_specs(str(args.exclude_specs).split())) + if exclude_specs: + # round trip specs to assure they are valid + mirror.update({"exclude": [str(s) for s in exclude_specs]}) + spack.mirrors.utils.add(mirror, args.scope) diff --git a/lib/spack/spack/mirrors/mirror.py b/lib/spack/spack/mirrors/mirror.py index 3ca97186cb9..a84f154619f 100644 --- a/lib/spack/spack/mirrors/mirror.py +++ b/lib/spack/spack/mirrors/mirror.py @@ -205,7 +205,7 @@ def _update_connection_dict(self, current_data: dict, new_data: dict, top_level: "endpoint_url", ] if top_level: - keys += ["binary", "source", "signed", "autopush"] + keys += ["binary", "source", "signed", "autopush", "exclude", "include"] changed = False for key in keys: if key in new_data and current_data.get(key) != new_data[key]: diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index f350874485d..1b3f1b96cce 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -186,6 +186,32 @@ def test_buildcache_autopush(tmp_path, install_mockery, mock_fetch): assert (mirror_autopush_dir / "build_cache" / metadata_file).exists() +def test_buildcache_exclude(tmp_path, install_mockery, mock_fetch): + """Test buildcache with autopush""" + mirror_dir = tmp_path / "mirror_a" + + mirror( + "add", + "--autopush", + "--exclude-specs", + "libelf", + "--unsigned", + "mirror-autopush", + mirror_dir.as_uri(), + ) + + s = spack.concretize.concretize_one("libdwarf") + + # Install and generate build cache index + PackageInstaller([s.package], fake=True, explicit=True).install() + + missing_file = spack.binary_distribution.tarball_name(s["libelf"], ".spec.json") + found_file = spack.binary_distribution.tarball_name(s, ".spec.json") + + assert (mirror_dir / "build_cache" / found_file).exists() + assert not (mirror_dir / "build_cache" / missing_file).exists() + + def test_buildcache_sync( mutable_mock_env_path, install_mockery, mock_packages, mock_fetch, mock_stage, tmpdir ): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index fdc0b88019f..a2c18fc1d44 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -342,6 +342,15 @@ def test_update_4(): assert m.fetch_url == "https://example.com" +@pytest.mark.parametrize("filter", ["exclude", "include"]) +def test_update_filters(filter): + # Change push url, ensure minimal config + m = spack.mirrors.mirror.Mirror("https://example.com") + assert m.update({filter: ["foo", "bar"]}) + assert m.to_dict() == {"url": "https://example.com", filter: ["foo", "bar"]} + assert m.fetch_url == "https://example.com" + + @pytest.mark.parametrize("direction", ["fetch", "push"]) def test_update_connection_params(direction, tmpdir, monkeypatch): """Test whether new connection params expand the mirror config to a dict."""