Mirrors: add option to exclude packages from "mirror create" (#14154)

* add an --exclude-file option to 'spack mirror create' which allows a user to specify a file of specs to exclude when creating a mirror. this is anticipated to be useful especially when using the '--all' option

* allow specifying number of versions when mirroring all packages

* when mirroring all specs within an environment, include dependencies of root specs

* add '--exclude-specs' option to allow user to specify that specs should be excluded on the command line

* add test for excluding specs

lib/spack/spack/cmd/mirror.py

@@ -45,6 +45,15 @@ def setup_parser(subparser):
              " (this requires significant time and space)")
     create_parser.add_argument(
         '-f', '--file', help="file with specs of packages to put in mirror")
+    create_parser.add_argument(
+        '--exclude-file',
+        help="specs which Spack should not try to add to a mirror"
+             " (listed in a file, one per line)")
+    create_parser.add_argument(
+        '--exclude-specs',
+        help="specs which Spack should not try to add to a mirror"
+             " (specified on command line)")
+
     create_parser.add_argument(
         '--skip-unstable-versions', action='store_true',
         help="don't cache versions unless they identify a stable (unchanging)"
@@ -232,9 +241,7 @@ def _read_specs_from_file(filename):
     return specs
 
 
-def mirror_create(args):
+def _determine_specs_to_mirror(args):
-    """Create a directory to be used as a spack mirror, and fill it with
-       package archives."""
     if args.specs and args.all:
         raise SpackError("Cannot specify specs on command line if you"
                          " chose to mirror all specs with '--all'")
@@ -264,6 +271,7 @@ def mirror_create(args):
                 tty.die("Cannot pass specs on the command line with --file.")
             specs = _read_specs_from_file(args.file)
 
+        env_specs = None
         if not specs:
             # If nothing is passed, use environment or all if no active env
             if not args.all:
@@ -273,12 +281,9 @@ def mirror_create(args):
 
             env = ev.get_env(args, 'mirror')
             if env:
-                mirror_specs = env.specs_by_hash.values()
+                env_specs = env.all_specs()
             else:
                 specs = [Spec(n) for n in spack.repo.all_package_names()]
-                mirror_specs = spack.mirror.get_all_versions(specs)
-                mirror_specs.sort(
-                    key=lambda s: (s.name, s.version))
         else:
             # If the user asked for dependencies, traverse spec DAG get them.
             if args.dependencies:
@@ -297,11 +302,38 @@ def mirror_create(args):
                 msg = 'Skipping {0} as it is an external spec.'
                 tty.msg(msg.format(spec.cshort_spec))
 
+        if env_specs:
+            if args.versions_per_spec:
+                tty.warn("Ignoring '--versions-per-spec' for mirroring specs"
+                         " in environment.")
+            mirror_specs = env_specs
+        else:
             if num_versions == 'all':
                 mirror_specs = spack.mirror.get_all_versions(specs)
             else:
                 mirror_specs = spack.mirror.get_matching_versions(
                     specs, num_versions=num_versions)
+            mirror_specs.sort(
+                key=lambda s: (s.name, s.version))
+
+    exclude_specs = []
+    if args.exclude_file:
+        exclude_specs.extend(_read_specs_from_file(args.exclude_file))
+    if args.exclude_specs:
+        exclude_specs.extend(
+            spack.cmd.parse_specs(str(args.exclude_specs).split()))
+    if exclude_specs:
+        mirror_specs = list(
+            x for x in mirror_specs
+            if not any(x.satisfies(y, strict=True) for y in exclude_specs))
+
+    return mirror_specs
+
+
+def mirror_create(args):
+    """Create a directory to be used as a spack mirror, and fill it with
+       package archives."""
+    mirror_specs = _determine_specs_to_mirror(args)
 
     mirror = spack.mirror.Mirror(
         args.directory or spack.config.get('config:source_cache'))

lib/spack/spack/environment.py

@@ -1232,26 +1232,19 @@ def install_all(self, args=None):
 
             self._install(spec, **kwargs)
 
-    def all_specs_by_hash(self):
-        """Map of hashes to spec for all specs in this environment."""
-        # Note this uses dag-hashes calculated without build deps as keys,
-        # whereas the environment tracks specs based on dag-hashes calculated
-        # with all dependencies. This function should not be used by an
-        # Environment object for management of its own data structures
-        hashes = {}
-        for h in self.concretized_order:
-            specs = self.specs_by_hash[h].traverse(deptype=('link', 'run'))
-            for spec in specs:
-                hashes[spec.dag_hash()] = spec
-        return hashes
-
     def all_specs(self):
         """Return all specs, even those a user spec would shadow."""
-        return sorted(self.all_specs_by_hash().values())
+        all_specs = set()
+        for h in self.concretized_order:
+            all_specs.update(self.specs_by_hash[h].traverse())
+
+        return sorted(all_specs)
 
     def all_hashes(self):
-        """Return all specs, even those a user spec would shadow."""
+        """Return hashes of all specs.
-        return list(self.all_specs_by_hash().keys())
+
+        Note these hashes exclude build dependencies."""
+        return list(set(s.dag_hash() for s in self.all_specs()))
 
     def roots(self):
         """Specs explicitly requested by the user *in this environment*.

lib/spack/spack/test/cmd/mirror.py

@@ -89,6 +89,56 @@ def test_mirror_skip_unstable(tmpdir_factory, mock_packages, config,
             set(['trivial-pkg-with-valid-hash']))
 
 
+class MockMirrorArgs(object):
+    def __init__(self, specs=None, all=False, file=None,
+                 versions_per_spec=None, dependencies=False,
+                 exclude_file=None, exclude_specs=None):
+        self.specs = specs or []
+        self.all = all
+        self.file = file
+        self.versions_per_spec = versions_per_spec
+        self.dependencies = dependencies
+        self.exclude_file = exclude_file
+        self.exclude_specs = exclude_specs
+
+
+def test_exclude_specs(mock_packages):
+    args = MockMirrorArgs(
+        specs=['mpich'],
+        versions_per_spec='all',
+        exclude_specs="mpich@3.0.1:3.0.2 mpich@1.0")
+
+    mirror_specs = spack.cmd.mirror._determine_specs_to_mirror(args)
+    expected_include = set(spack.spec.Spec(x) for x in
+                           ['mpich@3.0.3', 'mpich@3.0.4', 'mpich@3.0'])
+    expected_exclude = set(spack.spec.Spec(x) for x in
+                           ['mpich@3.0.1', 'mpich@3.0.2', 'mpich@1.0'])
+    assert expected_include <= set(mirror_specs)
+    assert (not expected_exclude & set(mirror_specs))
+
+
+def test_exclude_file(mock_packages, tmpdir):
+    exclude_path = os.path.join(str(tmpdir), 'test-exclude.txt')
+    with open(exclude_path, 'w') as exclude_file:
+        exclude_file.write("""\
+mpich@3.0.1:3.0.2
+mpich@1.0
+""")
+
+    args = MockMirrorArgs(
+        specs=['mpich'],
+        versions_per_spec='all',
+        exclude_file=exclude_path)
+
+    mirror_specs = spack.cmd.mirror._determine_specs_to_mirror(args)
+    expected_include = set(spack.spec.Spec(x) for x in
+                           ['mpich@3.0.3', 'mpich@3.0.4', 'mpich@3.0'])
+    expected_exclude = set(spack.spec.Spec(x) for x in
+                           ['mpich@3.0.1', 'mpich@3.0.2', 'mpich@1.0'])
+    assert expected_include <= set(mirror_specs)
+    assert (not expected_exclude & set(mirror_specs))
+
+
 def test_mirror_crud(tmp_scope, capsys):
     with capsys.disabled():
         mirror('add', '--scope', tmp_scope, 'mirror', 'http://spack.io')

share/spack/spack-completion.bash

@@ -1042,7 +1042,7 @@ _spack_mirror() {
 _spack_mirror_create() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -d --directory -a --all -f --file --skip-unstable-versions -D --dependencies -n --versions-per-spec"
+        SPACK_COMPREPLY="-h --help -d --directory -a --all -f --file --exclude-file --exclude-specs --skip-unstable-versions -D --dependencies -n --versions-per-spec"
     else
         _all_packages
     fi