spack repo zip

.gitignore

@@ -7,6 +7,7 @@
 /var/spack/environments
 /var/spack/repos/*/index.yaml
 /var/spack/repos/*/lock
+/var/spack/repos/*/packages.zip
 /opt
 # Ignore everything in /etc/spack except /etc/spack/defaults
 /etc/spack/*

lib/spack/spack/cmd/repo.py

@@ -3,8 +3,13 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import filecmp
 import os
+import pathlib
 import sys
+import tempfile
+import zipfile
+from typing import List, Optional, Tuple
 
 import llnl.util.tty as tty
 
@@ -12,6 +17,7 @@
 import spack.repo
 import spack.util.path
 from spack.cmd.common import arguments
+from spack.util.archive import reproducible_zipfile_from_prefix
 
 description = "manage package source repositories"
 section = "config"
@@ -67,6 +73,12 @@ def setup_parser(subparser):
         help="configuration scope to modify",
     )
 
+    # Zip
+    zip_parser = sp.add_parser("zip", help=repo_zip.__doc__)
+    zip_parser.add_argument(
+        "namespace_or_path", help="namespace or path of a Spack package repository"
+    )
+
 
 def repo_create(args):
     """create a new package repository"""
@@ -109,31 +121,18 @@ def repo_add(args):
 def repo_remove(args):
     """remove a repository from Spack's configuration"""
     repos = spack.config.get("repos", scope=args.scope)
-    namespace_or_path = args.namespace_or_path
 
-    # If the argument is a path, remove that repository from config.
-    canon_path = spack.util.path.canonicalize_path(namespace_or_path)
-    for repo_path in repos:
-        repo_canon_path = spack.util.path.canonicalize_path(repo_path)
-        if canon_path == repo_canon_path:
-            repos.remove(repo_path)
-            spack.config.set("repos", repos, args.scope)
-            tty.msg("Removed repository %s" % repo_path)
-            return
+    key, repo = _get_repo(repos, args.namespace_or_path)
 
-    # If it is a namespace, remove corresponding repo
-    for path in repos:
-        try:
-            repo = spack.repo.from_path(path)
-            if repo.namespace == namespace_or_path:
-                repos.remove(path)
-                spack.config.set("repos", repos, args.scope)
-                tty.msg("Removed repository %s with namespace '%s'." % (repo.root, repo.namespace))
-                return
-        except spack.repo.RepoError:
-            continue
+    if not key:
+        tty.die(f"No repository with path or namespace: {args.namespace_or_path}")
 
-    tty.die("No repository with path or namespace: %s" % namespace_or_path)
+    repos.remove(key)
+    spack.config.set("repos", repos, args.scope)
+    if repo:
+        tty.msg(f"Removed repository {repo.root} with namespace '{repo.namespace}'")
+    else:
+        tty.msg(f"Removed repository {key}")
 
 
 def repo_list(args):
@@ -147,17 +146,74 @@ def repo_list(args):
             continue
 
     if sys.stdout.isatty():
-        msg = "%d package repositor" % len(repos)
-        msg += "y." if len(repos) == 1 else "ies."
-        tty.msg(msg)
+        tty.msg(f"{len(repos)} package repositor{'y.' if len(repos) == 1 else 'ies.'}")
 
     if not repos:
         return
 
     max_ns_len = max(len(r.namespace) for r in repos)
     for repo in repos:
-        fmt = "%%-%ds%%s" % (max_ns_len + 4)
-        print(fmt % (repo.namespace, repo.root))
+        print(f"{repo.namespace:<{max_ns_len}} {repo.root}")
+
+
+def repo_zip(args):
+    """zip a package repository to make it immutable and faster to load"""
+    key, _ = _get_repo(spack.config.get("repos"), args.namespace_or_path)
+
+    if not key:
+        tty.die(f"No repository with path or namespace: {args.namespace_or_path}")
+
+    try:
+        repo = spack.repo.from_path(key)
+    except spack.repo.RepoError:
+        tty.die(f"No repository at path: {key}")
+
+    def _zip_repo_skip(entry: os.DirEntry):
+        return entry.name == "__pycache__"
+
+    def _zip_repo_path_to_name(path: str) -> str:
+        # strip `repo.packages_path`
+        return str(pathlib.Path(path).relative_to(repo.packages_path))
+
+    # Create a zipfile in a temporary file
+    with tempfile.NamedTemporaryFile(delete=False, mode="wb", dir=repo.root) as f, zipfile.ZipFile(
+        f, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9
+    ) as zip:
+        reproducible_zipfile_from_prefix(
+            zip, repo.packages_path, skip=_zip_repo_skip, path_to_name=_zip_repo_path_to_name
+        )
+
+    packages_zip = os.path.join(repo.root, "packages.zip")
+    try:
+        # Inform the user whether or not the repo was modified
+        if filecmp.cmp(f.name, packages_zip):
+            tty.msg(f"{repo.namespace}: {packages_zip} is up to date")
+            return
+        else:
+            os.rename(f.name, packages_zip)
+            tty.msg(f"{repo.namespace} was zipped: {packages_zip}")
+    finally:
+        try:
+            os.unlink(f.name)
+        except OSError:
+            pass
+
+
+def _get_repo(repos: List[str], path_or_name) -> Tuple[Optional[str], Optional[spack.repo.Repo]]:
+    """Find repo by path or namespace"""
+    canon_path = spack.util.path.canonicalize_path(path_or_name)
+    for path in repos:
+        if canon_path == spack.util.path.canonicalize_path(path):
+            return path, None
+
+    for path in repos:
+        try:
+            repo = spack.repo.from_path(path)
+        except spack.repo.RepoError:
+            continue
+        if repo.namespace == path_or_name:
+            return path, repo
+    return None, None
 
 
 def repo(parser, args):
@@ -167,5 +223,6 @@ def repo(parser, args):
         "add": repo_add,
         "remove": repo_remove,
         "rm": repo_remove,
+        "zip": repo_zip,
     }
     action[args.repo_command](args)

lib/spack/spack/detection/path.py

@@ -165,6 +165,7 @@ def libraries_in_path(search_paths: List[str]) -> Dict[str, str]:
                     path_to_lib[entry.path] = entry.name
     return path_to_lib
 
+
 def _group_by_prefix(paths: List[str]) -> Dict[str, Set[str]]:
     groups = collections.defaultdict(set)
     for p in paths:

lib/spack/spack/repo.py

@@ -216,9 +216,9 @@ def compute_loader(self, fullname):
 def packages_path():
     """Get the test repo if it is active, otherwise the builtin repo."""
     try:
-        return spack.repo.PATH.get_repo("builtin.mock").packages_path
-    except spack.repo.UnknownNamespaceError:
-        return spack.repo.PATH.get_repo("builtin").packages_path
+        return PATH.get_repo("builtin.mock").packages_path
+    except UnknownNamespaceError:
+        return PATH.get_repo("builtin").packages_path
 
 
 class GitExe:

lib/spack/spack/util/archive.py

@@ -7,7 +7,9 @@
 import io
 import os
 import pathlib
+import shutil
 import tarfile
+import zipfile
 from contextlib import closing, contextmanager
 from gzip import GzipFile
 from typing import Callable, Dict, Tuple
@@ -228,3 +230,51 @@ def reproducible_tarfile_from_prefix(
                     tar.addfile(file_info, f)
 
         dir_stack.extend(reversed(new_dirs))  # we pop, so reverse to stay alphabetical
+
+
+def reproducible_zipfile_from_prefix(
+    zip: zipfile.ZipFile,
+    prefix: str,
+    *,
+    skip: Callable[[os.DirEntry], bool] = lambda entry: False,
+    path_to_name: Callable[[str], str] = default_path_to_name,
+) -> None:
+    """Similar to ``reproducible_tarfile_from_prefix`` but for zipfiles."""
+    dir_stack = [prefix]
+    while dir_stack:
+        dir = dir_stack.pop()
+
+        # Add the dir before its contents. zip.mkdir is Python 3.11.
+        dir_info = zipfile.ZipInfo(path_to_name(dir))
+        dir_info.external_attr = (0o40755 << 16) | 0x10
+        dir_info.file_size = 0
+        with zip.open(dir_info, "w") as dest:
+            dest.write(b"")
+
+        # Sort by name for reproducibility
+        with os.scandir(dir) as it:
+            entries = sorted(it, key=lambda entry: entry.name)
+
+        new_dirs = []
+        for entry in entries:
+            if skip(entry):
+                continue
+
+            if entry.is_dir(follow_symlinks=False):
+                new_dirs.append(entry.path)
+                continue
+
+            # symlink / hardlink support in ZIP is poor or non-existent: make copies.
+            elif entry.is_file(follow_symlinks=True):
+                file_info = zipfile.ZipInfo(path_to_name(entry.path))
+
+                # Normalize permissions like git
+                s = entry.stat(follow_symlinks=True)
+                file_info.external_attr = (0o755 if s.st_mode & 0o100 else 0o644) << 16
+                file_info.file_size = s.st_size
+                file_info.compress_type = zip.compression
+
+                with open(entry.path, "rb") as src, zip.open(file_info, "w") as dest:
+                    shutil.copyfileobj(src, dest)  # type: ignore[misc]
+
+        dir_stack.extend(reversed(new_dirs))  # we pop, so reverse to stay alphabetical

share/spack/spack-completion.bash

@@ -1748,7 +1748,7 @@ _spack_repo() {
     then
         SPACK_COMPREPLY="-h --help"
     else
-        SPACK_COMPREPLY="create list add remove rm"
+        SPACK_COMPREPLY="create list add remove rm zip"
     fi
 }
 
@@ -1792,6 +1792,15 @@ _spack_repo_rm() {
     fi
 }
 
+_spack_repo_zip() {
+    if $list_options
+    then
+        SPACK_COMPREPLY="-h --help"
+    else
+        _repos
+    fi
+}
+
 _spack_resource() {
     if $list_options
     then

share/spack/spack-completion.fish

@@ -2671,6 +2671,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a list -d 'show
 complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a add -d 'add a package source to Spack\'s configuration'
 complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a remove -d 'remove a repository from Spack\'s configuration'
 complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a rm -d 'remove a repository from Spack\'s configuration'
+complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a zip -d 'zip a package repository to make it immutable and faster to load'
 complete -c spack -n '__fish_spack_using_command repo' -s h -l help -f -a help
 complete -c spack -n '__fish_spack_using_command repo' -s h -l help -d 'show this help message and exit'
 
@@ -2713,6 +2714,12 @@ complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show
 complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system site user command_line'
 complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify'
 
+# spack repo zip
+set -g __fish_spack_optspecs_spack_repo_zip h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 repo zip' $__fish_spack_force_files -a '(__fish_spack_repos)'
+complete -c spack -n '__fish_spack_using_command repo zip' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command repo zip' -s h -l help -d 'show this help message and exit'
+
 # spack resource
 set -g __fish_spack_optspecs_spack_resource h/help
 complete -c spack -n '__fish_spack_using_command_pos 0 resource' -f -a list -d 'list all resources known to spack (currently just patches)'