zipfile repo: quick and dirty wip

.gitignore

@@ -7,6 +7,7 @@
 /var/spack/environments
 /var/spack/repos/*/index.yaml
 /var/spack/repos/*/lock
+/var/spack/repos/*/packages.zip
 /opt
 # Ignore everything in /etc/spack except /etc/spack/defaults
 /etc/spack/*

lib/spack/spack/cmd/external.py

@@ -135,9 +135,7 @@ def external_find(args):
     candidate_packages = packages_to_search_for(
         names=args.packages, tags=args.tags, exclude=args.exclude
     )
-    detected_packages = spack.detection.by_path(
-        candidate_packages, path_hints=args.path, max_workers=args.jobs
-    )
+    detected_packages = spack.detection.by_path(candidate_packages, path_hints=args.path)
 
     new_specs = spack.detection.update_configuration(
         detected_packages, scope=args.scope, buildable=not args.not_buildable

lib/spack/spack/cmd/repo.py

@@ -3,8 +3,13 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import filecmp
 import os
+import pathlib
 import sys
+import tempfile
+import zipfile
+from typing import List, Optional, Tuple
 
 import llnl.util.tty as tty
 
@@ -12,6 +17,7 @@
 import spack.repo
 import spack.util.path
 from spack.cmd.common import arguments
+from spack.util.archive import reproducible_zipfile_from_prefix
 
 description = "manage package source repositories"
 section = "config"
@@ -67,6 +73,12 @@ def setup_parser(subparser):
         help="configuration scope to modify",
     )
 
+    # Zip
+    zip_parser = sp.add_parser("zip", help=repo_zip.__doc__)
+    zip_parser.add_argument(
+        "namespace_or_path", help="namespace or path of a Spack package repository"
+    )
+
 
 def repo_create(args):
     """create a new package repository"""
@@ -109,31 +121,18 @@ def repo_add(args):
 def repo_remove(args):
     """remove a repository from Spack's configuration"""
     repos = spack.config.get("repos", scope=args.scope)
-    namespace_or_path = args.namespace_or_path
 
-    # If the argument is a path, remove that repository from config.
-    canon_path = spack.util.path.canonicalize_path(namespace_or_path)
-    for repo_path in repos:
-        repo_canon_path = spack.util.path.canonicalize_path(repo_path)
-        if canon_path == repo_canon_path:
-            repos.remove(repo_path)
+    key, repo = _get_repo(repos, args.namespace_or_path)
+
+    if not key:
+        tty.die(f"No repository with path or namespace: {args.namespace_or_path}")
+
+    repos.remove(key)
     spack.config.set("repos", repos, args.scope)
-            tty.msg("Removed repository %s" % repo_path)
-            return
-
-    # If it is a namespace, remove corresponding repo
-    for path in repos:
-        try:
-            repo = spack.repo.from_path(path)
-            if repo.namespace == namespace_or_path:
-                repos.remove(path)
-                spack.config.set("repos", repos, args.scope)
-                tty.msg("Removed repository %s with namespace '%s'." % (repo.root, repo.namespace))
-                return
-        except spack.repo.RepoError:
-            continue
-
-    tty.die("No repository with path or namespace: %s" % namespace_or_path)
+    if repo:
+        tty.msg(f"Removed repository {repo.root} with namespace '{repo.namespace}'")
+    else:
+        tty.msg(f"Removed repository {key}")
 
 
 def repo_list(args):
@@ -147,17 +146,77 @@ def repo_list(args):
             continue
 
     if sys.stdout.isatty():
-        msg = "%d package repositor" % len(repos)
-        msg += "y." if len(repos) == 1 else "ies."
-        tty.msg(msg)
+        tty.msg(f"{len(repos)} package repositor{'y.' if len(repos) == 1 else 'ies.'}")
 
     if not repos:
         return
 
     max_ns_len = max(len(r.namespace) for r in repos)
     for repo in repos:
-        fmt = "%%-%ds%%s" % (max_ns_len + 4)
-        print(fmt % (repo.namespace, repo.root))
+        print(f"{repo.namespace:<{max_ns_len}} {repo.root}")
+
+
+def repo_zip(args):
+    """zip a package repository to make it immutable and faster to load"""
+    key, _ = _get_repo(spack.config.get("repos"), args.namespace_or_path)
+
+    if not key:
+        tty.die(f"No repository with path or namespace: {args.namespace_or_path}")
+
+    try:
+        repo = spack.repo.from_path(key)
+    except spack.repo.RepoError:
+        tty.die(f"No repository at path: {key}")
+
+    def _zip_repo_skip(entry: os.DirEntry):
+        return entry.name == "__pycache__"
+
+    def _zip_repo_path_to_name(path: str) -> str:
+        # use spack/pkg/<repo>/* prefix and rename `package.py` as `__init__.py`
+        rel_path = pathlib.PurePath(path).relative_to(repo.packages_path)
+        if rel_path.name == "package.py":
+            rel_path = rel_path.with_name("__init__.py")
+        return str(rel_path)
+
+    # Create a zipfile in a temporary file
+    with tempfile.NamedTemporaryFile(delete=False, mode="wb", dir=repo.root) as f, zipfile.ZipFile(
+        f, "w", compression=zipfile.ZIP_DEFLATED
+    ) as zip:
+        reproducible_zipfile_from_prefix(
+            zip, repo.packages_path, skip=_zip_repo_skip, path_to_name=_zip_repo_path_to_name
+        )
+
+    packages_zip = os.path.join(repo.root, "packages.zip")
+    try:
+        # Inform the user whether or not the repo was modified since it was last zipped
+        if os.path.exists(packages_zip) and filecmp.cmp(f.name, packages_zip):
+            tty.msg(f"{repo.namespace}: {packages_zip} is up to date")
+            return
+        else:
+            os.rename(f.name, packages_zip)
+            tty.msg(f"{repo.namespace} was zipped: {packages_zip}")
+    finally:
+        try:
+            os.unlink(f.name)
+        except OSError:
+            pass
+
+
+def _get_repo(repos: List[str], path_or_name) -> Tuple[Optional[str], Optional[spack.repo.Repo]]:
+    """Find repo by path or namespace"""
+    canon_path = spack.util.path.canonicalize_path(path_or_name)
+    for path in repos:
+        if canon_path == spack.util.path.canonicalize_path(path):
+            return path, None
+
+    for path in repos:
+        try:
+            repo = spack.repo.from_path(path)
+        except spack.repo.RepoError:
+            continue
+        if repo.namespace == path_or_name:
+            return path, repo
+    return None, None
 
 
 def repo(parser, args):
@@ -167,5 +226,6 @@ def repo(parser, args):
         "add": repo_add,
         "remove": repo_remove,
         "rm": repo_remove,
+        "zip": repo_zip,
     }
     action[args.repo_command](args)

lib/spack/spack/detection/common.py

@@ -24,6 +24,7 @@
 import llnl.util.tty
 
 import spack.config
+import spack.error
 import spack.operating_systems.windows_os as winOs
 import spack.spec
 import spack.util.spack_yaml

lib/spack/spack/detection/path.py

@@ -18,10 +18,13 @@
 import llnl.util.lang
 import llnl.util.tty
 
+import spack.package_base
+import spack.repo
 import spack.util.elf as elf_utils
 import spack.util.environment
 import spack.util.environment as environment
 import spack.util.ld_so_conf
+import spack.util.parallel
 
 from .common import (
     DetectedPackage,
@@ -79,26 +82,27 @@ def executables_in_path(path_hints: List[str]) -> Dict[str, str]:
         path_hints: list of paths to be searched. If None the list will be
             constructed based on the PATH environment variable.
     """
-    search_paths = llnl.util.filesystem.search_paths_for_executables(*path_hints)
-    return path_to_dict(search_paths)
+    return path_to_dict(llnl.util.filesystem.search_paths_for_executables(*path_hints))
 
 
-def accept_elf(path, host_compat):
+def accept_elf(entry: os.DirEntry, host_compat: Tuple[bool, bool, int]):
     """Accept an ELF file if the header matches the given compat triplet. In case it's not an ELF
     (e.g. static library, or some arbitrary file, fall back to is_readable_file)."""
     # Fast path: assume libraries at least have .so in their basename.
     # Note: don't replace with splitext, because of libsmth.so.1.2.3 file names.
-    if ".so" not in os.path.basename(path):
-        return llnl.util.filesystem.is_readable_file(path)
+    if ".so" not in entry.name:
+        return is_readable_file(entry)
     try:
-        return host_compat == elf_utils.get_elf_compat(path)
+        return host_compat == elf_utils.get_elf_compat(entry.path)
     except (OSError, elf_utils.ElfParsingError):
-        return llnl.util.filesystem.is_readable_file(path)
+        return is_readable_file(entry)
 
 
-def libraries_in_ld_and_system_library_path(
-    path_hints: Optional[List[str]] = None,
-) -> Dict[str, str]:
+def is_readable_file(entry: os.DirEntry) -> bool:
+    return entry.is_file() and os.access(entry.path, os.R_OK)
+
+
+def system_library_paths() -> List[str]:
     """Get the paths of all libraries available from ``path_hints`` or the
     following defaults:
 
@@ -112,29 +116,27 @@ def libraries_in_ld_and_system_library_path(
     (i.e. the basename of the library path).
 
     There may be multiple paths with the same basename. In this case it is
-    assumed there are two different instances of the library.
+    assumed there are two different instances of the library."""
 
-    Args:
-        path_hints: list of paths to be searched. If None the list will be
-            constructed based on the set of LD_LIBRARY_PATH, LIBRARY_PATH,
-            DYLD_LIBRARY_PATH, and DYLD_FALLBACK_LIBRARY_PATH environment
-            variables as well as the standard system library paths.
-        path_hints (list): list of paths to be searched. If ``None``, the default
-            system paths are used.
-    """
-    if path_hints:
-        search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
-    else:
-        search_paths = []
+    search_paths: List[str] = []
 
-        # Environment variables
-        if sys.platform == "darwin":
+    if sys.platform == "win32":
+        search_hints = spack.util.environment.get_path("PATH")
+        search_paths.extend(llnl.util.filesystem.search_paths_for_libraries(*search_hints))
+        # on Windows, some libraries (.dlls) are found in the bin directory or sometimes
+        # at the search root. Add both of those options to the search scheme
+        search_paths.extend(llnl.util.filesystem.search_paths_for_executables(*search_hints))
+        # if no user provided path was given, add defaults to the search
+        search_paths.extend(WindowsKitExternalPaths.find_windows_kit_lib_paths())
+        # SDK and WGL should be handled by above, however on occasion the WDK is in an atypical
+        # location, so we handle that case specifically.
+        search_paths.extend(WindowsKitExternalPaths.find_windows_driver_development_kit_paths())
+    elif sys.platform == "darwin":
         search_paths.extend(environment.get_path("DYLD_LIBRARY_PATH"))
         search_paths.extend(environment.get_path("DYLD_FALLBACK_LIBRARY_PATH"))
+        search_paths.extend(spack.util.ld_so_conf.host_dynamic_linker_search_paths())
     elif sys.platform.startswith("linux"):
         search_paths.extend(environment.get_path("LD_LIBRARY_PATH"))
-
-        # Dynamic linker paths
         search_paths.extend(spack.util.ld_so_conf.host_dynamic_linker_search_paths())
 
     # Drop redundant paths
@@ -143,50 +145,27 @@ def libraries_in_ld_and_system_library_path(
     # Make use we don't doubly list /usr/lib and /lib etc
     search_paths = list(llnl.util.lang.dedupe(search_paths, key=file_identifier))
 
+    return search_paths
+
+
+def libraries_in_path(search_paths: List[str]) -> Dict[str, str]:
     try:
         host_compat = elf_utils.get_elf_compat(sys.executable)
-        accept = lambda path: accept_elf(path, host_compat)
+        accept = lambda entry: accept_elf(entry, host_compat)
     except (OSError, elf_utils.ElfParsingError):
-        accept = llnl.util.filesystem.is_readable_file
+        accept = is_readable_file
 
     path_to_lib = {}
     # Reverse order of search directories so that a lib in the first
     # search path entry overrides later entries
     for search_path in reversed(search_paths):
-        for lib in os.listdir(search_path):
-            lib_path = os.path.join(search_path, lib)
-            if accept(lib_path):
-                path_to_lib[lib_path] = lib
+        with os.scandir(search_path) as it:
+            for entry in it:
+                if accept(entry):
+                    path_to_lib[entry.path] = entry.name
     return path_to_lib
 
 
-def libraries_in_windows_paths(path_hints: Optional[List[str]] = None) -> Dict[str, str]:
-    """Get the paths of all libraries available from the system PATH paths.
-
-    For more details, see `libraries_in_ld_and_system_library_path` regarding
-    return type and contents.
-
-    Args:
-        path_hints: list of paths to be searched. If None the list will be
-            constructed based on the set of PATH environment
-            variables as well as the standard system library paths.
-    """
-    search_hints = (
-        path_hints if path_hints is not None else spack.util.environment.get_path("PATH")
-    )
-    search_paths = llnl.util.filesystem.search_paths_for_libraries(*search_hints)
-    # on Windows, some libraries (.dlls) are found in the bin directory or sometimes
-    # at the search root. Add both of those options to the search scheme
-    search_paths.extend(llnl.util.filesystem.search_paths_for_executables(*search_hints))
-    if path_hints is None:
-        # if no user provided path was given, add defaults to the search
-        search_paths.extend(WindowsKitExternalPaths.find_windows_kit_lib_paths())
-        # SDK and WGL should be handled by above, however on occasion the WDK is in an atypical
-        # location, so we handle that case specifically.
-        search_paths.extend(WindowsKitExternalPaths.find_windows_driver_development_kit_paths())
-    return path_to_dict(search_paths)
-
-
 def _group_by_prefix(paths: List[str]) -> Dict[str, Set[str]]:
     groups = collections.defaultdict(set)
     for p in paths:
@@ -197,10 +176,13 @@ def _group_by_prefix(paths: List[str]) -> Dict[str, Set[str]]:
 class Finder:
     """Inspects the file-system looking for packages. Guesses places where to look using PATH."""
 
+    def __init__(self, paths: Dict[str, str]):
+        self.paths = paths
+
     def default_path_hints(self) -> List[str]:
         return []
 
-    def search_patterns(self, *, pkg: Type["spack.package_base.PackageBase"]) -> List[str]:
+    def search_patterns(self, *, pkg: Type[spack.package_base.PackageBase]) -> Optional[List[str]]:
         """Returns the list of patterns used to match candidate files.
 
         Args:
@@ -208,15 +190,6 @@ def search_patterns(self, *, pkg: Type["spack.package_base.PackageBase"]) -> Lis
         """
         raise NotImplementedError("must be implemented by derived classes")
 
-    def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]:
-        """Returns a list of candidate files found on the system.
-
-        Args:
-            patterns: search patterns to be used for matching files
-            paths: paths where to search for files
-        """
-        raise NotImplementedError("must be implemented by derived classes")
-
     def prefix_from_path(self, *, path: str) -> str:
         """Given a path where a file was found, returns the corresponding prefix.
 
@@ -226,7 +199,7 @@ def prefix_from_path(self, *, path: str) -> str:
         raise NotImplementedError("must be implemented by derived classes")
 
     def detect_specs(
-        self, *, pkg: Type["spack.package_base.PackageBase"], paths: List[str]
+        self, *, pkg: Type[spack.package_base.PackageBase], paths: List[str]
     ) -> List[DetectedPackage]:
         """Given a list of files matching the search patterns, returns a list of detected specs.
 
@@ -300,45 +273,36 @@ def detect_specs(
 
         return result
 
-    def find(
-        self, *, pkg_name: str, repository, initial_guess: Optional[List[str]] = None
-    ) -> List[DetectedPackage]:
+    def find(self, *, pkg_name: str, repository: spack.repo.Repo) -> List[DetectedPackage]:
         """For a given package, returns a list of detected specs.
 
         Args:
             pkg_name: package being detected
             repository: repository to retrieve the package
-            initial_guess: initial list of paths to search from the caller if None, default paths
-                are searched. If this is an empty list, nothing will be searched.
         """
         pkg_cls = repository.get_pkg_class(pkg_name)
         patterns = self.search_patterns(pkg=pkg_cls)
         if not patterns:
             return []
-        if initial_guess is None:
-            initial_guess = self.default_path_hints()
-            initial_guess.extend(common_windows_package_paths(pkg_cls))
-        candidates = self.candidate_files(patterns=patterns, paths=initial_guess)
-        result = self.detect_specs(pkg=pkg_cls, paths=candidates)
-        return result
+        regex = re.compile("|".join(patterns))
+        paths = [path for path, file in self.paths.items() if regex.search(file)]
+        paths.sort()
+        return self.detect_specs(pkg=pkg_cls, paths=paths)
 
 
 class ExecutablesFinder(Finder):
-    def default_path_hints(self) -> List[str]:
-        return spack.util.environment.get_path("PATH")
+    @classmethod
+    def in_search_paths(cls, paths: List[str]):
+        return cls(executables_in_path(paths))
 
-    def search_patterns(self, *, pkg: Type["spack.package_base.PackageBase"]) -> List[str]:
-        result = []
+    @classmethod
+    def in_default_paths(cls):
+        return cls.in_search_paths(spack.util.environment.get_path("PATH"))
+
+    def search_patterns(self, *, pkg: Type[spack.package_base.PackageBase]) -> Optional[List[str]]:
         if hasattr(pkg, "executables") and hasattr(pkg, "platform_executables"):
-            result = pkg.platform_executables()
-        return result
-
-    def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]:
-        executables_by_path = executables_in_path(path_hints=paths)
-        joined_pattern = re.compile(r"|".join(patterns))
-        result = [path for path, exe in executables_by_path.items() if joined_pattern.search(exe)]
-        result.sort()
-        return result
+            return pkg.platform_executables()
+        return None
 
     def prefix_from_path(self, *, path: str) -> str:
         result = executable_prefix(path)
@@ -349,29 +313,18 @@ def prefix_from_path(self, *, path: str) -> str:
 
 
 class LibrariesFinder(Finder):
-    """Finds libraries on the system, searching by LD_LIBRARY_PATH, LIBRARY_PATH,
-    DYLD_LIBRARY_PATH, DYLD_FALLBACK_LIBRARY_PATH, and standard system library paths
-    """
+    """Finds libraries in the provided paths matching package search patterns."""
 
-    def search_patterns(self, *, pkg: Type["spack.package_base.PackageBase"]) -> List[str]:
-        result = []
-        if hasattr(pkg, "libraries"):
-            result = pkg.libraries
-        return result
+    @classmethod
+    def in_search_paths(cls, paths: List[str]):
+        return cls(libraries_in_path(paths))
 
-    def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]:
-        libraries_by_path = (
-            libraries_in_ld_and_system_library_path(path_hints=paths)
-            if sys.platform != "win32"
-            else libraries_in_windows_paths(path_hints=paths)
-        )
-        patterns = [re.compile(x) for x in patterns]
-        result = []
-        for compiled_re in patterns:
-            for path, exe in libraries_by_path.items():
-                if compiled_re.search(exe):
-                    result.append(path)
-        return result
+    @classmethod
+    def in_default_paths(cls):
+        return cls.in_search_paths(system_library_paths())
+
+    def search_patterns(self, *, pkg: Type[spack.package_base.PackageBase]) -> Optional[List[str]]:
+        return getattr(pkg, "libraries", None)
 
     def prefix_from_path(self, *, path: str) -> str:
         result = library_prefix(path)
@@ -382,10 +335,7 @@ def prefix_from_path(self, *, path: str) -> str:
 
 
 def by_path(
-    packages_to_search: Iterable[str],
-    *,
-    path_hints: Optional[List[str]] = None,
-    max_workers: Optional[int] = None,
+    packages_to_search: Iterable[str], *, path_hints: Optional[List[str]] = None
 ) -> Dict[str, List[DetectedPackage]]:
     """Return the list of packages that have been detected on the system, keyed by
     unqualified package name.
@@ -394,31 +344,26 @@ def by_path(
         packages_to_search: list of packages to be detected. Each package can be either unqualified
             of fully qualified
         path_hints: initial list of paths to be searched
-        max_workers: maximum number of workers to search for packages in parallel
     """
-    import spack.repo
-
     # TODO: Packages should be able to define both .libraries and .executables in the future
     # TODO: determine_spec_details should get all relevant libraries and executables in one call
-    executables_finder, libraries_finder = ExecutablesFinder(), LibrariesFinder()
+    if path_hints is None:
+        exe_finder = ExecutablesFinder.in_default_paths()
+        lib_finder = LibrariesFinder.in_default_paths()
+    else:
+        exe_finder = ExecutablesFinder.in_search_paths(path_hints)
+        lib_finder = LibrariesFinder.in_search_paths(path_hints)
+
     detected_specs_by_package: Dict[str, Tuple[concurrent.futures.Future, ...]] = {}
 
     result = collections.defaultdict(list)
     repository = spack.repo.PATH.ensure_unwrapped()
-    with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
+    with spack.util.parallel.make_concurrent_executor() as executor:
         for pkg in packages_to_search:
             executable_future = executor.submit(
-                executables_finder.find,
-                pkg_name=pkg,
-                initial_guess=path_hints,
-                repository=repository,
-            )
-            library_future = executor.submit(
-                libraries_finder.find,
-                pkg_name=pkg,
-                initial_guess=path_hints,
-                repository=repository,
+                exe_finder.find, pkg_name=pkg, repository=repository
             )
+            library_future = executor.submit(lib_finder.find, pkg_name=pkg, repository=repository)
             detected_specs_by_package[pkg] = executable_future, library_future
 
         for pkg_name, futures in detected_specs_by_package.items():
@@ -434,7 +379,7 @@ def by_path(
                     )
                 except Exception as e:
                     llnl.util.tty.debug(
-                        f"[EXTERNAL DETECTION] Skipping {pkg_name}: exception occured {e}"
+                        f"[EXTERNAL DETECTION] Skipping {pkg_name} due to: {e.__class__}: {e}"
                     )
 
     return result

lib/spack/spack/patch.py

@@ -9,6 +9,7 @@
 import os.path
 import pathlib
 import sys
+import zipfile
 from typing import Any, Dict, Optional, Tuple, Type, Union
 
 import llnl.util.filesystem
@@ -21,7 +22,7 @@
 import spack.repo
 import spack.stage
 import spack.util.spack_json as sjson
-from spack.util.crypto import Checker, checksum
+from spack.util.crypto import Checker, checksum_stream
 from spack.util.executable import which, which_string
 
 
@@ -155,6 +156,9 @@ def __hash__(self) -> int:
         return hash(self.sha256)
 
 
+zipfilecache = {}
+
+
 class FilePatch(Patch):
     """Describes a patch that is retrieved from a file in the repository."""
 
@@ -194,9 +198,27 @@ def __init__(
             # Cannot use pkg.package_dir because it's a property and we have
             # classes, not instances.
             pkg_dir = os.path.abspath(os.path.dirname(cls.module.__file__))
-            path = os.path.join(pkg_dir, self.relative_path)
-            if os.path.exists(path):
-                abs_path = path
+            path = pathlib.Path(os.path.join(pkg_dir, self.relative_path))
+
+            if "packages.zip" in path.parts:
+                # check if it exists in the zip file.
+                idx = path.parts.index("packages.zip")
+                zip_path, entry_path = pathlib.PurePath(*path.parts[: idx + 1]), pathlib.PurePath(
+                    *path.parts[idx + 1 :]
+                )
+
+                lookup = zipfilecache.get(zip_path)
+                if lookup is None:
+                    zip = zipfile.ZipFile(zip_path, "r")
+                    namelist = set(zip.namelist())
+                    zipfilecache[zip_path] = (zip, namelist)
+                else:
+                    zip, namelist = lookup
+                if str(entry_path) in namelist:
+                    abs_path = str(path)
+                    break
+            elif path.exists():
+                abs_path = str(path)
                 break
 
         if abs_path is None:
@@ -216,7 +238,24 @@ def sha256(self) -> str:
             The sha256 of the patch file.
         """
         if self._sha256 is None and self.path is not None:
-            self._sha256 = checksum(hashlib.sha256, self.path)
+            path = pathlib.PurePath(self.path)
+            if "packages.zip" in path.parts:
+                print("yes")
+                # split in path to packages.zip and the path within the zip
+                idx = path.parts.index("packages.zip")
+                path_to_zip, path_in_zip = pathlib.PurePath(
+                    *path.parts[: idx + 1]
+                ), pathlib.PurePath(*path.parts[idx + 1 :])
+                zip = zipfilecache.get(path_to_zip)
+                if not zip:
+                    zip = zipfile.ZipFile(path_to_zip, "r")
+                    zipfilecache[path_to_zip] = zip
+                f = zip.open(str(path_in_zip), "r")
+            else:
+                f = open(self.path, "rb")
+            self._sha256 = checksum_stream(hashlib.sha256, f)
+            f.close()
+
         assert isinstance(self._sha256, str)
         return self._sha256
 

lib/spack/spack/repo.py

@@ -26,6 +26,7 @@
 import types
 import uuid
 import warnings
+import zipimport
 from typing import Any, Dict, Generator, List, Optional, Set, Tuple, Type, Union
 
 import llnl.path
@@ -100,32 +101,6 @@ def get_data(self, path):
             return self.prepend.encode() + b"\n" + data
 
 
-class RepoLoader(_PrependFileLoader):
-    """Loads a Python module associated with a package in specific repository"""
-
-    #: Code in ``_package_prepend`` is prepended to imported packages.
-    #:
-    #: Spack packages are expected to call `from spack.package import *`
-    #: themselves, but we are allowing a deprecation period before breaking
-    #: external repos that don't do this yet.
-    _package_prepend = "from spack.package import *"
-
-    def __init__(self, fullname, repo, package_name):
-        self.repo = repo
-        self.package_name = package_name
-        self.package_py = repo.filename_for_package_name(package_name)
-        self.fullname = fullname
-        super().__init__(self.fullname, self.package_py, prepend=self._package_prepend)
-
-
-class SpackNamespaceLoader:
-    def create_module(self, spec):
-        return SpackNamespace(spec.name)
-
-    def exec_module(self, module):
-        module.__loader__ = self
-
-
 class ReposFinder:
     """MetaPathFinder class that loads a Python module corresponding to a Spack package.
 
@@ -165,10 +140,11 @@ def find_spec(self, fullname, python_path, target=None):
         if not fullname.startswith(ROOT_PYTHON_NAMESPACE):
             return None
 
-        loader = self.compute_loader(fullname)
-        if loader is None:
+        result = self.compute_loader(fullname)
+        if result is None:
             return None
-        return importlib.util.spec_from_loader(fullname, loader)
+        loader, actual_fullname = result
+        return importlib.util.spec_from_loader(actual_fullname, loader)
 
     def compute_loader(self, fullname):
         # namespaces are added to repo, and package modules are leaves.
@@ -187,16 +163,29 @@ def compute_loader(self, fullname):
                 # With 2 nested conditionals we can call "repo.real_name" only once
                 package_name = repo.real_name(module_name)
                 if package_name:
-                    return RepoLoader(fullname, repo, package_name)
+                    # annoyingly there is a many to one mapping for pkg module to file, have to
+                    # figure out how to deal with this properly.
+                    return (
+                        (repo.zipimporter, f"{namespace}.{package_name}")
+                        if repo.zipimporter
+                        else (
+                            _PrependFileLoader(
+                                fullname=fullname,
+                                path=repo.filename_for_package_name(package_name),
+                                prepend="from spack.package import *",
+                            ),
+                            fullname,
+                        )
+                    )
 
             # We are importing a full namespace like 'spack.pkg.builtin'
             if fullname == repo.full_namespace:
-                return SpackNamespaceLoader()
+                return SpackNamespaceLoader(), fullname
 
         # No repo provides the namespace, but it is a valid prefix of
         # something in the RepoPath.
         if is_repo_path and self.current_repository.by_namespace.is_prefix(fullname):
-            return SpackNamespaceLoader()
+            return SpackNamespaceLoader(), fullname
 
         return None
 
@@ -207,6 +196,7 @@ def compute_loader(self, fullname):
 repo_config_name = "repo.yaml"  # Top-level filename for repo config.
 repo_index_name = "index.yaml"  # Top-level filename for repository index.
 packages_dir_name = "packages"  # Top-level repo directory containing pkgs.
+packages_zip_name = "packages.zip"  # Top-level filename for zipped packages.
 package_file_name = "package.py"  # Filename for packages in a repository.
 
 #: Guaranteed unused default value for some functions.
@@ -216,9 +206,9 @@ def compute_loader(self, fullname):
 def packages_path():
     """Get the test repo if it is active, otherwise the builtin repo."""
     try:
-        return spack.repo.PATH.get_repo("builtin.mock").packages_path
-    except spack.repo.UnknownNamespaceError:
-        return spack.repo.PATH.get_repo("builtin").packages_path
+        return PATH.get_repo("builtin.mock").packages_path
+    except UnknownNamespaceError:
+        return PATH.get_repo("builtin").packages_path
 
 
 class GitExe:
@@ -1009,9 +999,14 @@ def check(condition, msg):
         self._names = self.full_namespace.split(".")
 
         packages_dir = config.get("subdirectory", packages_dir_name)
+        packages_zip = os.path.join(self.root, "packages.zip")
+        self.zipimporter = (
+            zipimport.zipimporter(packages_zip) if os.path.exists(packages_zip) else None
+        )
         self.packages_path = os.path.join(self.root, packages_dir)
         check(
-            os.path.isdir(self.packages_path), f"No directory '{packages_dir}' found in '{root}'"
+            self.zipimporter or os.path.isdir(self.packages_path),
+            f"No '{self.packages_path}' or '{packages_zip} found in '{root}'",
         )
 
         # Class attribute overrides by package name
@@ -1507,6 +1502,14 @@ def use_repositories(
         PATH = saved
 
 
+class SpackNamespaceLoader:
+    def create_module(self, spec):
+        return SpackNamespace(spec.name)
+
+    def exec_module(self, module):
+        module.__loader__ = self
+
+
 class MockRepositoryBuilder:
     """Build a mock repository in a directory"""
 

lib/spack/spack/stage.py

@@ -927,7 +927,7 @@ def interactive_version_filter(
     orig_url_dict = url_dict  # only copy when using editor to modify
     print_header = True
     VERSION_COLOR = spack.spec.VERSION_COLOR
-    while True:
+    while sys.stdin.isatty():
         if print_header:
             has_filter = version_filter != VersionList([":"])
             header = []
@@ -944,7 +944,9 @@ def interactive_version_filter(
                 num_new = sum(1 for v in sorted_and_filtered if v not in known_versions)
                 header.append(f"{llnl.string.plural(num_new, 'new version')}")
             if has_filter:
-                header.append(colorize(f"Filtered by {VERSION_COLOR}@@{version_filter}@."))
+                header.append(
+                    colorize(f"Filtered by {VERSION_COLOR}@@{version_filter}@. (clear with c)")
+                )
 
             version_with_url = [
                 colorize(

lib/spack/spack/util/archive.py

@@ -7,7 +7,9 @@
 import io
 import os
 import pathlib
+import shutil
 import tarfile
+import zipfile
 from contextlib import closing, contextmanager
 from gzip import GzipFile
 from typing import Callable, Dict, Tuple
@@ -228,3 +230,53 @@ def reproducible_tarfile_from_prefix(
                     tar.addfile(file_info, f)
 
         dir_stack.extend(reversed(new_dirs))  # we pop, so reverse to stay alphabetical
+
+
+def reproducible_zipfile_from_prefix(
+    zip: zipfile.ZipFile,
+    prefix: str,
+    *,
+    skip: Callable[[os.DirEntry], bool] = lambda entry: False,
+    path_to_name: Callable[[str], str] = default_path_to_name,
+) -> None:
+    """Similar to ``reproducible_tarfile_from_prefix`` but for zipfiles."""
+    dir_stack = [prefix]
+    while dir_stack:
+        dir = dir_stack.pop()
+
+        # Add the dir before its contents. zip.mkdir is Python 3.11.
+        dir_info = zipfile.ZipInfo(path_to_name(dir))
+        if not dir_info.filename.endswith("/"):
+            dir_info.filename += "/"
+        dir_info.external_attr = (0o40755 << 16) | 0x10
+        dir_info.file_size = 0
+        with zip.open(dir_info, "w") as dest:
+            dest.write(b"")
+
+        # Sort by name for reproducibility
+        with os.scandir(dir) as it:
+            entries = sorted(it, key=lambda entry: entry.name)
+
+        new_dirs = []
+        for entry in entries:
+            if skip(entry):
+                continue
+
+            if entry.is_dir(follow_symlinks=False):
+                new_dirs.append(entry.path)
+                continue
+
+            # symlink / hardlink support in ZIP is poor or non-existent: make copies.
+            elif entry.is_file(follow_symlinks=True):
+                file_info = zipfile.ZipInfo(path_to_name(entry.path))
+
+                # Normalize permissions like git
+                s = entry.stat(follow_symlinks=True)
+                file_info.external_attr = (0o755 if s.st_mode & 0o100 else 0o644) << 16
+                file_info.file_size = s.st_size
+                file_info.compress_type = zip.compression
+
+                with open(entry.path, "rb") as src, zip.open(file_info, "w") as dest:
+                    shutil.copyfileobj(src, dest)  # type: ignore[misc]
+
+        dir_stack.extend(reversed(new_dirs))  # we pop, so reverse to stay alphabetical

share/spack/spack-completion.bash

@@ -1748,7 +1748,7 @@ _spack_repo() {
     then
         SPACK_COMPREPLY="-h --help"
     else
-        SPACK_COMPREPLY="create list add remove rm"
+        SPACK_COMPREPLY="create list add remove rm zip"
     fi
 }
 
@@ -1792,6 +1792,15 @@ _spack_repo_rm() {
     fi
 }
 
+_spack_repo_zip() {
+    if $list_options
+    then
+        SPACK_COMPREPLY="-h --help"
+    else
+        _repos
+    fi
+}
+
 _spack_resource() {
     if $list_options
     then

share/spack/spack-completion.fish

@@ -2671,6 +2671,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a list -d 'show
 complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a add -d 'add a package source to Spack\'s configuration'
 complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a remove -d 'remove a repository from Spack\'s configuration'
 complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a rm -d 'remove a repository from Spack\'s configuration'
+complete -c spack -n '__fish_spack_using_command_pos 0 repo' -f -a zip -d 'zip a package repository to make it immutable and faster to load'
 complete -c spack -n '__fish_spack_using_command repo' -s h -l help -f -a help
 complete -c spack -n '__fish_spack_using_command repo' -s h -l help -d 'show this help message and exit'
 
@@ -2713,6 +2714,12 @@ complete -c spack -n '__fish_spack_using_command repo rm' -s h -l help -d 'show
 complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -f -a '_builtin defaults system site user command_line'
 complete -c spack -n '__fish_spack_using_command repo rm' -l scope -r -d 'configuration scope to modify'
 
+# spack repo zip
+set -g __fish_spack_optspecs_spack_repo_zip h/help
+complete -c spack -n '__fish_spack_using_command_pos 0 repo zip' $__fish_spack_force_files -a '(__fish_spack_repos)'
+complete -c spack -n '__fish_spack_using_command repo zip' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command repo zip' -s h -l help -d 'show this help message and exit'
+
 # spack resource
 set -g __fish_spack_optspecs_spack_resource h/help
 complete -c spack -n '__fish_spack_using_command_pos 0 resource' -f -a list -d 'list all resources known to spack (currently just patches)'

var/spack/repos/builtin/packages/chai/package.py

@@ -6,8 +6,7 @@
 import socket
 
 from spack.package import *
-
-from .blt import llnl_link_helpers
+from spack.pkg.builtin.blt import llnl_link_helpers
 
 
 class Chai(CachedCMakePackage, CudaPackage, ROCmPackage):

var/spack/repos/builtin/packages/raja/package.py

@@ -6,8 +6,7 @@
 import socket
 
 from spack.package import *
-
-from .blt import llnl_link_helpers
+from spack.pkg.builtin.blt import llnl_link_helpers
 
 
 # Starting with 2022.03.0, the only submodule we want to fetch is tpl/desul

var/spack/repos/builtin/packages/umpire/package.py

@@ -7,8 +7,7 @@
 import socket
 
 from spack.package import *
-
-from .blt import llnl_link_helpers
+from spack.pkg.builtin.blt import llnl_link_helpers
 
 
 class Umpire(CachedCMakePackage, CudaPackage, ROCmPackage):