slightly faster

lib/spack/spack/patch.py

@@ -8,8 +8,7 @@
 import os.path
 import pathlib
 import sys
-import zipfile
+from typing import Any, Dict, Optional, Tuple, Type, Union
-from typing import Any, Dict, Optional, Set, Tuple, Type, Union
 
 import llnl.util.filesystem
 from llnl.url import allowed_archive
@@ -21,6 +20,7 @@
 import spack.repo
 import spack.stage
 import spack.util.spack_json as sjson
+import spack.zipcache
 from spack.util.crypto import Checker, checksum_stream
 from spack.util.executable import which, which_string
 
@@ -155,9 +155,6 @@ def __hash__(self) -> int:
         return hash(self.sha256)
 
 
-zipfilecache: Dict[str, Tuple[zipfile.ZipFile, Set[str]]] = {}
-
-
 class FilePatch(Patch):
     """Describes a patch that is retrieved from a file in the repository."""
 
@@ -205,13 +202,7 @@ def __init__(
                 zip_path = str(pathlib.PurePath(*path.parts[: idx + 1]))
                 entry_path = str(pathlib.PurePath(*path.parts[idx + 1 :]))
 
-                lookup = zipfilecache.get(zip_path)
+                _, namelist = spack.zipcache.get(zip_path)
-                if lookup is None:
-                    zip = zipfile.ZipFile(zip_path, "r")
-                    namelist = set(zip.namelist())
-                    zipfilecache[zip_path] = (zip, namelist)
-                else:
-                    zip, namelist = lookup
                 if entry_path in namelist:
                     abs_path = str(path)
                     break
@@ -242,13 +233,7 @@ def sha256(self) -> str:
                 idx = path.parts.index("packages.zip")
                 zip_path = str(pathlib.PurePath(*path.parts[: idx + 1]))
                 entry_path = str(pathlib.PurePath(*path.parts[idx + 1 :]))
-                lookup = zipfilecache.get(zip_path)
+                zip, _ = spack.zipcache.get(zip_path)
-                if lookup is None:
-                    zip = zipfile.ZipFile(zip_path, "r")
-                    namelist = set(zip.namelist())
-                    zipfilecache[zip_path] = (zip, namelist)
-                else:
-                    zip, namelist = lookup
                 f = zip.open(entry_path, "r")
             else:
                 f = open(self.path, "rb")

lib/spack/spack/repo.py

@@ -26,7 +26,6 @@
 import types
 import uuid
 import warnings
-import zipfile
 import zipimport
 from typing import Any, Dict, Generator, List, Optional, Set, Tuple, Type, Union
 
@@ -48,6 +47,7 @@
 import spack.util.naming as nm
 import spack.util.path
 import spack.util.spack_yaml as syaml
+import spack.zipcache
 
 #: Package modules are imported as spack.pkg.<repo-namespace>.<pkg-name>
 ROOT_PYTHON_NAMESPACE = "spack.pkg"
@@ -365,17 +365,16 @@ def __getattr__(self, name):
 
 
 class EvenFasterPackageChecker(collections.abc.Mapping):
-    def __init__(self, packages_path):
+    def __init__(self, zip_path):
         # The path of the repository managed by this instance
-        self.packages_path = packages_path
+        self.zipfile, self.namelist = spack.zipcache.get(zip_path)
-        self.zipfile = zipfile.ZipFile(os.path.join(packages_path, "..", "packages.zip"), "r")
         self.invalidate()
 
     def invalidate(self):
         self.mtime = os.stat(self.zipfile.filename).st_mtime
         self.pkgs = {
             f.rstrip("/"): self.mtime
-            for f in self.zipfile.namelist()
+            for f in self.namelist
             if f.endswith("/") and f.count("/") == 1 and f != "./"
         }
 
@@ -614,9 +613,6 @@ def __init__(
         cache: "spack.caches.FileCacheType",
     ):
         self.checker = package_checker
-        self.packages_path = self.checker.packages_path
-        if sys.platform == "win32":
-            self.packages_path = llnl.path.convert_to_posix_path(self.packages_path)
         self.namespace = namespace
 
         self.indexers: Dict[str, Indexer] = {}
@@ -1226,7 +1222,7 @@ def filename_for_package_name(self, pkg_name: str) -> str:
     def _pkg_checker(self) -> Union[FastPackageChecker, EvenFasterPackageChecker]:
         if self._fast_package_checker is None:
             if self.zipimporter:
-                self._fast_package_checker = EvenFasterPackageChecker(self.packages_path)
+                self._fast_package_checker = EvenFasterPackageChecker(self.zipimporter.archive)
             else:
                 self._fast_package_checker = FastPackageChecker(self.packages_path)
         return self._fast_package_checker

lib/spack/spack/zipcache.py

@@ -0,0 +1,18 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import zipfile
+from typing import Dict, Set, Tuple
+
+zipfilecache: Dict[str, Tuple[zipfile.ZipFile, Set[str]]] = {}
+
+
+def get(path: str):
+    if path not in zipfilecache:
+        file = zipfile.ZipFile(path)
+        names = set(file.namelist())
+        zipfilecache[path] = (file, names)
+        return file, names
+    return zipfilecache[path]