From 14b51ce450dced57128256ffc5776eb627f52280 Mon Sep 17 00:00:00 2001 From: Harmen Stoppels Date: Fri, 23 Aug 2024 15:12:39 +0200 Subject: [PATCH] slightly faster --- lib/spack/spack/patch.py | 23 ++++------------------- lib/spack/spack/repo.py | 14 +++++--------- lib/spack/spack/zipcache.py | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+), 28 deletions(-) create mode 100644 lib/spack/spack/zipcache.py diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index fb1964e06c6..853a092754e 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -8,8 +8,7 @@ import os.path import pathlib import sys -import zipfile -from typing import Any, Dict, Optional, Set, Tuple, Type, Union +from typing import Any, Dict, Optional, Tuple, Type, Union import llnl.util.filesystem from llnl.url import allowed_archive @@ -21,6 +20,7 @@ import spack.repo import spack.stage import spack.util.spack_json as sjson +import spack.zipcache from spack.util.crypto import Checker, checksum_stream from spack.util.executable import which, which_string @@ -155,9 +155,6 @@ def __hash__(self) -> int: return hash(self.sha256) -zipfilecache: Dict[str, Tuple[zipfile.ZipFile, Set[str]]] = {} - - class FilePatch(Patch): """Describes a patch that is retrieved from a file in the repository.""" @@ -205,13 +202,7 @@ def __init__( zip_path = str(pathlib.PurePath(*path.parts[: idx + 1])) entry_path = str(pathlib.PurePath(*path.parts[idx + 1 :])) - lookup = zipfilecache.get(zip_path) - if lookup is None: - zip = zipfile.ZipFile(zip_path, "r") - namelist = set(zip.namelist()) - zipfilecache[zip_path] = (zip, namelist) - else: - zip, namelist = lookup + _, namelist = spack.zipcache.get(zip_path) if entry_path in namelist: abs_path = str(path) break @@ -242,13 +233,7 @@ def sha256(self) -> str: idx = path.parts.index("packages.zip") zip_path = str(pathlib.PurePath(*path.parts[: idx + 1])) entry_path = str(pathlib.PurePath(*path.parts[idx + 1 :])) - lookup = zipfilecache.get(zip_path) - if lookup is None: - zip = zipfile.ZipFile(zip_path, "r") - namelist = set(zip.namelist()) - zipfilecache[zip_path] = (zip, namelist) - else: - zip, namelist = lookup + zip, _ = spack.zipcache.get(zip_path) f = zip.open(entry_path, "r") else: f = open(self.path, "rb") diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py index fb78bbe2379..7aed566b86f 100644 --- a/lib/spack/spack/repo.py +++ b/lib/spack/spack/repo.py @@ -26,7 +26,6 @@ import types import uuid import warnings -import zipfile import zipimport from typing import Any, Dict, Generator, List, Optional, Set, Tuple, Type, Union @@ -48,6 +47,7 @@ import spack.util.naming as nm import spack.util.path import spack.util.spack_yaml as syaml +import spack.zipcache #: Package modules are imported as spack.pkg.. ROOT_PYTHON_NAMESPACE = "spack.pkg" @@ -365,17 +365,16 @@ def __getattr__(self, name): class EvenFasterPackageChecker(collections.abc.Mapping): - def __init__(self, packages_path): + def __init__(self, zip_path): # The path of the repository managed by this instance - self.packages_path = packages_path - self.zipfile = zipfile.ZipFile(os.path.join(packages_path, "..", "packages.zip"), "r") + self.zipfile, self.namelist = spack.zipcache.get(zip_path) self.invalidate() def invalidate(self): self.mtime = os.stat(self.zipfile.filename).st_mtime self.pkgs = { f.rstrip("/"): self.mtime - for f in self.zipfile.namelist() + for f in self.namelist if f.endswith("/") and f.count("/") == 1 and f != "./" } @@ -614,9 +613,6 @@ def __init__( cache: "spack.caches.FileCacheType", ): self.checker = package_checker - self.packages_path = self.checker.packages_path - if sys.platform == "win32": - self.packages_path = llnl.path.convert_to_posix_path(self.packages_path) self.namespace = namespace self.indexers: Dict[str, Indexer] = {} @@ -1226,7 +1222,7 @@ def filename_for_package_name(self, pkg_name: str) -> str: def _pkg_checker(self) -> Union[FastPackageChecker, EvenFasterPackageChecker]: if self._fast_package_checker is None: if self.zipimporter: - self._fast_package_checker = EvenFasterPackageChecker(self.packages_path) + self._fast_package_checker = EvenFasterPackageChecker(self.zipimporter.archive) else: self._fast_package_checker = FastPackageChecker(self.packages_path) return self._fast_package_checker diff --git a/lib/spack/spack/zipcache.py b/lib/spack/spack/zipcache.py new file mode 100644 index 00000000000..a8ce75cfde9 --- /dev/null +++ b/lib/spack/spack/zipcache.py @@ -0,0 +1,18 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import zipfile +from typing import Dict, Set, Tuple + +zipfilecache: Dict[str, Tuple[zipfile.ZipFile, Set[str]]] = {} + + +def get(path: str): + if path not in zipfilecache: + file = zipfile.ZipFile(path) + names = set(file.namelist()) + zipfilecache[path] = (file, names) + return file, names + return zipfilecache[path]