1824 lines
69 KiB
Python
1824 lines
69 KiB
Python
# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
|
|
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
|
#
|
|
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
|
"""Spack's installation tracking database.
|
|
|
|
The database serves two purposes:
|
|
|
|
1. It implements a cache on top of a potentially very large Spack
|
|
directory hierarchy, speeding up many operations that would
|
|
otherwise require filesystem access.
|
|
|
|
2. It will allow us to track external installations as well as lost
|
|
packages and their dependencies.
|
|
|
|
Prior to the implementation of this store, a directory layout served
|
|
as the authoritative database of packages in Spack. This module
|
|
provides a cache and a sanity checking mechanism for what is in the
|
|
filesystem.
|
|
"""
|
|
import contextlib
|
|
import datetime
|
|
import os
|
|
import pathlib
|
|
import socket
|
|
import sys
|
|
import time
|
|
from json import JSONDecoder
|
|
from typing import (
|
|
Any,
|
|
Callable,
|
|
Container,
|
|
Dict,
|
|
Generator,
|
|
Iterable,
|
|
List,
|
|
NamedTuple,
|
|
Optional,
|
|
Set,
|
|
Tuple,
|
|
Type,
|
|
Union,
|
|
)
|
|
|
|
try:
|
|
import uuid
|
|
|
|
_use_uuid = True
|
|
except ImportError:
|
|
_use_uuid = False
|
|
pass
|
|
|
|
import llnl.util.filesystem as fs
|
|
import llnl.util.lang
|
|
import llnl.util.tty as tty
|
|
|
|
import spack.deptypes as dt
|
|
import spack.hash_types as ht
|
|
import spack.spec
|
|
import spack.traverse as tr
|
|
import spack.util.lock as lk
|
|
import spack.util.spack_json as sjson
|
|
import spack.version as vn
|
|
from spack.directory_layout import (
|
|
DirectoryLayout,
|
|
DirectoryLayoutError,
|
|
InconsistentInstallDirectoryError,
|
|
)
|
|
from spack.error import SpackError
|
|
from spack.util.crypto import bit_length
|
|
|
|
from .enums import InstallRecordStatus
|
|
|
|
# TODO: Provide an API automatically retyring a build after detecting and
|
|
# TODO: clearing a failure.
|
|
|
|
#: DB goes in this directory underneath the root
|
|
_DB_DIRNAME = ".spack-db"
|
|
|
|
#: DB version. This is stuck in the DB file to track changes in format.
|
|
#: Increment by one when the database format changes.
|
|
#: Versions before 5 were not integers.
|
|
_DB_VERSION = vn.Version("8")
|
|
|
|
#: For any version combinations here, skip reindex when upgrading.
|
|
#: Reindexing can take considerable time and is not always necessary.
|
|
_SKIP_REINDEX = [
|
|
# reindexing takes a significant amount of time, and there's
|
|
# no reason to do it from DB version 0.9.3 to version 5. The
|
|
# only difference is that v5 can contain "deprecated_for"
|
|
# fields. So, skip the reindex for this transition. The new
|
|
# version is saved to disk the first time the DB is written.
|
|
(vn.Version("0.9.3"), vn.Version("5")),
|
|
(vn.Version("5"), vn.Version("6")),
|
|
(vn.Version("6"), vn.Version("7")),
|
|
(vn.Version("6"), vn.Version("8")),
|
|
(vn.Version("7"), vn.Version("8")),
|
|
]
|
|
|
|
#: Default timeout for spack database locks in seconds or None (no timeout).
|
|
#: A balance needs to be struck between quick turnaround for parallel installs
|
|
#: (to avoid excess delays) and waiting long enough when the system is busy
|
|
#: (to ensure the database is updated).
|
|
_DEFAULT_DB_LOCK_TIMEOUT = 120
|
|
|
|
#: Default timeout for spack package locks in seconds or None (no timeout).
|
|
#: A balance needs to be struck between quick turnaround for parallel installs
|
|
#: (to avoid excess delays when performing a parallel installation) and waiting
|
|
#: long enough for the next possible spec to install (to avoid excessive
|
|
#: checking of the last high priority package) or holding on to a lock (to
|
|
#: ensure a failed install is properly tracked).
|
|
_DEFAULT_PKG_LOCK_TIMEOUT = None
|
|
|
|
#: Types of dependencies tracked by the database
|
|
#: We store by DAG hash, so we track the dependencies that the DAG hash includes.
|
|
_TRACKED_DEPENDENCIES = ht.dag_hash.depflag
|
|
|
|
#: Default list of fields written for each install record
|
|
DEFAULT_INSTALL_RECORD_FIELDS = (
|
|
"spec",
|
|
"ref_count",
|
|
"path",
|
|
"installed",
|
|
"explicit",
|
|
"installation_time",
|
|
"deprecated_for",
|
|
)
|
|
|
|
|
|
@llnl.util.lang.memoized
|
|
def _getfqdn():
|
|
"""Memoized version of `getfqdn()`.
|
|
|
|
If we call `getfqdn()` too many times, DNS can be very slow. We only need to call it
|
|
one time per process, so we cache it here.
|
|
|
|
"""
|
|
return socket.getfqdn()
|
|
|
|
|
|
def reader(version: vn.StandardVersion) -> Type["spack.spec.SpecfileReaderBase"]:
|
|
reader_cls = {
|
|
vn.Version("5"): spack.spec.SpecfileV1,
|
|
vn.Version("6"): spack.spec.SpecfileV3,
|
|
vn.Version("7"): spack.spec.SpecfileV4,
|
|
vn.Version("8"): spack.spec.SpecfileV5,
|
|
}
|
|
return reader_cls[version]
|
|
|
|
|
|
def _now() -> float:
|
|
"""Returns the time since the epoch"""
|
|
return time.time()
|
|
|
|
|
|
def _autospec(function):
|
|
"""Decorator that automatically converts the argument of a single-arg
|
|
function to a Spec."""
|
|
|
|
def converter(self, spec_like, *args, **kwargs):
|
|
if not isinstance(spec_like, spack.spec.Spec):
|
|
spec_like = spack.spec.Spec(spec_like)
|
|
return function(self, spec_like, *args, **kwargs)
|
|
|
|
return converter
|
|
|
|
|
|
def normalize_query(installed: Union[bool, InstallRecordStatus]) -> InstallRecordStatus:
|
|
if installed is True:
|
|
installed = InstallRecordStatus.INSTALLED
|
|
elif installed is False:
|
|
installed = InstallRecordStatus.MISSING
|
|
return installed
|
|
|
|
|
|
class InstallRecord:
|
|
"""A record represents one installation in the DB.
|
|
|
|
The record keeps track of the spec for the installation, its
|
|
install path, AND whether or not it is installed. We need the
|
|
installed flag in case a user either:
|
|
|
|
a) blew away a directory, or
|
|
b) used spack uninstall -f to get rid of it
|
|
|
|
If, in either case, the package was removed but others still
|
|
depend on it, we still need to track its spec, so we don't
|
|
actually remove from the database until a spec has no installed
|
|
dependents left.
|
|
|
|
Args:
|
|
spec: spec tracked by the install record
|
|
path: path where the spec has been installed
|
|
installed: whether or not the spec is currently installed
|
|
ref_count (int): number of specs that depend on this one
|
|
explicit (bool or None): whether or not this spec was explicitly
|
|
installed, or pulled-in as a dependency of something else
|
|
installation_time (datetime.datetime or None): time of the installation
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
spec: "spack.spec.Spec",
|
|
path: Optional[str],
|
|
installed: bool,
|
|
ref_count: int = 0,
|
|
explicit: bool = False,
|
|
installation_time: Optional[float] = None,
|
|
deprecated_for: Optional[str] = None,
|
|
in_buildcache: bool = False,
|
|
origin: Optional[str] = None,
|
|
) -> None:
|
|
self.spec = spec
|
|
self.path = str(path) if path else None
|
|
self.installed = bool(installed)
|
|
self.ref_count = ref_count
|
|
self.explicit = explicit
|
|
self.installation_time = installation_time or _now()
|
|
self.deprecated_for = deprecated_for
|
|
self.in_buildcache = in_buildcache
|
|
self.origin = origin
|
|
|
|
def install_type_matches(self, installed: InstallRecordStatus) -> bool:
|
|
if self.installed:
|
|
return InstallRecordStatus.INSTALLED in installed
|
|
elif self.deprecated_for:
|
|
return InstallRecordStatus.DEPRECATED in installed
|
|
return InstallRecordStatus.MISSING in installed
|
|
|
|
def to_dict(self, include_fields=DEFAULT_INSTALL_RECORD_FIELDS):
|
|
rec_dict = {}
|
|
|
|
for field_name in include_fields:
|
|
if field_name == "spec":
|
|
rec_dict.update({"spec": self.spec.node_dict_with_hashes()})
|
|
elif field_name == "deprecated_for" and self.deprecated_for:
|
|
rec_dict.update({"deprecated_for": self.deprecated_for})
|
|
else:
|
|
rec_dict.update({field_name: getattr(self, field_name)})
|
|
|
|
if self.origin:
|
|
rec_dict["origin"] = self.origin
|
|
|
|
return rec_dict
|
|
|
|
@classmethod
|
|
def from_dict(cls, spec, dictionary):
|
|
d = dict(dictionary.items())
|
|
d.pop("spec", None)
|
|
|
|
# Old databases may have "None" for path for externals
|
|
if "path" not in d or d["path"] == "None":
|
|
d["path"] = None
|
|
|
|
if "installed" not in d:
|
|
d["installed"] = False
|
|
|
|
return InstallRecord(spec, **d)
|
|
|
|
|
|
class ForbiddenLockError(SpackError):
|
|
"""Raised when an upstream DB attempts to acquire a lock"""
|
|
|
|
|
|
class ForbiddenLock:
|
|
def __getattr__(self, name):
|
|
raise ForbiddenLockError("Cannot access attribute '{0}' of lock".format(name))
|
|
|
|
def __reduce__(self):
|
|
return ForbiddenLock, tuple()
|
|
|
|
|
|
class LockConfiguration(NamedTuple):
|
|
"""Data class to configure locks in Database objects
|
|
|
|
Args:
|
|
enable: whether to enable locks or not.
|
|
database_timeout: timeout for the database lock
|
|
package_timeout: timeout for the package lock
|
|
"""
|
|
|
|
enable: bool
|
|
database_timeout: Optional[int]
|
|
package_timeout: Optional[int]
|
|
|
|
|
|
#: Configure a database to avoid using locks
|
|
NO_LOCK: LockConfiguration = LockConfiguration(
|
|
enable=False, database_timeout=None, package_timeout=None
|
|
)
|
|
|
|
|
|
#: Configure the database to use locks without a timeout
|
|
NO_TIMEOUT: LockConfiguration = LockConfiguration(
|
|
enable=True, database_timeout=None, package_timeout=None
|
|
)
|
|
|
|
#: Default configuration for database locks
|
|
DEFAULT_LOCK_CFG: LockConfiguration = LockConfiguration(
|
|
enable=True,
|
|
database_timeout=_DEFAULT_DB_LOCK_TIMEOUT,
|
|
package_timeout=_DEFAULT_PKG_LOCK_TIMEOUT,
|
|
)
|
|
|
|
|
|
def lock_configuration(configuration):
|
|
"""Return a LockConfiguration from a spack.config.Configuration object."""
|
|
return LockConfiguration(
|
|
enable=configuration.get("config:locks", True),
|
|
database_timeout=configuration.get("config:db_lock_timeout"),
|
|
package_timeout=configuration.get("config:package_lock_timeout"),
|
|
)
|
|
|
|
|
|
def prefix_lock_path(root_dir: Union[str, pathlib.Path]) -> pathlib.Path:
|
|
"""Returns the path of the prefix lock file, given the root directory.
|
|
|
|
Args:
|
|
root_dir: root directory containing the database directory
|
|
"""
|
|
return pathlib.Path(root_dir) / _DB_DIRNAME / "prefix_lock"
|
|
|
|
|
|
def failures_lock_path(root_dir: Union[str, pathlib.Path]) -> pathlib.Path:
|
|
"""Returns the path of the failures lock file, given the root directory.
|
|
|
|
Args:
|
|
root_dir: root directory containing the database directory
|
|
"""
|
|
return pathlib.Path(root_dir) / _DB_DIRNAME / "prefix_failures"
|
|
|
|
|
|
class SpecLocker:
|
|
"""Manages acquiring and releasing read or write locks on concrete specs."""
|
|
|
|
def __init__(self, lock_path: Union[str, pathlib.Path], default_timeout: Optional[float]):
|
|
self.lock_path = pathlib.Path(lock_path)
|
|
self.default_timeout = default_timeout
|
|
|
|
# Maps (spec.dag_hash(), spec.name) to the corresponding lock object
|
|
self.locks: Dict[Tuple[str, str], lk.Lock] = {}
|
|
|
|
def lock(self, spec: "spack.spec.Spec", timeout: Optional[float] = None) -> lk.Lock:
|
|
"""Returns a lock on a concrete spec.
|
|
|
|
The lock is a byte range lock on the nth byte of a file.
|
|
|
|
The lock file is ``self.lock_path``.
|
|
|
|
n is the sys.maxsize-bit prefix of the DAG hash. This makes likelihood of collision is
|
|
very low AND it gives us readers-writer lock semantics with just a single lockfile, so
|
|
no cleanup required.
|
|
"""
|
|
assert spec.concrete, "cannot lock a non-concrete spec"
|
|
timeout = timeout or self.default_timeout
|
|
key = self._lock_key(spec)
|
|
|
|
if key not in self.locks:
|
|
self.locks[key] = self.raw_lock(spec, timeout=timeout)
|
|
else:
|
|
self.locks[key].default_timeout = timeout
|
|
|
|
return self.locks[key]
|
|
|
|
def raw_lock(self, spec: "spack.spec.Spec", timeout: Optional[float] = None) -> lk.Lock:
|
|
"""Returns a raw lock for a Spec, but doesn't keep track of it."""
|
|
return lk.Lock(
|
|
str(self.lock_path),
|
|
start=spec.dag_hash_bit_prefix(bit_length(sys.maxsize)),
|
|
length=1,
|
|
default_timeout=timeout,
|
|
desc=spec.name,
|
|
)
|
|
|
|
def has_lock(self, spec: "spack.spec.Spec") -> bool:
|
|
"""Returns True if the spec is already managed by this spec locker"""
|
|
return self._lock_key(spec) in self.locks
|
|
|
|
def _lock_key(self, spec: "spack.spec.Spec") -> Tuple[str, str]:
|
|
return (spec.dag_hash(), spec.name)
|
|
|
|
@contextlib.contextmanager
|
|
def write_lock(self, spec: "spack.spec.Spec") -> Generator["SpecLocker", None, None]:
|
|
lock = self.lock(spec)
|
|
lock.acquire_write()
|
|
|
|
try:
|
|
yield self
|
|
except lk.LockError:
|
|
# This addresses the case where a nested lock attempt fails inside
|
|
# of this context manager
|
|
raise
|
|
except (Exception, KeyboardInterrupt):
|
|
lock.release_write()
|
|
raise
|
|
else:
|
|
lock.release_write()
|
|
|
|
def clear(self, spec: "spack.spec.Spec") -> Tuple[bool, Optional[lk.Lock]]:
|
|
key = self._lock_key(spec)
|
|
lock = self.locks.pop(key, None)
|
|
return bool(lock), lock
|
|
|
|
def clear_all(self, clear_fn: Optional[Callable[[lk.Lock], Any]] = None) -> None:
|
|
if clear_fn is not None:
|
|
for lock in self.locks.values():
|
|
clear_fn(lock)
|
|
self.locks.clear()
|
|
|
|
|
|
class FailureTracker:
|
|
"""Tracks installation failures.
|
|
|
|
Prefix failure marking takes the form of a byte range lock on the nth
|
|
byte of a file for coordinating between concurrent parallel build
|
|
processes and a persistent file, named with the full hash and
|
|
containing the spec, in a subdirectory of the database to enable
|
|
persistence across overlapping but separate related build processes.
|
|
|
|
The failure lock file lives alongside the install DB.
|
|
|
|
``n`` is the sys.maxsize-bit prefix of the associated DAG hash to make
|
|
the likelihood of collision very low with no cleanup required.
|
|
"""
|
|
|
|
def __init__(self, root_dir: Union[str, pathlib.Path], default_timeout: Optional[float]):
|
|
#: Ensure a persistent location for dealing with parallel installation
|
|
#: failures (e.g., across near-concurrent processes).
|
|
self.dir = pathlib.Path(root_dir) / _DB_DIRNAME / "failures"
|
|
self.dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
self.locker = SpecLocker(failures_lock_path(root_dir), default_timeout=default_timeout)
|
|
|
|
def clear(self, spec: "spack.spec.Spec", force: bool = False) -> None:
|
|
"""Removes any persistent and cached failure tracking for the spec.
|
|
|
|
see `mark()`.
|
|
|
|
Args:
|
|
spec: the spec whose failure indicators are being removed
|
|
force: True if the failure information should be cleared when a failure lock
|
|
exists for the file, or False if the failure should not be cleared (e.g.,
|
|
it may be associated with a concurrent build)
|
|
"""
|
|
locked = self.lock_taken(spec)
|
|
if locked and not force:
|
|
tty.msg(f"Retaining failure marking for {spec.name} due to lock")
|
|
return
|
|
|
|
if locked:
|
|
tty.warn(f"Removing failure marking despite lock for {spec.name}")
|
|
|
|
succeeded, lock = self.locker.clear(spec)
|
|
if succeeded and lock is not None:
|
|
lock.release_write()
|
|
|
|
if self.persistent_mark(spec):
|
|
path = self._path(spec)
|
|
tty.debug(f"Removing failure marking for {spec.name}")
|
|
try:
|
|
path.unlink()
|
|
except OSError as err:
|
|
tty.warn(
|
|
f"Unable to remove failure marking for {spec.name} ({str(path)}): {str(err)}"
|
|
)
|
|
|
|
def clear_all(self) -> None:
|
|
"""Force remove install failure tracking files."""
|
|
tty.debug("Releasing prefix failure locks")
|
|
self.locker.clear_all(
|
|
clear_fn=lambda x: x.release_write() if x.is_write_locked() else True
|
|
)
|
|
|
|
tty.debug("Removing prefix failure tracking files")
|
|
try:
|
|
for fail_mark in os.listdir(str(self.dir)):
|
|
try:
|
|
(self.dir / fail_mark).unlink()
|
|
except OSError as exc:
|
|
tty.warn(f"Unable to remove failure marking file {fail_mark}: {str(exc)}")
|
|
except OSError as exc:
|
|
tty.warn(f"Unable to remove failure marking files: {str(exc)}")
|
|
|
|
def mark(self, spec: "spack.spec.Spec") -> lk.Lock:
|
|
"""Marks a spec as failing to install.
|
|
|
|
Args:
|
|
spec: spec that failed to install
|
|
"""
|
|
# Dump the spec to the failure file for (manual) debugging purposes
|
|
path = self._path(spec)
|
|
path.write_text(spec.to_json())
|
|
|
|
# Also ensure a failure lock is taken to prevent cleanup removal
|
|
# of failure status information during a concurrent parallel build.
|
|
if not self.locker.has_lock(spec):
|
|
try:
|
|
mark = self.locker.lock(spec)
|
|
mark.acquire_write()
|
|
except lk.LockTimeoutError:
|
|
# Unlikely that another process failed to install at the same
|
|
# time but log it anyway.
|
|
tty.debug(f"PID {os.getpid()} failed to mark install failure for {spec.name}")
|
|
tty.warn(f"Unable to mark {spec.name} as failed.")
|
|
|
|
return self.locker.lock(spec)
|
|
|
|
def has_failed(self, spec: "spack.spec.Spec") -> bool:
|
|
"""Return True if the spec is marked as failed."""
|
|
# The failure was detected in this process.
|
|
if self.locker.has_lock(spec):
|
|
return True
|
|
|
|
# The failure was detected by a concurrent process (e.g., an srun),
|
|
# which is expected to be holding a write lock if that is the case.
|
|
if self.lock_taken(spec):
|
|
return True
|
|
|
|
# Determine if the spec may have been marked as failed by a separate
|
|
# spack build process running concurrently.
|
|
return self.persistent_mark(spec)
|
|
|
|
def lock_taken(self, spec: "spack.spec.Spec") -> bool:
|
|
"""Return True if another process has a failure lock on the spec."""
|
|
check = self.locker.raw_lock(spec)
|
|
return check.is_write_locked()
|
|
|
|
def persistent_mark(self, spec: "spack.spec.Spec") -> bool:
|
|
"""Determine if the spec has a persistent failure marking."""
|
|
return self._path(spec).exists()
|
|
|
|
def _path(self, spec: "spack.spec.Spec") -> pathlib.Path:
|
|
"""Return the path to the spec's failure file, which may not exist."""
|
|
assert spec.concrete, "concrete spec required for failure path"
|
|
return self.dir / f"{spec.name}-{spec.dag_hash()}"
|
|
|
|
|
|
SelectType = Callable[[InstallRecord], bool]
|
|
|
|
|
|
class Database:
|
|
#: Fields written for each install record
|
|
record_fields: Tuple[str, ...] = DEFAULT_INSTALL_RECORD_FIELDS
|
|
|
|
def __init__(
|
|
self,
|
|
root: str,
|
|
*,
|
|
upstream_dbs: Optional[List["Database"]] = None,
|
|
is_upstream: bool = False,
|
|
lock_cfg: LockConfiguration = DEFAULT_LOCK_CFG,
|
|
layout: Optional[DirectoryLayout] = None,
|
|
) -> None:
|
|
"""Database for Spack installations.
|
|
|
|
A Database is a cache of Specs data from ``$prefix/spec.yaml`` files
|
|
in Spack installation directories.
|
|
|
|
Database files (data and lock files) are stored under ``root/.spack-db``, which is
|
|
created if it does not exist. This is the "database directory".
|
|
|
|
The database will attempt to read an ``index.json`` file in the database directory.
|
|
If that does not exist, it will create a database when needed by scanning the entire
|
|
store root for ``spec.json`` files according to Spack's directory layout.
|
|
|
|
Args:
|
|
root: root directory where to create the database directory.
|
|
upstream_dbs: upstream databases for this repository.
|
|
is_upstream: whether this repository is an upstream.
|
|
lock_cfg: configuration for the locks to be used by this repository.
|
|
Relevant only if the repository is not an upstream.
|
|
"""
|
|
self.root = root
|
|
self.database_directory = os.path.join(self.root, _DB_DIRNAME)
|
|
self.layout = layout
|
|
|
|
# Set up layout of database files within the db dir
|
|
self._index_path = os.path.join(self.database_directory, "index.json")
|
|
self._verifier_path = os.path.join(self.database_directory, "index_verifier")
|
|
self._lock_path = os.path.join(self.database_directory, "lock")
|
|
|
|
# Create needed directories and files
|
|
if not is_upstream and not os.path.exists(self.database_directory):
|
|
fs.mkdirp(self.database_directory)
|
|
|
|
self.is_upstream = is_upstream
|
|
self.last_seen_verifier = ""
|
|
# Failed write transactions (interrupted by exceptions) will alert
|
|
# _write. When that happens, we set this flag to indicate that
|
|
# future read/write transactions should re-read the DB. Normally it
|
|
# would make more sense to resolve this at the end of the transaction
|
|
# but typically a failed transaction will terminate the running
|
|
# instance of Spack and we don't want to incur an extra read in that
|
|
# case, so we defer the cleanup to when we begin the next transaction
|
|
self._state_is_inconsistent = False
|
|
|
|
# initialize rest of state.
|
|
self.db_lock_timeout = lock_cfg.database_timeout
|
|
tty.debug("DATABASE LOCK TIMEOUT: {0}s".format(str(self.db_lock_timeout)))
|
|
|
|
self.lock: Union[ForbiddenLock, lk.Lock]
|
|
if self.is_upstream:
|
|
self.lock = ForbiddenLock()
|
|
else:
|
|
self.lock = lk.Lock(
|
|
self._lock_path,
|
|
default_timeout=self.db_lock_timeout,
|
|
desc="database",
|
|
enable=lock_cfg.enable,
|
|
)
|
|
self._data: Dict[str, InstallRecord] = {}
|
|
|
|
# For every installed spec we keep track of its install prefix, so that
|
|
# we can answer the simple query whether a given path is already taken
|
|
# before installing a different spec.
|
|
self._installed_prefixes: Set[str] = set()
|
|
|
|
self.upstream_dbs = list(upstream_dbs) if upstream_dbs else []
|
|
|
|
self._write_transaction_impl = lk.WriteTransaction
|
|
self._read_transaction_impl = lk.ReadTransaction
|
|
|
|
def write_transaction(self):
|
|
"""Get a write lock context manager for use in a `with` block."""
|
|
return self._write_transaction_impl(self.lock, acquire=self._read, release=self._write)
|
|
|
|
def read_transaction(self):
|
|
"""Get a read lock context manager for use in a `with` block."""
|
|
return self._read_transaction_impl(self.lock, acquire=self._read)
|
|
|
|
def _write_to_file(self, stream):
|
|
"""Write out the database in JSON format to the stream passed
|
|
as argument.
|
|
|
|
This function does not do any locking or transactions.
|
|
"""
|
|
# map from per-spec hash code to installation record.
|
|
installs = dict(
|
|
(k, v.to_dict(include_fields=self.record_fields)) for k, v in self._data.items()
|
|
)
|
|
|
|
# database includes installation list and version.
|
|
|
|
# NOTE: this DB version does not handle multiple installs of
|
|
# the same spec well. If there are 2 identical specs with
|
|
# different paths, it can't differentiate.
|
|
# TODO: fix this before we support multiple install locations.
|
|
database = {
|
|
"database": {
|
|
# TODO: move this to a top-level _meta section if we ever
|
|
# TODO: bump the DB version to 7
|
|
"version": str(_DB_VERSION),
|
|
# dictionary of installation records, keyed by DAG hash
|
|
"installs": installs,
|
|
}
|
|
}
|
|
|
|
try:
|
|
sjson.dump(database, stream)
|
|
except (TypeError, ValueError) as e:
|
|
raise sjson.SpackJSONError("error writing JSON database:", str(e))
|
|
|
|
def _read_spec_from_dict(self, spec_reader, hash_key, installs, hash=ht.dag_hash):
|
|
"""Recursively construct a spec from a hash in a YAML database.
|
|
|
|
Does not do any locking.
|
|
"""
|
|
spec_dict = installs[hash_key]["spec"]
|
|
|
|
# Install records don't include hash with spec, so we add it in here
|
|
# to ensure it is read properly.
|
|
if "name" not in spec_dict.keys():
|
|
# old format, can't update format here
|
|
for name in spec_dict:
|
|
spec_dict[name]["hash"] = hash_key
|
|
else:
|
|
# new format, already a singleton
|
|
spec_dict[hash.name] = hash_key
|
|
|
|
# Build spec from dict first.
|
|
return spec_reader.from_node_dict(spec_dict)
|
|
|
|
def db_for_spec_hash(self, hash_key):
|
|
with self.read_transaction():
|
|
if hash_key in self._data:
|
|
return self
|
|
|
|
for db in self.upstream_dbs:
|
|
if hash_key in db._data:
|
|
return db
|
|
|
|
def query_by_spec_hash(
|
|
self, hash_key: str, data: Optional[Dict[str, InstallRecord]] = None
|
|
) -> Tuple[bool, Optional[InstallRecord]]:
|
|
"""Get a spec for hash, and whether it's installed upstream.
|
|
|
|
Return:
|
|
(tuple): (bool, optional InstallRecord): bool tells us whether
|
|
the spec is installed upstream. Its InstallRecord is also
|
|
returned if it's installed at all; otherwise None.
|
|
"""
|
|
if data and hash_key in data:
|
|
return False, data[hash_key]
|
|
if not data:
|
|
with self.read_transaction():
|
|
if hash_key in self._data:
|
|
return False, self._data[hash_key]
|
|
for db in self.upstream_dbs:
|
|
if hash_key in db._data:
|
|
return True, db._data[hash_key]
|
|
return False, None
|
|
|
|
def query_local_by_spec_hash(self, hash_key):
|
|
"""Get a spec by hash in the local database
|
|
|
|
Return:
|
|
(InstallRecord or None): InstallRecord when installed
|
|
locally, otherwise None."""
|
|
with self.read_transaction():
|
|
return self._data.get(hash_key, None)
|
|
|
|
def _assign_dependencies(
|
|
self,
|
|
spec_reader: Type["spack.spec.SpecfileReaderBase"],
|
|
hash_key: str,
|
|
installs: dict,
|
|
data: Dict[str, InstallRecord],
|
|
):
|
|
# Add dependencies from other records in the install DB to
|
|
# form a full spec.
|
|
spec = data[hash_key].spec
|
|
spec_node_dict = installs[hash_key]["spec"]
|
|
if "name" not in spec_node_dict:
|
|
# old format
|
|
spec_node_dict = spec_node_dict[spec.name]
|
|
if "dependencies" in spec_node_dict:
|
|
yaml_deps = spec_node_dict["dependencies"]
|
|
for dname, dhash, dtypes, _, virtuals in spec_reader.read_specfile_dep_specs(
|
|
yaml_deps
|
|
):
|
|
# It is important that we always check upstream installations in the same order,
|
|
# and that we always check the local installation first: if a downstream Spack
|
|
# installs a package then dependents in that installation could be using it. If a
|
|
# hash is installed locally and upstream, there isn't enough information to
|
|
# determine which one a local package depends on, so the convention ensures that
|
|
# this isn't an issue.
|
|
_, record = self.query_by_spec_hash(dhash, data=data)
|
|
child = record.spec if record else None
|
|
|
|
if not child:
|
|
tty.warn(
|
|
f"Missing dependency not in database: "
|
|
f"{spec.cformat('{name}{/hash:7}')} needs {dname}-{dhash[:7]}"
|
|
)
|
|
continue
|
|
|
|
spec._add_dependency(child, depflag=dt.canonicalize(dtypes), virtuals=virtuals)
|
|
|
|
def _read_from_file(self, filename):
|
|
"""Fill database from file, do not maintain old data.
|
|
Translate the spec portions from node-dict form to spec form.
|
|
|
|
Does not do any locking.
|
|
"""
|
|
try:
|
|
with open(filename, "r") as f:
|
|
# In the future we may use a stream of JSON objects, hence `raw_decode` for compat.
|
|
fdata, _ = JSONDecoder().raw_decode(f.read())
|
|
except Exception as e:
|
|
raise CorruptDatabaseError("error parsing database:", str(e)) from e
|
|
|
|
if fdata is None:
|
|
return
|
|
|
|
def check(cond, msg):
|
|
if not cond:
|
|
raise CorruptDatabaseError("Spack database is corrupt: %s" % msg, self._index_path)
|
|
|
|
check("database" in fdata, "no 'database' attribute in JSON DB.")
|
|
|
|
# High-level file checks
|
|
db = fdata["database"]
|
|
check("version" in db, "no 'version' in JSON DB.")
|
|
|
|
# TODO: better version checking semantics.
|
|
version = vn.Version(db["version"])
|
|
if version > _DB_VERSION:
|
|
raise InvalidDatabaseVersionError(self, _DB_VERSION, version)
|
|
elif version < _DB_VERSION and not any(
|
|
old == version and new == _DB_VERSION for old, new in _SKIP_REINDEX
|
|
):
|
|
tty.warn(f"Spack database version changed from {version} to {_DB_VERSION}. Upgrading.")
|
|
|
|
self.reindex()
|
|
installs = dict(
|
|
(k, v.to_dict(include_fields=self._record_fields)) for k, v in self._data.items()
|
|
)
|
|
else:
|
|
check("installs" in db, "no 'installs' in JSON DB.")
|
|
installs = db["installs"]
|
|
|
|
spec_reader = reader(version)
|
|
|
|
def invalid_record(hash_key, error):
|
|
return CorruptDatabaseError(
|
|
f"Invalid record in Spack database: hash: {hash_key}, cause: "
|
|
f"{type(error).__name__}: {error}",
|
|
self._index_path,
|
|
)
|
|
|
|
# Build up the database in three passes:
|
|
#
|
|
# 1. Read in all specs without dependencies.
|
|
# 2. Hook dependencies up among specs.
|
|
# 3. Mark all specs concrete.
|
|
#
|
|
# The database is built up so that ALL specs in it share nodes
|
|
# (i.e., its specs are a true Merkle DAG, unlike most specs.)
|
|
|
|
# Pass 1: Iterate through database and build specs w/o dependencies
|
|
data: Dict[str, InstallRecord] = {}
|
|
installed_prefixes: Set[str] = set()
|
|
for hash_key, rec in installs.items():
|
|
try:
|
|
# This constructs a spec DAG from the list of all installs
|
|
spec = self._read_spec_from_dict(spec_reader, hash_key, installs)
|
|
|
|
# Insert the brand new spec in the database. Each
|
|
# spec has its own copies of its dependency specs.
|
|
# TODO: would a more immmutable spec implementation simplify
|
|
# this?
|
|
data[hash_key] = InstallRecord.from_dict(spec, rec)
|
|
|
|
if not spec.external and "installed" in rec and rec["installed"]:
|
|
installed_prefixes.add(rec["path"])
|
|
except Exception as e:
|
|
raise invalid_record(hash_key, e) from e
|
|
|
|
# Pass 2: Assign dependencies once all specs are created.
|
|
for hash_key in data:
|
|
try:
|
|
self._assign_dependencies(spec_reader, hash_key, installs, data)
|
|
except MissingDependenciesError:
|
|
raise
|
|
except Exception as e:
|
|
raise invalid_record(hash_key, e) from e
|
|
|
|
# Pass 3: Mark all specs concrete. Specs representing real
|
|
# installations must be explicitly marked.
|
|
# We do this *after* all dependencies are connected because if we
|
|
# do it *while* we're constructing specs,it causes hashes to be
|
|
# cached prematurely.
|
|
for hash_key, rec in data.items():
|
|
rec.spec._mark_root_concrete()
|
|
|
|
self._data = data
|
|
self._installed_prefixes = installed_prefixes
|
|
|
|
def reindex(self):
|
|
"""Build database index from scratch based on a directory layout.
|
|
|
|
Locks the DB if it isn't locked already.
|
|
"""
|
|
if self.is_upstream:
|
|
raise UpstreamDatabaseLockingError("Cannot reindex an upstream database")
|
|
|
|
# Special transaction to avoid recursive reindex calls and to
|
|
# ignore errors if we need to rebuild a corrupt database.
|
|
def _read_suppress_error():
|
|
try:
|
|
if os.path.isfile(self._index_path):
|
|
self._read_from_file(self._index_path)
|
|
except CorruptDatabaseError as e:
|
|
tty.warn(f"Reindexing corrupt database, error was: {e}")
|
|
self._data = {}
|
|
self._installed_prefixes = set()
|
|
|
|
with lk.WriteTransaction(self.lock, acquire=_read_suppress_error, release=self._write):
|
|
old_installed_prefixes, self._installed_prefixes = self._installed_prefixes, set()
|
|
old_data, self._data = self._data, {}
|
|
try:
|
|
self._reindex(old_data)
|
|
except BaseException:
|
|
# If anything explodes, restore old data, skip write.
|
|
self._data = old_data
|
|
self._installed_prefixes = old_installed_prefixes
|
|
raise
|
|
|
|
def _reindex(self, old_data: Dict[str, InstallRecord]):
|
|
# Specs on the file system are the source of truth for record.spec. The old database values
|
|
# if available are the source of truth for the rest of the record.
|
|
assert self.layout, "Database layout must be set to reindex"
|
|
|
|
specs_from_fs = self.layout.all_specs()
|
|
deprecated_for = self.layout.deprecated_for(specs_from_fs)
|
|
|
|
known_specs: List[spack.spec.Spec] = [
|
|
*specs_from_fs,
|
|
*(deprecated for _, deprecated in deprecated_for),
|
|
*(rec.spec for rec in old_data.values()),
|
|
]
|
|
|
|
upstream_hashes = {
|
|
dag_hash for upstream in self.upstream_dbs for dag_hash in upstream._data
|
|
}
|
|
upstream_hashes.difference_update(spec.dag_hash() for spec in known_specs)
|
|
|
|
def create_node(edge: spack.spec.DependencySpec, is_upstream: bool):
|
|
if is_upstream:
|
|
return
|
|
|
|
self._data[edge.spec.dag_hash()] = InstallRecord(
|
|
spec=edge.spec.copy(deps=False),
|
|
path=edge.spec.external_path if edge.spec.external else None,
|
|
installed=edge.spec.external,
|
|
)
|
|
|
|
# Store all nodes of known specs, excluding ones found in upstreams
|
|
tr.traverse_breadth_first_with_visitor(
|
|
known_specs,
|
|
tr.CoverNodesVisitor(
|
|
NoUpstreamVisitor(upstream_hashes, create_node), key=tr.by_dag_hash
|
|
),
|
|
)
|
|
|
|
# Store the prefix and other information for specs were found on the file system
|
|
for s in specs_from_fs:
|
|
record = self._data[s.dag_hash()]
|
|
record.path = s.prefix
|
|
record.installed = True
|
|
record.explicit = True # conservative assumption
|
|
record.installation_time = os.stat(s.prefix).st_ctime
|
|
|
|
# Deprecate specs
|
|
for new, old in deprecated_for:
|
|
self._data[old.dag_hash()].deprecated_for = new.dag_hash()
|
|
|
|
# Copy data we have from the old database
|
|
for old_record in old_data.values():
|
|
record = self._data[old_record.spec.dag_hash()]
|
|
record.explicit = old_record.explicit
|
|
record.installation_time = old_record.installation_time
|
|
record.origin = old_record.origin
|
|
record.deprecated_for = old_record.deprecated_for
|
|
|
|
# Warn when the spec has been removed from the file system (i.e. it was not detected)
|
|
if not record.installed and old_record.installed:
|
|
tty.warn(
|
|
f"Spec {old_record.spec.short_spec} was marked installed in the database "
|
|
"but was not found on the file system. It is now marked as missing."
|
|
)
|
|
|
|
def create_edge(edge: spack.spec.DependencySpec, is_upstream: bool):
|
|
if not edge.parent:
|
|
return
|
|
parent_record = self._data[edge.parent.dag_hash()]
|
|
if is_upstream:
|
|
upstream, child_record = self.query_by_spec_hash(edge.spec.dag_hash())
|
|
assert upstream and child_record, "Internal error: upstream spec not found"
|
|
else:
|
|
child_record = self._data[edge.spec.dag_hash()]
|
|
parent_record.spec._add_dependency(
|
|
child_record.spec, depflag=edge.depflag, virtuals=edge.virtuals
|
|
)
|
|
|
|
# Then store edges
|
|
tr.traverse_breadth_first_with_visitor(
|
|
known_specs,
|
|
tr.CoverEdgesVisitor(
|
|
NoUpstreamVisitor(upstream_hashes, create_edge), key=tr.by_dag_hash
|
|
),
|
|
)
|
|
|
|
# Finally update the ref counts
|
|
for record in self._data.values():
|
|
for dep in record.spec.dependencies(deptype=_TRACKED_DEPENDENCIES):
|
|
dep_record = self._data.get(dep.dag_hash())
|
|
if dep_record: # dep might be upstream
|
|
dep_record.ref_count += 1
|
|
if record.deprecated_for:
|
|
self._data[record.deprecated_for].ref_count += 1
|
|
|
|
self._check_ref_counts()
|
|
|
|
def _check_ref_counts(self):
|
|
"""Ensure consistency of reference counts in the DB.
|
|
|
|
Raise an AssertionError if something is amiss.
|
|
|
|
Does no locking.
|
|
"""
|
|
counts: Dict[str, int] = {}
|
|
for key, rec in self._data.items():
|
|
counts.setdefault(key, 0)
|
|
for dep in rec.spec.dependencies(deptype=_TRACKED_DEPENDENCIES):
|
|
dep_key = dep.dag_hash()
|
|
counts.setdefault(dep_key, 0)
|
|
counts[dep_key] += 1
|
|
|
|
if rec.deprecated_for:
|
|
counts.setdefault(rec.deprecated_for, 0)
|
|
counts[rec.deprecated_for] += 1
|
|
|
|
for rec in self._data.values():
|
|
key = rec.spec.dag_hash()
|
|
expected = counts[key]
|
|
found = rec.ref_count
|
|
if not expected == found:
|
|
raise AssertionError(
|
|
"Invalid ref_count: %s: %d (expected %d), in DB %s"
|
|
% (key, found, expected, self._index_path)
|
|
)
|
|
|
|
def _write(self, type, value, traceback):
|
|
"""Write the in-memory database index to its file path.
|
|
|
|
This is a helper function called by the WriteTransaction context
|
|
manager. If there is an exception while the write lock is active,
|
|
nothing will be written to the database file, but the in-memory
|
|
database *may* be left in an inconsistent state. It will be consistent
|
|
after the start of the next transaction, when it read from disk again.
|
|
|
|
This routine does no locking.
|
|
"""
|
|
# Do not write if exceptions were raised
|
|
if type is not None:
|
|
# A failure interrupted a transaction, so we should record that
|
|
# the Database is now in an inconsistent state: we should
|
|
# restore it in the next transaction
|
|
self._state_is_inconsistent = True
|
|
return
|
|
|
|
temp_file = self._index_path + (".%s.%s.temp" % (_getfqdn(), os.getpid()))
|
|
|
|
# Write a temporary database file them move it into place
|
|
try:
|
|
with open(temp_file, "w") as f:
|
|
self._write_to_file(f)
|
|
fs.rename(temp_file, self._index_path)
|
|
|
|
if _use_uuid:
|
|
with open(self._verifier_path, "w") as f:
|
|
new_verifier = str(uuid.uuid4())
|
|
f.write(new_verifier)
|
|
self.last_seen_verifier = new_verifier
|
|
except BaseException as e:
|
|
tty.debug(e)
|
|
# Clean up temp file if something goes wrong.
|
|
if os.path.exists(temp_file):
|
|
os.remove(temp_file)
|
|
raise
|
|
|
|
def _read(self):
|
|
"""Re-read Database from the data in the set location. This does no locking."""
|
|
if os.path.isfile(self._index_path):
|
|
current_verifier = ""
|
|
if _use_uuid:
|
|
try:
|
|
with open(self._verifier_path, "r") as f:
|
|
current_verifier = f.read()
|
|
except BaseException:
|
|
pass
|
|
if (current_verifier != self.last_seen_verifier) or (current_verifier == ""):
|
|
self.last_seen_verifier = current_verifier
|
|
# Read from file if a database exists
|
|
self._read_from_file(self._index_path)
|
|
elif self._state_is_inconsistent:
|
|
self._read_from_file(self._index_path)
|
|
self._state_is_inconsistent = False
|
|
return
|
|
elif self.is_upstream:
|
|
tty.warn("upstream not found: {0}".format(self._index_path))
|
|
|
|
def _add(
|
|
self,
|
|
spec: "spack.spec.Spec",
|
|
explicit: bool = False,
|
|
installation_time: Optional[float] = None,
|
|
allow_missing: bool = False,
|
|
):
|
|
"""Add an install record for this spec to the database.
|
|
|
|
Also ensures dependencies are present and updated in the DB as either installed or missing.
|
|
|
|
Args:
|
|
spec: spec to be added
|
|
explicit:
|
|
Possible values: True, False, any
|
|
|
|
A spec that was installed following a specific user request is marked as explicit.
|
|
If instead it was pulled-in as a dependency of a user requested spec it's
|
|
considered implicit.
|
|
|
|
installation_time:
|
|
Date and time of installation
|
|
allow_missing: if True, don't warn when installation is not found on on disk
|
|
This is useful when installing specs without build deps.
|
|
"""
|
|
if not spec.concrete:
|
|
raise NonConcreteSpecAddError("Specs added to DB must be concrete.")
|
|
|
|
key = spec.dag_hash()
|
|
spec_pkg_hash = spec._package_hash # type: ignore[attr-defined]
|
|
upstream, record = self.query_by_spec_hash(key)
|
|
if upstream:
|
|
return
|
|
|
|
installation_time = installation_time or _now()
|
|
|
|
for edge in spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES):
|
|
if edge.spec.dag_hash() in self._data:
|
|
continue
|
|
self._add(
|
|
edge.spec,
|
|
explicit=False,
|
|
installation_time=installation_time,
|
|
# allow missing build-only deps. This prevents excessive warnings when a spec is
|
|
# installed, and its build dep is missing a build dep; there's no need to install
|
|
# the build dep's build dep first, and there's no need to warn about it missing.
|
|
allow_missing=allow_missing or edge.depflag == dt.BUILD,
|
|
)
|
|
|
|
# Make sure the directory layout agrees whether the spec is installed
|
|
if not spec.external and self.layout:
|
|
path = self.layout.path_for_spec(spec)
|
|
installed = False
|
|
try:
|
|
self.layout.ensure_installed(spec)
|
|
installed = True
|
|
self._installed_prefixes.add(path)
|
|
except DirectoryLayoutError as e:
|
|
if not (allow_missing and isinstance(e, InconsistentInstallDirectoryError)):
|
|
action = "updated" if key in self._data else "registered"
|
|
tty.warn(
|
|
f"{spec.short_spec} is being {action} in the database with prefix {path}, "
|
|
"but this directory does not contain an installation of "
|
|
f"the spec, due to: {e}"
|
|
)
|
|
elif spec.external_path:
|
|
path = spec.external_path
|
|
installed = True
|
|
else:
|
|
path = None
|
|
installed = True
|
|
|
|
if key not in self._data:
|
|
# Create a new install record with no deps initially.
|
|
new_spec = spec.copy(deps=False)
|
|
self._data[key] = InstallRecord(
|
|
new_spec,
|
|
path=path,
|
|
installed=installed,
|
|
ref_count=0,
|
|
explicit=explicit,
|
|
installation_time=installation_time,
|
|
origin=None if not hasattr(spec, "origin") else spec.origin,
|
|
)
|
|
|
|
# Connect dependencies from the DB to the new copy.
|
|
for dep in spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES):
|
|
dkey = dep.spec.dag_hash()
|
|
upstream, record = self.query_by_spec_hash(dkey)
|
|
assert record, f"Missing dependency {dep.spec.short_spec} in DB"
|
|
new_spec._add_dependency(record.spec, depflag=dep.depflag, virtuals=dep.virtuals)
|
|
if not upstream:
|
|
record.ref_count += 1
|
|
|
|
# Mark concrete once everything is built, and preserve the original hashes of concrete
|
|
# specs.
|
|
new_spec._mark_concrete()
|
|
new_spec._hash = key
|
|
new_spec._package_hash = spec_pkg_hash
|
|
|
|
else:
|
|
# It is already in the database
|
|
self._data[key].installed = installed
|
|
self._data[key].installation_time = _now()
|
|
|
|
self._data[key].explicit = explicit
|
|
|
|
@_autospec
|
|
def add(self, spec: "spack.spec.Spec", *, explicit: bool = False, allow_missing=False) -> None:
|
|
"""Add spec at path to database, locking and reading DB to sync.
|
|
|
|
``add()`` will lock and read from the DB on disk.
|
|
|
|
"""
|
|
# TODO: ensure that spec is concrete?
|
|
# Entire add is transactional.
|
|
with self.write_transaction():
|
|
self._add(spec, explicit=explicit, allow_missing=allow_missing)
|
|
|
|
def _get_matching_spec_key(self, spec: "spack.spec.Spec", **kwargs) -> str:
|
|
"""Get the exact spec OR get a single spec that matches."""
|
|
key = spec.dag_hash()
|
|
upstream, record = self.query_by_spec_hash(key)
|
|
if not record:
|
|
match = self.query_one(spec, **kwargs)
|
|
if match:
|
|
return match.dag_hash()
|
|
raise NoSuchSpecError(spec)
|
|
return key
|
|
|
|
@_autospec
|
|
def get_record(self, spec: "spack.spec.Spec", **kwargs) -> Optional[InstallRecord]:
|
|
key = self._get_matching_spec_key(spec, **kwargs)
|
|
upstream, record = self.query_by_spec_hash(key)
|
|
return record
|
|
|
|
def _decrement_ref_count(self, spec: "spack.spec.Spec") -> None:
|
|
key = spec.dag_hash()
|
|
|
|
if key not in self._data:
|
|
# TODO: print something here? DB is corrupt, but
|
|
# not much we can do.
|
|
return
|
|
|
|
rec = self._data[key]
|
|
rec.ref_count -= 1
|
|
|
|
if rec.ref_count == 0 and not rec.installed:
|
|
del self._data[key]
|
|
|
|
for dep in spec.dependencies(deptype=_TRACKED_DEPENDENCIES):
|
|
self._decrement_ref_count(dep)
|
|
|
|
def _increment_ref_count(self, spec: "spack.spec.Spec") -> None:
|
|
key = spec.dag_hash()
|
|
|
|
if key not in self._data:
|
|
return
|
|
|
|
rec = self._data[key]
|
|
rec.ref_count += 1
|
|
|
|
def _remove(self, spec: "spack.spec.Spec") -> "spack.spec.Spec":
|
|
"""Non-locking version of remove(); does real work."""
|
|
key = self._get_matching_spec_key(spec)
|
|
rec = self._data[key]
|
|
|
|
# This install prefix is now free for other specs to use, even if the
|
|
# spec is only marked uninstalled.
|
|
if not rec.spec.external and rec.installed and rec.path:
|
|
self._installed_prefixes.remove(rec.path)
|
|
|
|
if rec.ref_count > 0:
|
|
rec.installed = False
|
|
return rec.spec
|
|
|
|
del self._data[key]
|
|
|
|
# Remove any reference to this node from dependencies and
|
|
# decrement the reference count
|
|
rec.spec.detach(deptype=_TRACKED_DEPENDENCIES)
|
|
for dep in rec.spec.dependencies(deptype=_TRACKED_DEPENDENCIES):
|
|
self._decrement_ref_count(dep)
|
|
|
|
if rec.deprecated_for:
|
|
new_spec = self._data[rec.deprecated_for].spec
|
|
self._decrement_ref_count(new_spec)
|
|
|
|
# Returns the concrete spec so we know it in the case where a
|
|
# query spec was passed in.
|
|
return rec.spec
|
|
|
|
@_autospec
|
|
def remove(self, spec: "spack.spec.Spec") -> "spack.spec.Spec":
|
|
"""Removes a spec from the database. To be called on uninstall.
|
|
|
|
Reads the database, then:
|
|
|
|
1. Marks the spec as not installed.
|
|
2. Removes the spec if it has no more dependents.
|
|
3. If removed, recursively updates dependencies' ref counts
|
|
and removes them if they are no longer needed.
|
|
|
|
"""
|
|
# Take a lock around the entire removal.
|
|
with self.write_transaction():
|
|
return self._remove(spec)
|
|
|
|
def deprecator(self, spec: "spack.spec.Spec") -> Optional["spack.spec.Spec"]:
|
|
"""Return the spec that the given spec is deprecated for, or None"""
|
|
with self.read_transaction():
|
|
spec_key = self._get_matching_spec_key(spec)
|
|
spec_rec = self._data[spec_key]
|
|
|
|
if spec_rec.deprecated_for:
|
|
return self._data[spec_rec.deprecated_for].spec
|
|
else:
|
|
return None
|
|
|
|
def specs_deprecated_by(self, spec: "spack.spec.Spec") -> List["spack.spec.Spec"]:
|
|
"""Return all specs deprecated in favor of the given spec"""
|
|
with self.read_transaction():
|
|
return [
|
|
rec.spec for rec in self._data.values() if rec.deprecated_for == spec.dag_hash()
|
|
]
|
|
|
|
def _deprecate(self, spec: "spack.spec.Spec", deprecator: "spack.spec.Spec") -> None:
|
|
spec_key = self._get_matching_spec_key(spec)
|
|
spec_rec = self._data[spec_key]
|
|
|
|
deprecator_key = self._get_matching_spec_key(deprecator)
|
|
|
|
self._increment_ref_count(deprecator)
|
|
|
|
# If spec was already deprecated, update old deprecator's ref count
|
|
if spec_rec.deprecated_for:
|
|
old_repl_rec = self._data[spec_rec.deprecated_for]
|
|
self._decrement_ref_count(old_repl_rec.spec)
|
|
|
|
spec_rec.deprecated_for = deprecator_key
|
|
spec_rec.installed = False
|
|
self._data[spec_key] = spec_rec
|
|
|
|
@_autospec
|
|
def mark(self, spec: "spack.spec.Spec", key: str, value: Any) -> None:
|
|
"""Mark an arbitrary record on a spec."""
|
|
with self.write_transaction():
|
|
return self._mark(spec, key, value)
|
|
|
|
def _mark(self, spec: "spack.spec.Spec", key, value) -> None:
|
|
record = self._data[self._get_matching_spec_key(spec)]
|
|
setattr(record, key, value)
|
|
|
|
@_autospec
|
|
def deprecate(self, spec: "spack.spec.Spec", deprecator: "spack.spec.Spec") -> None:
|
|
"""Marks a spec as deprecated in favor of its deprecator"""
|
|
with self.write_transaction():
|
|
return self._deprecate(spec, deprecator)
|
|
|
|
@_autospec
|
|
def installed_relatives(
|
|
self,
|
|
spec: "spack.spec.Spec",
|
|
direction: str = "children",
|
|
transitive: bool = True,
|
|
deptype: Union[dt.DepFlag, dt.DepTypes] = dt.ALL,
|
|
) -> Set["spack.spec.Spec"]:
|
|
"""Return installed specs related to this one."""
|
|
if direction not in ("parents", "children"):
|
|
raise ValueError("Invalid direction: %s" % direction)
|
|
|
|
relatives: Set[spack.spec.Spec] = set()
|
|
for spec in self.query(spec):
|
|
if transitive:
|
|
to_add = spec.traverse(direction=direction, root=False, deptype=deptype)
|
|
elif direction == "parents":
|
|
to_add = spec.dependents(deptype=deptype)
|
|
else: # direction == 'children'
|
|
to_add = spec.dependencies(deptype=deptype)
|
|
|
|
for relative in to_add:
|
|
hash_key = relative.dag_hash()
|
|
_, record = self.query_by_spec_hash(hash_key)
|
|
if not record:
|
|
tty.warn(
|
|
f"Inconsistent state: "
|
|
f"{'dependent' if direction == 'parents' else 'dependency'} {hash_key} of "
|
|
f"{spec.dag_hash()} not in DB"
|
|
)
|
|
continue
|
|
|
|
if not record.installed:
|
|
continue
|
|
|
|
relatives.add(relative)
|
|
return relatives
|
|
|
|
@_autospec
|
|
def installed_extensions_for(self, extendee_spec: "spack.spec.Spec"):
|
|
"""Returns the specs of all packages that extend the given spec"""
|
|
for spec in self.query():
|
|
if spec.package.extends(extendee_spec):
|
|
yield spec.package
|
|
|
|
def _get_by_hash_local(
|
|
self,
|
|
dag_hash: str,
|
|
default: Optional[List["spack.spec.Spec"]] = None,
|
|
installed: Union[bool, InstallRecordStatus] = InstallRecordStatus.ANY,
|
|
) -> Optional[List["spack.spec.Spec"]]:
|
|
installed = normalize_query(installed)
|
|
# hash is a full hash and is in the data somewhere
|
|
if dag_hash in self._data:
|
|
rec = self._data[dag_hash]
|
|
if rec.install_type_matches(installed):
|
|
return [rec.spec]
|
|
else:
|
|
return default
|
|
|
|
# check if hash is a prefix of some installed (or previously installed) spec.
|
|
matches = [
|
|
record.spec
|
|
for h, record in self._data.items()
|
|
if h.startswith(dag_hash) and record.install_type_matches(installed)
|
|
]
|
|
if matches:
|
|
return matches
|
|
|
|
# nothing found
|
|
return default
|
|
|
|
def get_by_hash_local(
|
|
self,
|
|
dag_hash: str,
|
|
default: Optional[List["spack.spec.Spec"]] = None,
|
|
installed: Union[bool, InstallRecordStatus] = InstallRecordStatus.ANY,
|
|
) -> Optional[List["spack.spec.Spec"]]:
|
|
"""Look up a spec in *this DB* by DAG hash, or by a DAG hash prefix.
|
|
|
|
Args:
|
|
dag_hash: hash (or hash prefix) to look up
|
|
default: default value to return if dag_hash is not in the DB
|
|
installed: if ``True``, includes only installed specs in the search; if ``False``
|
|
only missing specs. Otherwise, a InstallRecordStatus flag.
|
|
|
|
``installed`` defaults to ``InstallRecordStatus.ANY`` so we can refer to any known hash.
|
|
|
|
``query()`` and ``query_one()`` differ in that they only return installed specs by default.
|
|
"""
|
|
with self.read_transaction():
|
|
return self._get_by_hash_local(dag_hash, default=default, installed=installed)
|
|
|
|
def get_by_hash(
|
|
self,
|
|
dag_hash: str,
|
|
default: Optional[List["spack.spec.Spec"]] = None,
|
|
installed: Union[bool, InstallRecordStatus] = InstallRecordStatus.ANY,
|
|
) -> Optional[List["spack.spec.Spec"]]:
|
|
"""Look up a spec by DAG hash, or by a DAG hash prefix.
|
|
|
|
Args:
|
|
dag_hash: hash (or hash prefix) to look up
|
|
default: default value to return if dag_hash is not in the DB
|
|
installed: if ``True``, includes only installed specs in the search; if ``False``
|
|
only missing specs. Otherwise, a InstallRecordStatus flag.
|
|
|
|
``installed`` defaults to ``InstallRecordStatus.ANY`` so we can refer to any known hash.
|
|
``query()`` and ``query_one()`` differ in that they only return installed specs by default.
|
|
|
|
"""
|
|
|
|
spec = self.get_by_hash_local(dag_hash, default=default, installed=installed)
|
|
if spec is not None:
|
|
return spec
|
|
|
|
for upstream_db in self.upstream_dbs:
|
|
spec = upstream_db._get_by_hash_local(dag_hash, default=default, installed=installed)
|
|
if spec is not None:
|
|
return spec
|
|
|
|
return default
|
|
|
|
def _query(
|
|
self,
|
|
query_spec: Optional[Union[str, "spack.spec.Spec"]] = None,
|
|
*,
|
|
predicate_fn: Optional[SelectType] = None,
|
|
installed: Union[bool, InstallRecordStatus] = True,
|
|
explicit: Optional[bool] = None,
|
|
start_date: Optional[datetime.datetime] = None,
|
|
end_date: Optional[datetime.datetime] = None,
|
|
hashes: Optional[Iterable[str]] = None,
|
|
in_buildcache: Optional[bool] = None,
|
|
origin: Optional[str] = None,
|
|
) -> List["spack.spec.Spec"]:
|
|
installed = normalize_query(installed)
|
|
|
|
# Restrict the set of records over which we iterate first
|
|
matching_hashes = self._data
|
|
if hashes is not None:
|
|
matching_hashes = {h: self._data[h] for h in hashes if h in self._data}
|
|
|
|
if isinstance(query_spec, str):
|
|
query_spec = spack.spec.Spec(query_spec)
|
|
|
|
if query_spec is not None and query_spec.concrete:
|
|
hash_key = query_spec.dag_hash()
|
|
if hash_key not in matching_hashes:
|
|
return []
|
|
matching_hashes = {hash_key: matching_hashes[hash_key]}
|
|
|
|
results = []
|
|
start_date = start_date or datetime.datetime.min
|
|
end_date = end_date or datetime.datetime.max
|
|
|
|
deferred = []
|
|
for rec in matching_hashes.values():
|
|
if origin and not (origin == rec.origin):
|
|
continue
|
|
|
|
if not rec.install_type_matches(installed):
|
|
continue
|
|
|
|
if in_buildcache is not None and rec.in_buildcache != in_buildcache:
|
|
continue
|
|
|
|
if explicit is not None and rec.explicit != explicit:
|
|
continue
|
|
|
|
if predicate_fn is not None and not predicate_fn(rec):
|
|
continue
|
|
|
|
if start_date or end_date:
|
|
inst_date = datetime.datetime.fromtimestamp(rec.installation_time)
|
|
if not (start_date < inst_date < end_date):
|
|
continue
|
|
|
|
if query_spec is None or query_spec.concrete:
|
|
results.append(rec.spec)
|
|
continue
|
|
|
|
# check anon specs and exact name matches first
|
|
if not query_spec.name or rec.spec.name == query_spec.name:
|
|
if rec.spec.satisfies(query_spec):
|
|
results.append(rec.spec)
|
|
|
|
# save potential virtual matches for later, but not if we already found a match
|
|
elif not results:
|
|
deferred.append(rec.spec)
|
|
|
|
# Checking for virtuals is expensive, so we save it for last and only if needed.
|
|
# If we get here, we didn't find anything in the DB that matched by name.
|
|
# If we did fine something, the query spec can't be virtual b/c we matched an actual
|
|
# package installation, so skip the virtual check entirely. If we *didn't* find anything,
|
|
# check all the deferred specs *if* the query is virtual.
|
|
if not results and query_spec is not None and deferred and query_spec.virtual:
|
|
results = [spec for spec in deferred if spec.satisfies(query_spec)]
|
|
|
|
return results
|
|
|
|
def query_local(
|
|
self,
|
|
query_spec: Optional[Union[str, "spack.spec.Spec"]] = None,
|
|
*,
|
|
predicate_fn: Optional[SelectType] = None,
|
|
installed: Union[bool, InstallRecordStatus] = True,
|
|
explicit: Optional[bool] = None,
|
|
start_date: Optional[datetime.datetime] = None,
|
|
end_date: Optional[datetime.datetime] = None,
|
|
hashes: Optional[List[str]] = None,
|
|
in_buildcache: Optional[bool] = None,
|
|
origin: Optional[str] = None,
|
|
) -> List["spack.spec.Spec"]:
|
|
"""Queries the local Spack database.
|
|
|
|
This function doesn't guarantee any sorting of the returned data for performance reason,
|
|
since comparing specs for __lt__ may be an expensive operation.
|
|
|
|
Args:
|
|
query_spec: if query_spec is ``None``, match all specs in the database.
|
|
If it is a spec, return all specs matching ``spec.satisfies(query_spec)``.
|
|
|
|
predicate_fn: optional predicate taking an InstallRecord as argument, and returning
|
|
whether that record is selected for the query. It can be used to craft criteria
|
|
that need some data for selection not provided by the Database itself.
|
|
|
|
installed: if ``True``, includes only installed specs in the search. If ``False`` only
|
|
missing specs, and if ``any``, all specs in database. If an InstallStatus or
|
|
iterable of InstallStatus, returns specs whose install status matches at least
|
|
one of the InstallStatus.
|
|
|
|
explicit: a spec that was installed following a specific user request is marked as
|
|
explicit. If instead it was pulled-in as a dependency of a user requested spec
|
|
it's considered implicit.
|
|
|
|
start_date: if set considers only specs installed from the starting date.
|
|
|
|
end_date: if set considers only specs installed until the ending date.
|
|
|
|
in_buildcache: specs that are marked in this database as part of an associated binary
|
|
cache are ``in_buildcache``. All other specs are not. This field is used for
|
|
querying mirror indices. By default, it does not check this status.
|
|
|
|
hashes: list of hashes used to restrict the search
|
|
|
|
origin: origin of the spec
|
|
"""
|
|
with self.read_transaction():
|
|
return self._query(
|
|
query_spec,
|
|
predicate_fn=predicate_fn,
|
|
installed=installed,
|
|
explicit=explicit,
|
|
start_date=start_date,
|
|
end_date=end_date,
|
|
hashes=hashes,
|
|
in_buildcache=in_buildcache,
|
|
origin=origin,
|
|
)
|
|
|
|
def query(
|
|
self,
|
|
query_spec: Optional[Union[str, "spack.spec.Spec"]] = None,
|
|
*,
|
|
predicate_fn: Optional[SelectType] = None,
|
|
installed: Union[bool, InstallRecordStatus] = True,
|
|
explicit: Optional[bool] = None,
|
|
start_date: Optional[datetime.datetime] = None,
|
|
end_date: Optional[datetime.datetime] = None,
|
|
in_buildcache: Optional[bool] = None,
|
|
hashes: Optional[List[str]] = None,
|
|
origin: Optional[str] = None,
|
|
install_tree: str = "all",
|
|
) -> List["spack.spec.Spec"]:
|
|
"""Queries the Spack database including all upstream databases.
|
|
|
|
Args:
|
|
query_spec: if query_spec is ``None``, match all specs in the database.
|
|
If it is a spec, return all specs matching ``spec.satisfies(query_spec)``.
|
|
|
|
predicate_fn: optional predicate taking an InstallRecord as argument, and returning
|
|
whether that record is selected for the query. It can be used to craft criteria
|
|
that need some data for selection not provided by the Database itself.
|
|
|
|
installed: if ``True``, includes only installed specs in the search. If ``False`` only
|
|
missing specs, and if ``any``, all specs in database. If an InstallStatus or
|
|
iterable of InstallStatus, returns specs whose install status matches at least
|
|
one of the InstallStatus.
|
|
|
|
explicit: a spec that was installed following a specific user request is marked as
|
|
explicit. If instead it was pulled-in as a dependency of a user requested spec
|
|
it's considered implicit.
|
|
|
|
start_date: if set considers only specs installed from the starting date.
|
|
|
|
end_date: if set considers only specs installed until the ending date.
|
|
|
|
in_buildcache: specs that are marked in this database as part of an associated binary
|
|
cache are ``in_buildcache``. All other specs are not. This field is used for
|
|
querying mirror indices. By default, it does not check this status.
|
|
|
|
hashes: list of hashes used to restrict the search
|
|
|
|
install_tree: query 'all' (default), 'local', 'upstream', or upstream path
|
|
|
|
origin: origin of the spec
|
|
"""
|
|
valid_trees = ["all", "upstream", "local", self.root] + [u.root for u in self.upstream_dbs]
|
|
if install_tree not in valid_trees:
|
|
msg = "Invalid install_tree argument to Database.query()\n"
|
|
msg += f"Try one of {', '.join(valid_trees)}"
|
|
tty.error(msg)
|
|
return []
|
|
|
|
upstream_results = []
|
|
upstreams = self.upstream_dbs
|
|
if install_tree not in ("all", "upstream"):
|
|
upstreams = [u for u in self.upstream_dbs if u.root == install_tree]
|
|
for upstream_db in upstreams:
|
|
# queries for upstream DBs need to *not* lock - we may not
|
|
# have permissions to do this and the upstream DBs won't know about
|
|
# us anyway (so e.g. they should never uninstall specs)
|
|
upstream_results.extend(
|
|
upstream_db._query(
|
|
query_spec,
|
|
predicate_fn=predicate_fn,
|
|
installed=installed,
|
|
explicit=explicit,
|
|
start_date=start_date,
|
|
end_date=end_date,
|
|
hashes=hashes,
|
|
in_buildcache=in_buildcache,
|
|
origin=origin,
|
|
)
|
|
or []
|
|
)
|
|
|
|
local_results: Set["spack.spec.Spec"] = set()
|
|
if install_tree in ("all", "local") or self.root == install_tree:
|
|
local_results = set(
|
|
self.query_local(
|
|
query_spec,
|
|
predicate_fn=predicate_fn,
|
|
installed=installed,
|
|
explicit=explicit,
|
|
start_date=start_date,
|
|
end_date=end_date,
|
|
hashes=hashes,
|
|
in_buildcache=in_buildcache,
|
|
origin=origin,
|
|
)
|
|
)
|
|
|
|
results = list(local_results) + list(x for x in upstream_results if x not in local_results)
|
|
results.sort()
|
|
return results
|
|
|
|
def query_one(
|
|
self,
|
|
query_spec: Optional[Union[str, "spack.spec.Spec"]],
|
|
predicate_fn: Optional[SelectType] = None,
|
|
installed: Union[bool, InstallRecordStatus] = True,
|
|
) -> Optional["spack.spec.Spec"]:
|
|
"""Query for exactly one spec that matches the query spec.
|
|
|
|
Returns None if no installed package matches.
|
|
|
|
Raises:
|
|
AssertionError: if more than one spec matches the query.
|
|
"""
|
|
concrete_specs = self.query(query_spec, predicate_fn=predicate_fn, installed=installed)
|
|
assert len(concrete_specs) <= 1
|
|
return concrete_specs[0] if concrete_specs else None
|
|
|
|
def missing(self, spec):
|
|
key = spec.dag_hash()
|
|
upstream, record = self.query_by_spec_hash(key)
|
|
return record and not record.installed
|
|
|
|
def is_occupied_install_prefix(self, path):
|
|
with self.read_transaction():
|
|
return path in self._installed_prefixes
|
|
|
|
def all_hashes(self):
|
|
"""Return dag hash of every spec in the database."""
|
|
with self.read_transaction():
|
|
return list(self._data.keys())
|
|
|
|
def unused_specs(
|
|
self,
|
|
root_hashes: Optional[Container[str]] = None,
|
|
deptype: Union[dt.DepFlag, dt.DepTypes] = dt.LINK | dt.RUN,
|
|
) -> List["spack.spec.Spec"]:
|
|
"""Return all specs that are currently installed but not needed by root specs.
|
|
|
|
By default, roots are all explicit specs in the database. If a set of root
|
|
hashes are passed in, they are instead used as the roots.
|
|
|
|
Arguments:
|
|
root_hashes: optional list of roots to consider when evaluating needed installations.
|
|
deptype: if a spec is reachable from a root via these dependency types, it is
|
|
considered needed. By default only link and run dependency types are considered.
|
|
"""
|
|
|
|
def root(key, record):
|
|
"""Whether a DB record is a root for garbage collection."""
|
|
return key in root_hashes if root_hashes is not None else record.explicit
|
|
|
|
with self.read_transaction():
|
|
roots = [rec.spec for key, rec in self._data.items() if root(key, rec)]
|
|
needed = set(id(spec) for spec in tr.traverse_nodes(roots, deptype=deptype))
|
|
return [
|
|
rec.spec
|
|
for rec in self._data.values()
|
|
if id(rec.spec) not in needed and rec.installed
|
|
]
|
|
|
|
|
|
class NoUpstreamVisitor:
|
|
"""Gives edges to upstream specs, but does follow edges from upstream specs."""
|
|
|
|
def __init__(
|
|
self,
|
|
upstream_hashes: Set[str],
|
|
on_visit: Callable[["spack.spec.DependencySpec", bool], None],
|
|
):
|
|
self.upstream_hashes = upstream_hashes
|
|
self.on_visit = on_visit
|
|
|
|
def accept(self, item: tr.EdgeAndDepth) -> bool:
|
|
self.on_visit(item.edge, self.is_upstream(item))
|
|
return True
|
|
|
|
def is_upstream(self, item: tr.EdgeAndDepth) -> bool:
|
|
return item.edge.spec.dag_hash() in self.upstream_hashes
|
|
|
|
def neighbors(self, item: tr.EdgeAndDepth):
|
|
# Prune edges from upstream nodes, only follow database tracked dependencies
|
|
return (
|
|
[]
|
|
if self.is_upstream(item)
|
|
else item.edge.spec.edges_to_dependencies(depflag=_TRACKED_DEPENDENCIES)
|
|
)
|
|
|
|
|
|
class UpstreamDatabaseLockingError(SpackError):
|
|
"""Raised when an operation would need to lock an upstream database"""
|
|
|
|
|
|
class CorruptDatabaseError(SpackError):
|
|
"""Raised when errors are found while reading the database."""
|
|
|
|
|
|
class NonConcreteSpecAddError(SpackError):
|
|
"""Raised when attempting to add non-concrete spec to DB."""
|
|
|
|
|
|
class MissingDependenciesError(SpackError):
|
|
"""Raised when DB cannot find records for dependencies"""
|
|
|
|
|
|
class InvalidDatabaseVersionError(SpackError):
|
|
"""Exception raised when the database metadata is newer than current Spack."""
|
|
|
|
def __init__(self, database, expected, found):
|
|
self.expected = expected
|
|
self.found = found
|
|
msg = (
|
|
f"you need a newer Spack version to read the DB in '{database.root}'. "
|
|
f"{self.database_version_message}"
|
|
)
|
|
super().__init__(msg)
|
|
|
|
@property
|
|
def database_version_message(self):
|
|
return f"The expected DB version is '{self.expected}', but '{self.found}' was found."
|
|
|
|
|
|
class NoSuchSpecError(KeyError):
|
|
"""Raised when a spec is not found in the database."""
|
|
|
|
def __init__(self, spec):
|
|
self.spec = spec
|
|
super().__init__(spec)
|
|
|
|
def __str__(self):
|
|
# This exception is raised frequently, and almost always
|
|
# caught, so ensure we don't pay the cost of Spec.__str__
|
|
# unless the exception is actually printed.
|
|
return f"No such spec in database: {self.spec}"
|