version: move to module, avoid circular imports (#39077)
This commit is contained in:
parent
03c0d74139
commit
e7fa6d99bf
@ -45,6 +45,7 @@
|
|||||||
import spack.util.url as url_util
|
import spack.util.url as url_util
|
||||||
import spack.util.web as web_util
|
import spack.util.web as web_util
|
||||||
import spack.version
|
import spack.version
|
||||||
|
import spack.version.git_ref_lookup
|
||||||
from spack.util.compression import decompressor_for, extension_from_path
|
from spack.util.compression import decompressor_for, extension_from_path
|
||||||
from spack.util.executable import CommandNotFoundError, which
|
from spack.util.executable import CommandNotFoundError, which
|
||||||
from spack.util.string import comma_and, quote
|
from spack.util.string import comma_and, quote
|
||||||
@ -1540,7 +1541,7 @@ def for_package_version(pkg, version=None):
|
|||||||
f"Cannot fetch git version for {pkg.name}. Package has no 'git' attribute"
|
f"Cannot fetch git version for {pkg.name}. Package has no 'git' attribute"
|
||||||
)
|
)
|
||||||
# Populate the version with comparisons to other commits
|
# Populate the version with comparisons to other commits
|
||||||
version.attach_git_lookup_from_package(pkg.name)
|
version.attach_lookup(spack.version.git_ref_lookup.GitRefLookup(pkg.name))
|
||||||
|
|
||||||
# For GitVersion, we have no way to determine whether a ref is a branch or tag
|
# For GitVersion, we have no way to determine whether a ref is a branch or tag
|
||||||
# Fortunately, we handle branches and tags identically, except tags are
|
# Fortunately, we handle branches and tags identically, except tags are
|
||||||
|
@ -48,6 +48,7 @@
|
|||||||
import spack.util.timer
|
import spack.util.timer
|
||||||
import spack.variant
|
import spack.variant
|
||||||
import spack.version as vn
|
import spack.version as vn
|
||||||
|
import spack.version.git_ref_lookup
|
||||||
|
|
||||||
# these are from clingo.ast and bootstrapped later
|
# these are from clingo.ast and bootstrapped later
|
||||||
ASTType = None
|
ASTType = None
|
||||||
@ -2672,7 +2673,9 @@ def build_specs(self, function_tuples):
|
|||||||
for root in self._specs.values():
|
for root in self._specs.values():
|
||||||
for spec in root.traverse():
|
for spec in root.traverse():
|
||||||
if isinstance(spec.version, vn.GitVersion):
|
if isinstance(spec.version, vn.GitVersion):
|
||||||
spec.version.attach_git_lookup_from_package(spec.fullname)
|
spec.version.attach_lookup(
|
||||||
|
spack.version.git_ref_lookup.GitRefLookup(spec.fullname)
|
||||||
|
)
|
||||||
|
|
||||||
return self._specs
|
return self._specs
|
||||||
|
|
||||||
|
@ -88,6 +88,7 @@
|
|||||||
import spack.util.string
|
import spack.util.string
|
||||||
import spack.variant as vt
|
import spack.variant as vt
|
||||||
import spack.version as vn
|
import spack.version as vn
|
||||||
|
import spack.version.git_ref_lookup
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"CompilerSpec",
|
"CompilerSpec",
|
||||||
@ -4801,7 +4802,7 @@ def attach_git_version_lookup(self):
|
|||||||
return
|
return
|
||||||
for v in self.versions:
|
for v in self.versions:
|
||||||
if isinstance(v, vn.GitVersion) and v._ref_version is None:
|
if isinstance(v, vn.GitVersion) and v._ref_version is None:
|
||||||
v.attach_git_lookup_from_package(self.fullname)
|
v.attach_lookup(spack.version.git_ref_lookup.GitRefLookup(self.fullname))
|
||||||
|
|
||||||
|
|
||||||
def parse_with_version_concrete(string: str, compiler: bool = False):
|
def parse_with_version_concrete(string: str, compiler: bool = False):
|
||||||
|
58
lib/spack/spack/version/__init__.py
Normal file
58
lib/spack/spack/version/__init__.py
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
|
||||||
|
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||||
|
|
||||||
|
"""
|
||||||
|
This module implements Version and version-ish objects. These are:
|
||||||
|
|
||||||
|
StandardVersion: A single version of a package.
|
||||||
|
ClosedOpenRange: A range of versions of a package.
|
||||||
|
VersionList: A ordered list of Version and VersionRange elements.
|
||||||
|
|
||||||
|
The set of Version and ClosedOpenRange is totally ordered wiht <
|
||||||
|
defined as Version(x) < VersionRange(Version(y), Version(x))
|
||||||
|
if Version(x) <= Version(y).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .common import (
|
||||||
|
VersionChecksumError,
|
||||||
|
VersionError,
|
||||||
|
VersionLookupError,
|
||||||
|
infinity_versions,
|
||||||
|
is_git_version,
|
||||||
|
)
|
||||||
|
from .version_types import (
|
||||||
|
ClosedOpenRange,
|
||||||
|
GitVersion,
|
||||||
|
StandardVersion,
|
||||||
|
Version,
|
||||||
|
VersionList,
|
||||||
|
VersionRange,
|
||||||
|
from_string,
|
||||||
|
next_version,
|
||||||
|
prev_version,
|
||||||
|
ver,
|
||||||
|
)
|
||||||
|
|
||||||
|
#: This version contains all possible versions.
|
||||||
|
any_version: VersionList = VersionList([":"])
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"Version",
|
||||||
|
"VersionRange",
|
||||||
|
"ver",
|
||||||
|
"from_string",
|
||||||
|
"is_git_version",
|
||||||
|
"infinity_versions",
|
||||||
|
"prev_version",
|
||||||
|
"next_version",
|
||||||
|
"VersionList",
|
||||||
|
"ClosedOpenRange",
|
||||||
|
"StandardVersion",
|
||||||
|
"GitVersion",
|
||||||
|
"VersionError",
|
||||||
|
"VersionChecksumError",
|
||||||
|
"VersionLookupError",
|
||||||
|
"any_version",
|
||||||
|
]
|
37
lib/spack/spack/version/common.py
Normal file
37
lib/spack/spack/version/common.py
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
|
||||||
|
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
import spack.error
|
||||||
|
|
||||||
|
# regex for a commit version
|
||||||
|
COMMIT_VERSION = re.compile(r"^[a-f0-9]{40}$")
|
||||||
|
|
||||||
|
# Infinity-like versions. The order in the list implies the comparison rules
|
||||||
|
infinity_versions = ["stable", "trunk", "head", "master", "main", "develop"]
|
||||||
|
|
||||||
|
iv_min_len = min(len(s) for s in infinity_versions)
|
||||||
|
|
||||||
|
|
||||||
|
def is_git_version(string: str) -> bool:
|
||||||
|
return (
|
||||||
|
string.startswith("git.")
|
||||||
|
or len(string) == 40
|
||||||
|
and bool(COMMIT_VERSION.match(string))
|
||||||
|
or "=" in string[1:]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class VersionError(spack.error.SpackError):
|
||||||
|
"""This is raised when something is wrong with a version."""
|
||||||
|
|
||||||
|
|
||||||
|
class VersionChecksumError(VersionError):
|
||||||
|
"""Raised for version checksum errors."""
|
||||||
|
|
||||||
|
|
||||||
|
class VersionLookupError(VersionError):
|
||||||
|
"""Raised for errors looking up git commits as versions."""
|
222
lib/spack/spack/version/git_ref_lookup.py
Normal file
222
lib/spack/spack/version/git_ref_lookup.py
Normal file
@ -0,0 +1,222 @@
|
|||||||
|
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
|
||||||
|
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from typing import Dict, Optional, Tuple
|
||||||
|
|
||||||
|
from llnl.util.filesystem import mkdirp, working_dir
|
||||||
|
|
||||||
|
import spack.caches
|
||||||
|
import spack.fetch_strategy
|
||||||
|
import spack.paths
|
||||||
|
import spack.repo
|
||||||
|
import spack.util.executable
|
||||||
|
import spack.util.spack_json as sjson
|
||||||
|
import spack.util.url
|
||||||
|
import spack.version
|
||||||
|
|
||||||
|
from .common import VersionLookupError
|
||||||
|
from .lookup import AbstractRefLookup
|
||||||
|
|
||||||
|
# regular expression for semantic versioning
|
||||||
|
SEMVER_REGEX = re.compile(
|
||||||
|
".+(?P<semver>([0-9]+)[.]([0-9]+)[.]([0-9]+)"
|
||||||
|
"(?:-([0-9A-Za-z-]+(?:[.][0-9A-Za-z-]+)*))?"
|
||||||
|
"(?:[+][0-9A-Za-z-]+)?)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class GitRefLookup(AbstractRefLookup):
|
||||||
|
"""An object for cached lookups of git refs
|
||||||
|
|
||||||
|
GitRefLookup objects delegate to the misc_cache for locking. GitRefLookup objects may
|
||||||
|
be attached to a GitVersion to allow for comparisons between git refs and versions as
|
||||||
|
represented by tags in the git repository.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, pkg_name):
|
||||||
|
self.pkg_name = pkg_name
|
||||||
|
|
||||||
|
self.data: Dict[str, Tuple[Optional[str], int]] = {}
|
||||||
|
|
||||||
|
self._pkg = None
|
||||||
|
self._fetcher = None
|
||||||
|
self._cache_key = None
|
||||||
|
self._cache_path = None
|
||||||
|
|
||||||
|
# The following properties are used as part of a lazy reference scheme
|
||||||
|
# to avoid querying the package repository until it is necessary (and
|
||||||
|
# in particular to wait until after the configuration has been
|
||||||
|
# assembled)
|
||||||
|
@property
|
||||||
|
def cache_key(self):
|
||||||
|
if not self._cache_key:
|
||||||
|
key_base = "git_metadata"
|
||||||
|
if not self.repository_uri.startswith("/"):
|
||||||
|
key_base += "/"
|
||||||
|
self._cache_key = key_base + self.repository_uri
|
||||||
|
|
||||||
|
# Cache data in misc_cache
|
||||||
|
# If this is the first lazy access, initialize the cache as well
|
||||||
|
spack.caches.misc_cache.init_entry(self.cache_key)
|
||||||
|
return self._cache_key
|
||||||
|
|
||||||
|
@property
|
||||||
|
def cache_path(self):
|
||||||
|
if not self._cache_path:
|
||||||
|
self._cache_path = spack.caches.misc_cache.cache_path(self.cache_key)
|
||||||
|
return self._cache_path
|
||||||
|
|
||||||
|
@property
|
||||||
|
def pkg(self):
|
||||||
|
if not self._pkg:
|
||||||
|
try:
|
||||||
|
pkg = spack.repo.path.get_pkg_class(self.pkg_name)
|
||||||
|
pkg.git
|
||||||
|
except (spack.repo.RepoError, AttributeError) as e:
|
||||||
|
raise VersionLookupError(f"Couldn't get the git repo for {self.pkg_name}") from e
|
||||||
|
self._pkg = pkg
|
||||||
|
return self._pkg
|
||||||
|
|
||||||
|
@property
|
||||||
|
def fetcher(self):
|
||||||
|
if not self._fetcher:
|
||||||
|
# We require the full git repository history
|
||||||
|
fetcher = spack.fetch_strategy.GitFetchStrategy(git=self.pkg.git)
|
||||||
|
fetcher.get_full_repo = True
|
||||||
|
self._fetcher = fetcher
|
||||||
|
return self._fetcher
|
||||||
|
|
||||||
|
@property
|
||||||
|
def repository_uri(self):
|
||||||
|
"""Identifier for git repos used within the repo and metadata caches."""
|
||||||
|
try:
|
||||||
|
components = [
|
||||||
|
str(c).lstrip("/") for c in spack.util.url.parse_git_url(self.pkg.git) if c
|
||||||
|
]
|
||||||
|
return os.path.join(*components)
|
||||||
|
except ValueError:
|
||||||
|
# If it's not a git url, it's a local path
|
||||||
|
return os.path.abspath(self.pkg.git)
|
||||||
|
|
||||||
|
def save(self):
|
||||||
|
"""Save the data to file"""
|
||||||
|
with spack.caches.misc_cache.write_transaction(self.cache_key) as (old, new):
|
||||||
|
sjson.dump(self.data, new)
|
||||||
|
|
||||||
|
def load_data(self):
|
||||||
|
"""Load data if the path already exists."""
|
||||||
|
if os.path.isfile(self.cache_path):
|
||||||
|
with spack.caches.misc_cache.read_transaction(self.cache_key) as cache_file:
|
||||||
|
self.data = sjson.load(cache_file)
|
||||||
|
|
||||||
|
def get(self, ref) -> Tuple[Optional[str], int]:
|
||||||
|
if not self.data:
|
||||||
|
self.load_data()
|
||||||
|
|
||||||
|
if ref not in self.data:
|
||||||
|
self.data[ref] = self.lookup_ref(ref)
|
||||||
|
self.save()
|
||||||
|
|
||||||
|
return self.data[ref]
|
||||||
|
|
||||||
|
def lookup_ref(self, ref) -> Tuple[Optional[str], int]:
|
||||||
|
"""Lookup the previous version and distance for a given commit.
|
||||||
|
|
||||||
|
We use git to compare the known versions from package to the git tags,
|
||||||
|
as well as any git tags that are SEMVER versions, and find the latest
|
||||||
|
known version prior to the commit, as well as the distance from that version
|
||||||
|
to the commit in the git repo. Those values are used to compare Version objects.
|
||||||
|
"""
|
||||||
|
dest = os.path.join(spack.paths.user_repos_cache_path, self.repository_uri)
|
||||||
|
if dest.endswith(".git"):
|
||||||
|
dest = dest[:-4]
|
||||||
|
|
||||||
|
# prepare a cache for the repository
|
||||||
|
dest_parent = os.path.dirname(dest)
|
||||||
|
if not os.path.exists(dest_parent):
|
||||||
|
mkdirp(dest_parent)
|
||||||
|
|
||||||
|
# Only clone if we don't have it!
|
||||||
|
if not os.path.exists(dest):
|
||||||
|
self.fetcher.clone(dest, bare=True)
|
||||||
|
|
||||||
|
# Lookup commit info
|
||||||
|
with working_dir(dest):
|
||||||
|
# TODO: we need to update the local tags if they changed on the
|
||||||
|
# remote instance, simply adding '-f' may not be sufficient
|
||||||
|
# (if commits are deleted on the remote, this command alone
|
||||||
|
# won't properly update the local rev-list)
|
||||||
|
self.fetcher.git("fetch", "--tags", output=os.devnull, error=os.devnull)
|
||||||
|
|
||||||
|
# Ensure ref is a commit object known to git
|
||||||
|
# Note the brackets are literals, the ref replaces the format string
|
||||||
|
try:
|
||||||
|
self.fetcher.git(
|
||||||
|
"cat-file", "-e", "%s^{commit}" % ref, output=os.devnull, error=os.devnull
|
||||||
|
)
|
||||||
|
except spack.util.executable.ProcessError:
|
||||||
|
raise VersionLookupError("%s is not a valid git ref for %s" % (ref, self.pkg_name))
|
||||||
|
|
||||||
|
# List tags (refs) by date, so last reference of a tag is newest
|
||||||
|
tag_info = self.fetcher.git(
|
||||||
|
"for-each-ref",
|
||||||
|
"--sort=creatordate",
|
||||||
|
"--format",
|
||||||
|
"%(objectname) %(refname)",
|
||||||
|
"refs/tags",
|
||||||
|
output=str,
|
||||||
|
).split("\n")
|
||||||
|
|
||||||
|
# Lookup of commits to spack versions
|
||||||
|
commit_to_version = {}
|
||||||
|
|
||||||
|
for entry in tag_info:
|
||||||
|
if not entry:
|
||||||
|
continue
|
||||||
|
tag_commit, tag = entry.split()
|
||||||
|
tag = tag.replace("refs/tags/", "", 1)
|
||||||
|
|
||||||
|
# For each tag, try to match to a version
|
||||||
|
for v in [v.string for v in self.pkg.versions]:
|
||||||
|
if v == tag or "v" + v == tag:
|
||||||
|
commit_to_version[tag_commit] = v
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
# try to parse tag to copare versions spack does not know
|
||||||
|
match = SEMVER_REGEX.match(tag)
|
||||||
|
if match:
|
||||||
|
semver = match.groupdict()["semver"]
|
||||||
|
commit_to_version[tag_commit] = semver
|
||||||
|
|
||||||
|
ancestor_commits = []
|
||||||
|
for tag_commit in commit_to_version:
|
||||||
|
self.fetcher.git("merge-base", "--is-ancestor", tag_commit, ref, ignore_errors=[1])
|
||||||
|
if self.fetcher.git.returncode == 0:
|
||||||
|
distance = self.fetcher.git(
|
||||||
|
"rev-list", "%s..%s" % (tag_commit, ref), "--count", output=str, error=str
|
||||||
|
).strip()
|
||||||
|
ancestor_commits.append((tag_commit, int(distance)))
|
||||||
|
|
||||||
|
if ancestor_commits:
|
||||||
|
# Get nearest ancestor that is a known version
|
||||||
|
prev_version_commit, distance = min(ancestor_commits, key=lambda x: x[1])
|
||||||
|
prev_version = commit_to_version[prev_version_commit]
|
||||||
|
else:
|
||||||
|
# Get list of all commits, this is in reverse order
|
||||||
|
# We use this to get the first commit below
|
||||||
|
ref_info = self.fetcher.git("log", "--all", "--pretty=format:%H", output=str)
|
||||||
|
commits = [c for c in ref_info.split("\n") if c]
|
||||||
|
|
||||||
|
# No previous version and distance from first commit
|
||||||
|
prev_version = None
|
||||||
|
distance = int(
|
||||||
|
self.fetcher.git(
|
||||||
|
"rev-list", "%s..%s" % (commits[-1], ref), "--count", output=str, error=str
|
||||||
|
).strip()
|
||||||
|
)
|
||||||
|
|
||||||
|
return prev_version, distance
|
17
lib/spack/spack/version/lookup.py
Normal file
17
lib/spack/spack/version/lookup.py
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
|
||||||
|
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||||
|
|
||||||
|
from typing import Optional, Tuple
|
||||||
|
|
||||||
|
|
||||||
|
class AbstractRefLookup:
|
||||||
|
def get(self, ref) -> Tuple[Optional[str], int]:
|
||||||
|
"""Get the version string and distance for a given git ref.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
ref (str): git ref to lookup
|
||||||
|
|
||||||
|
Returns: optional version string and distance"""
|
||||||
|
return None, 0
|
@ -3,54 +3,28 @@
|
|||||||
#
|
#
|
||||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||||
|
|
||||||
"""
|
|
||||||
This module implements Version and version-ish objects. These are:
|
|
||||||
|
|
||||||
StandardVersion: A single version of a package.
|
|
||||||
ClosedOpenRange: A range of versions of a package.
|
|
||||||
VersionList: A ordered list of Version and VersionRange elements.
|
|
||||||
|
|
||||||
The set of Version and ClosedOpenRange is totally ordered wiht <
|
|
||||||
defined as Version(x) < VersionRange(Version(y), Version(x))
|
|
||||||
if Version(x) <= Version(y).
|
|
||||||
"""
|
|
||||||
import numbers
|
import numbers
|
||||||
import os
|
|
||||||
import re
|
import re
|
||||||
from bisect import bisect_left
|
from bisect import bisect_left
|
||||||
from typing import Dict, List, Optional, Tuple, Union
|
from typing import List, Optional, Tuple, Union
|
||||||
|
|
||||||
from llnl.util.filesystem import mkdirp, working_dir
|
|
||||||
|
|
||||||
import spack.caches
|
|
||||||
import spack.error
|
|
||||||
import spack.paths
|
|
||||||
import spack.util.executable
|
|
||||||
import spack.util.spack_json as sjson
|
|
||||||
import spack.util.url
|
|
||||||
from spack.util.spack_yaml import syaml_dict
|
from spack.util.spack_yaml import syaml_dict
|
||||||
|
|
||||||
|
from .common import (
|
||||||
|
COMMIT_VERSION,
|
||||||
|
VersionLookupError,
|
||||||
|
infinity_versions,
|
||||||
|
is_git_version,
|
||||||
|
iv_min_len,
|
||||||
|
)
|
||||||
|
from .lookup import AbstractRefLookup
|
||||||
|
|
||||||
# Valid version characters
|
# Valid version characters
|
||||||
VALID_VERSION = re.compile(r"^[A-Za-z0-9_.-][=A-Za-z0-9_.-]*$")
|
VALID_VERSION = re.compile(r"^[A-Za-z0-9_.-][=A-Za-z0-9_.-]*$")
|
||||||
|
|
||||||
# regex for a commit version
|
|
||||||
COMMIT_VERSION = re.compile(r"^[a-f0-9]{40}$")
|
|
||||||
|
|
||||||
# regex for version segments
|
# regex for version segments
|
||||||
SEGMENT_REGEX = re.compile(r"(?:(?P<num>[0-9]+)|(?P<str>[a-zA-Z]+))(?P<sep>[_.-]*)")
|
SEGMENT_REGEX = re.compile(r"(?:(?P<num>[0-9]+)|(?P<str>[a-zA-Z]+))(?P<sep>[_.-]*)")
|
||||||
|
|
||||||
# regular expression for semantic versioning
|
|
||||||
SEMVER_REGEX = re.compile(
|
|
||||||
".+(?P<semver>([0-9]+)[.]([0-9]+)[.]([0-9]+)"
|
|
||||||
"(?:-([0-9A-Za-z-]+(?:[.][0-9A-Za-z-]+)*))?"
|
|
||||||
"(?:[+][0-9A-Za-z-]+)?)"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Infinity-like versions. The order in the list implies the comparison rules
|
|
||||||
infinity_versions = ["stable", "trunk", "head", "master", "main", "develop"]
|
|
||||||
|
|
||||||
iv_min_len = min(len(s) for s in infinity_versions)
|
|
||||||
|
|
||||||
|
|
||||||
class VersionStrComponent:
|
class VersionStrComponent:
|
||||||
__slots__ = ["data"]
|
__slots__ = ["data"]
|
||||||
@ -407,7 +381,7 @@ class GitVersion(ConcreteVersion):
|
|||||||
|
|
||||||
def __init__(self, string: str):
|
def __init__(self, string: str):
|
||||||
# An object that can lookup git refs to compare them to versions
|
# An object that can lookup git refs to compare them to versions
|
||||||
self._ref_lookup: Optional[CommitLookup] = None
|
self._ref_lookup: Optional[AbstractRefLookup] = None
|
||||||
|
|
||||||
# This is the effective version.
|
# This is the effective version.
|
||||||
self._ref_version: Optional[StandardVersion]
|
self._ref_version: Optional[StandardVersion]
|
||||||
@ -440,8 +414,7 @@ def ref_version(self) -> StandardVersion:
|
|||||||
|
|
||||||
if self.ref_lookup is None:
|
if self.ref_lookup is None:
|
||||||
raise VersionLookupError(
|
raise VersionLookupError(
|
||||||
f"git ref '{self.ref}' cannot be looked up: "
|
f"git ref '{self.ref}' cannot be looked up: " "call attach_lookup first"
|
||||||
"call attach_git_lookup_from_package first"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
version_string, distance = self.ref_lookup.get(self.ref)
|
version_string, distance = self.ref_lookup.get(self.ref)
|
||||||
@ -573,7 +546,7 @@ def ref_lookup(self):
|
|||||||
self._ref_lookup.get(self.ref)
|
self._ref_lookup.get(self.ref)
|
||||||
return self._ref_lookup
|
return self._ref_lookup
|
||||||
|
|
||||||
def attach_git_lookup_from_package(self, pkg_name):
|
def attach_lookup(self, lookup: AbstractRefLookup):
|
||||||
"""
|
"""
|
||||||
Use the git fetcher to look up a version for a commit.
|
Use the git fetcher to look up a version for a commit.
|
||||||
|
|
||||||
@ -585,7 +558,7 @@ def attach_git_lookup_from_package(self, pkg_name):
|
|||||||
alongside the GitFetcher because eventually the git repos cache will
|
alongside the GitFetcher because eventually the git repos cache will
|
||||||
be one and the same with the source cache.
|
be one and the same with the source cache.
|
||||||
"""
|
"""
|
||||||
self._ref_lookup = self._ref_lookup or CommitLookup(pkg_name)
|
self._ref_lookup = lookup
|
||||||
|
|
||||||
def __iter__(self):
|
def __iter__(self):
|
||||||
return self.ref_version.__iter__()
|
return self.ref_version.__iter__()
|
||||||
@ -758,7 +731,7 @@ def __init__(self, vlist=None):
|
|||||||
if vlist is not None:
|
if vlist is not None:
|
||||||
if isinstance(vlist, str):
|
if isinstance(vlist, str):
|
||||||
vlist = from_string(vlist)
|
vlist = from_string(vlist)
|
||||||
if type(vlist) == VersionList:
|
if isinstance(vlist, VersionList):
|
||||||
self.versions = vlist.versions
|
self.versions = vlist.versions
|
||||||
else:
|
else:
|
||||||
self.versions = [vlist]
|
self.versions = [vlist]
|
||||||
@ -792,7 +765,7 @@ def add(self, item):
|
|||||||
|
|
||||||
self.versions.insert(i, item)
|
self.versions.insert(i, item)
|
||||||
|
|
||||||
elif type(item) == VersionList:
|
elif isinstance(item, VersionList):
|
||||||
for v in item:
|
for v in item:
|
||||||
self.add(v)
|
self.add(v)
|
||||||
|
|
||||||
@ -1073,15 +1046,6 @@ def prev_version(v: StandardVersion) -> StandardVersion:
|
|||||||
return StandardVersion("".join(string_components), v.version[:-1] + (prev,), v.separators)
|
return StandardVersion("".join(string_components), v.version[:-1] + (prev,), v.separators)
|
||||||
|
|
||||||
|
|
||||||
def is_git_version(string: str) -> bool:
|
|
||||||
return (
|
|
||||||
string.startswith("git.")
|
|
||||||
or len(string) == 40
|
|
||||||
and bool(COMMIT_VERSION.match(string))
|
|
||||||
or "=" in string[1:]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def Version(string: Union[str, int]) -> Union[GitVersion, StandardVersion]:
|
def Version(string: Union[str, int]) -> Union[GitVersion, StandardVersion]:
|
||||||
if not isinstance(string, (str, int)):
|
if not isinstance(string, (str, int)):
|
||||||
raise ValueError(f"Cannot construct a version from {type(string)}")
|
raise ValueError(f"Cannot construct a version from {type(string)}")
|
||||||
@ -1140,216 +1104,3 @@ def ver(obj) -> Union[VersionList, ClosedOpenRange, StandardVersion, GitVersion]
|
|||||||
return obj
|
return obj
|
||||||
else:
|
else:
|
||||||
raise TypeError("ver() can't convert %s to version!" % type(obj))
|
raise TypeError("ver() can't convert %s to version!" % type(obj))
|
||||||
|
|
||||||
|
|
||||||
#: This version contains all possible versions.
|
|
||||||
any_version: VersionList = VersionList([":"])
|
|
||||||
|
|
||||||
|
|
||||||
class VersionError(spack.error.SpackError):
|
|
||||||
"""This is raised when something is wrong with a version."""
|
|
||||||
|
|
||||||
|
|
||||||
class VersionChecksumError(VersionError):
|
|
||||||
"""Raised for version checksum errors."""
|
|
||||||
|
|
||||||
|
|
||||||
class VersionLookupError(VersionError):
|
|
||||||
"""Raised for errors looking up git commits as versions."""
|
|
||||||
|
|
||||||
|
|
||||||
class CommitLookup:
|
|
||||||
"""An object for cached lookups of git commits
|
|
||||||
|
|
||||||
CommitLookup objects delegate to the misc_cache for locking. CommitLookup objects may
|
|
||||||
be attached to a GitVersion to allow for comparisons between git refs and versions as
|
|
||||||
represented by tags in the git repository.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, pkg_name):
|
|
||||||
self.pkg_name = pkg_name
|
|
||||||
|
|
||||||
self.data: Dict[str, Tuple[Optional[str], int]] = {}
|
|
||||||
|
|
||||||
self._pkg = None
|
|
||||||
self._fetcher = None
|
|
||||||
self._cache_key = None
|
|
||||||
self._cache_path = None
|
|
||||||
|
|
||||||
# The following properties are used as part of a lazy reference scheme
|
|
||||||
# to avoid querying the package repository until it is necessary (and
|
|
||||||
# in particular to wait until after the configuration has been
|
|
||||||
# assembled)
|
|
||||||
@property
|
|
||||||
def cache_key(self):
|
|
||||||
if not self._cache_key:
|
|
||||||
key_base = "git_metadata"
|
|
||||||
if not self.repository_uri.startswith("/"):
|
|
||||||
key_base += "/"
|
|
||||||
self._cache_key = key_base + self.repository_uri
|
|
||||||
|
|
||||||
# Cache data in misc_cache
|
|
||||||
# If this is the first lazy access, initialize the cache as well
|
|
||||||
spack.caches.misc_cache.init_entry(self.cache_key)
|
|
||||||
return self._cache_key
|
|
||||||
|
|
||||||
@property
|
|
||||||
def cache_path(self):
|
|
||||||
if not self._cache_path:
|
|
||||||
self._cache_path = spack.caches.misc_cache.cache_path(self.cache_key)
|
|
||||||
return self._cache_path
|
|
||||||
|
|
||||||
@property
|
|
||||||
def pkg(self):
|
|
||||||
if not self._pkg:
|
|
||||||
import spack.repo # break cycle
|
|
||||||
|
|
||||||
try:
|
|
||||||
pkg = spack.repo.path.get_pkg_class(self.pkg_name)
|
|
||||||
pkg.git
|
|
||||||
except (spack.repo.RepoError, AttributeError) as e:
|
|
||||||
raise VersionLookupError(f"Couldn't get the git repo for {self.pkg_name}") from e
|
|
||||||
self._pkg = pkg
|
|
||||||
return self._pkg
|
|
||||||
|
|
||||||
@property
|
|
||||||
def fetcher(self):
|
|
||||||
if not self._fetcher:
|
|
||||||
# We require the full git repository history
|
|
||||||
import spack.fetch_strategy # break cycle
|
|
||||||
|
|
||||||
fetcher = spack.fetch_strategy.GitFetchStrategy(git=self.pkg.git)
|
|
||||||
fetcher.get_full_repo = True
|
|
||||||
self._fetcher = fetcher
|
|
||||||
return self._fetcher
|
|
||||||
|
|
||||||
@property
|
|
||||||
def repository_uri(self):
|
|
||||||
"""Identifier for git repos used within the repo and metadata caches."""
|
|
||||||
try:
|
|
||||||
components = [
|
|
||||||
str(c).lstrip("/") for c in spack.util.url.parse_git_url(self.pkg.git) if c
|
|
||||||
]
|
|
||||||
return os.path.join(*components)
|
|
||||||
except ValueError:
|
|
||||||
# If it's not a git url, it's a local path
|
|
||||||
return os.path.abspath(self.pkg.git)
|
|
||||||
|
|
||||||
def save(self):
|
|
||||||
"""Save the data to file"""
|
|
||||||
with spack.caches.misc_cache.write_transaction(self.cache_key) as (old, new):
|
|
||||||
sjson.dump(self.data, new)
|
|
||||||
|
|
||||||
def load_data(self):
|
|
||||||
"""Load data if the path already exists."""
|
|
||||||
if os.path.isfile(self.cache_path):
|
|
||||||
with spack.caches.misc_cache.read_transaction(self.cache_key) as cache_file:
|
|
||||||
self.data = sjson.load(cache_file)
|
|
||||||
|
|
||||||
def get(self, ref) -> Tuple[Optional[str], int]:
|
|
||||||
if not self.data:
|
|
||||||
self.load_data()
|
|
||||||
|
|
||||||
if ref not in self.data:
|
|
||||||
self.data[ref] = self.lookup_ref(ref)
|
|
||||||
self.save()
|
|
||||||
|
|
||||||
return self.data[ref]
|
|
||||||
|
|
||||||
def lookup_ref(self, ref) -> Tuple[Optional[str], int]:
|
|
||||||
"""Lookup the previous version and distance for a given commit.
|
|
||||||
|
|
||||||
We use git to compare the known versions from package to the git tags,
|
|
||||||
as well as any git tags that are SEMVER versions, and find the latest
|
|
||||||
known version prior to the commit, as well as the distance from that version
|
|
||||||
to the commit in the git repo. Those values are used to compare Version objects.
|
|
||||||
"""
|
|
||||||
dest = os.path.join(spack.paths.user_repos_cache_path, self.repository_uri)
|
|
||||||
if dest.endswith(".git"):
|
|
||||||
dest = dest[:-4]
|
|
||||||
|
|
||||||
# prepare a cache for the repository
|
|
||||||
dest_parent = os.path.dirname(dest)
|
|
||||||
if not os.path.exists(dest_parent):
|
|
||||||
mkdirp(dest_parent)
|
|
||||||
|
|
||||||
# Only clone if we don't have it!
|
|
||||||
if not os.path.exists(dest):
|
|
||||||
self.fetcher.clone(dest, bare=True)
|
|
||||||
|
|
||||||
# Lookup commit info
|
|
||||||
with working_dir(dest):
|
|
||||||
# TODO: we need to update the local tags if they changed on the
|
|
||||||
# remote instance, simply adding '-f' may not be sufficient
|
|
||||||
# (if commits are deleted on the remote, this command alone
|
|
||||||
# won't properly update the local rev-list)
|
|
||||||
self.fetcher.git("fetch", "--tags", output=os.devnull, error=os.devnull)
|
|
||||||
|
|
||||||
# Ensure ref is a commit object known to git
|
|
||||||
# Note the brackets are literals, the ref replaces the format string
|
|
||||||
try:
|
|
||||||
self.fetcher.git(
|
|
||||||
"cat-file", "-e", "%s^{commit}" % ref, output=os.devnull, error=os.devnull
|
|
||||||
)
|
|
||||||
except spack.util.executable.ProcessError:
|
|
||||||
raise VersionLookupError("%s is not a valid git ref for %s" % (ref, self.pkg_name))
|
|
||||||
|
|
||||||
# List tags (refs) by date, so last reference of a tag is newest
|
|
||||||
tag_info = self.fetcher.git(
|
|
||||||
"for-each-ref",
|
|
||||||
"--sort=creatordate",
|
|
||||||
"--format",
|
|
||||||
"%(objectname) %(refname)",
|
|
||||||
"refs/tags",
|
|
||||||
output=str,
|
|
||||||
).split("\n")
|
|
||||||
|
|
||||||
# Lookup of commits to spack versions
|
|
||||||
commit_to_version = {}
|
|
||||||
|
|
||||||
for entry in tag_info:
|
|
||||||
if not entry:
|
|
||||||
continue
|
|
||||||
tag_commit, tag = entry.split()
|
|
||||||
tag = tag.replace("refs/tags/", "", 1)
|
|
||||||
|
|
||||||
# For each tag, try to match to a version
|
|
||||||
for v in [v.string for v in self.pkg.versions]:
|
|
||||||
if v == tag or "v" + v == tag:
|
|
||||||
commit_to_version[tag_commit] = v
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
# try to parse tag to copare versions spack does not know
|
|
||||||
match = SEMVER_REGEX.match(tag)
|
|
||||||
if match:
|
|
||||||
semver = match.groupdict()["semver"]
|
|
||||||
commit_to_version[tag_commit] = semver
|
|
||||||
|
|
||||||
ancestor_commits = []
|
|
||||||
for tag_commit in commit_to_version:
|
|
||||||
self.fetcher.git("merge-base", "--is-ancestor", tag_commit, ref, ignore_errors=[1])
|
|
||||||
if self.fetcher.git.returncode == 0:
|
|
||||||
distance = self.fetcher.git(
|
|
||||||
"rev-list", "%s..%s" % (tag_commit, ref), "--count", output=str, error=str
|
|
||||||
).strip()
|
|
||||||
ancestor_commits.append((tag_commit, int(distance)))
|
|
||||||
|
|
||||||
if ancestor_commits:
|
|
||||||
# Get nearest ancestor that is a known version
|
|
||||||
prev_version_commit, distance = min(ancestor_commits, key=lambda x: x[1])
|
|
||||||
prev_version = commit_to_version[prev_version_commit]
|
|
||||||
else:
|
|
||||||
# Get list of all commits, this is in reverse order
|
|
||||||
# We use this to get the first commit below
|
|
||||||
ref_info = self.fetcher.git("log", "--all", "--pretty=format:%H", output=str)
|
|
||||||
commits = [c for c in ref_info.split("\n") if c]
|
|
||||||
|
|
||||||
# No previous version and distance from first commit
|
|
||||||
prev_version = None
|
|
||||||
distance = int(
|
|
||||||
self.fetcher.git(
|
|
||||||
"rev-list", "%s..%s" % (commits[-1], ref), "--count", output=str, error=str
|
|
||||||
).strip()
|
|
||||||
)
|
|
||||||
|
|
||||||
return prev_version, distance
|
|
Loading…
Reference in New Issue
Block a user