zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable and constrain reuse for GitVersion installations (#43859)

Browse Source 
* Preserve higher weight for CLI git ref versions

Currently the concretizer fails if you reuse a git ref version
that has already been installed but modify the spec at all.

See #38484 for futher diagnosis

The issue here is that since there is no established provenance for
these versions the highest weight they are currently assigned is
that of prior install. Re-use checks then fail because the weight of
the version is identical to the solver.

Ironically, these versions are given the highest weights possible when
specified on the CLI for the first time.  They should only appear in a
DAG if they are an exact match or if the user specifies them at the CLI.
Therefore it makes sense to preserve their higher ordering.

Getting this right is critical to moving all branch based versions to a pinned
git-ref in the future.

* [@spackbot] updating style on behalf of psakievich

* Update lib/spack/spack/solver/asp.py

Co-authored-by: Greg Becker <becker33@llnl.gov>

* Add provenance specific to git ref installs

* Sensitvity to name that I could not track down

* Add regression test

* Adjust test

* Add prefer standard unit-test

* Style

* Add required mock

* Format and mark

* Make unit-test case reproduce CLI investigation

* Remove unnecessary mock package

* [@spackbot] updating style on behalf of psakievich

* Use already developed fixture

* Add zlib-ng to mocks again

* Remove accidental adds

* Remove maintainer

* [@spackbot] updating style on behalf of psakievich

* Rename test file

* [@spackbot] updating style on behalf of psakievich

* Remove unused imports

* Update tests

* [@spackbot] updating style on behalf of psakievich

* Style

* Update lib/spack/spack/test/concretize.py

Co-authored-by: Greg Becker <becker33@llnl.gov>

* Update solver rule

* Duplicate installed rules for installed_git_version

* Revert "Duplicate installed rules for installed_git_version"

This reverts commit 17223fc8d1.

---------

Co-authored-by: psakievich <psakievich@users.noreply.github.com>
Co-authored-by: Greg Becker <becker33@llnl.gov>
This commit is contained in:
psakievich 2024-06-11 14:57:09 -06:00 committed by GitHub
parent a49b2f4f16
commit 337bf3b944
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 97 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
lib/spack/spack/solver/asp.py
View File

@ -116,6 +116,8 @@ class Provenance(enum.IntEnum):
PACKAGE_PY = enum.auto()
# An installed spec
INSTALLED = enum.auto()
# lower provenance for installed git refs so concretizer prefers StandardVersion installs
INSTALLED_GIT_VERSION = enum.auto()
# A runtime injected from another package (e.g. a compiler)
RUNTIME = enum.auto()
@ -2072,7 +2074,7 @@ def define_ad_hoc_versions_from_specs(
# best possible, so they're guaranteed to be used preferentially.
version = s.versions.concrete
if version is None or any(v == version for v in self.possible_versions[s.name]):
if version is None or (any((v == version) for v in self.possible_versions[s.name])):
continue
if require_checksum and not _is_checksummed_git_version(version):
@ -2386,9 +2388,16 @@ def concrete_specs(self):
# - Add OS to possible OS's
for dep in spec.traverse():
self.possible_versions[dep.name].add(dep.version)
self.declared_versions[dep.name].append(
DeclaredVersion(version=dep.version, idx=0, origin=Provenance.INSTALLED)
)
if isinstance(dep.version, vn.GitVersion):
self.declared_versions[dep.name].append(
DeclaredVersion(
version=dep.version, idx=0, origin=Provenance.INSTALLED_GIT_VERSION
)
)
else:
self.declared_versions[dep.name].append(
DeclaredVersion(version=dep.version, idx=0, origin=Provenance.INSTALLED)
)
self.possible_oses.add(dep.os)
def define_concrete_input_specs(self, specs, possible):

1
lib/spack/spack/solver/concretize.lp
View File

@ -256,6 +256,7 @@ possible_version_weight(node(ID, Package), Weight)
:- attr("version", node(ID, Package), Version),
version_weight(node(ID, Package), Weight),
not pkg_fact(Package, version_declared(Version, Weight, "installed")),
not pkg_fact(Package, version_declared(Version, Weight, "installed_git_version")),
not build(node(ID, Package)),
internal_error("Build version weight used for reused package").

42
lib/spack/spack/test/concretize.py
View File

@ -3043,3 +3043,45 @@ def test_spec_filters(specs, include, exclude, expected):
factory=lambda: specs, is_usable=lambda x: True, include=include, exclude=exclude
)
assert f.selected_specs() == expected
@pytest.mark.only_clingo("clingo only reuse feature being tested")
@pytest.mark.regression("38484")
def test_git_ref_version_can_be_reused(
install_mockery_mutable_config, do_not_check_runtimes_on_reuse
):
first_spec = spack.spec.Spec("git-ref-package@git.2.1.5=2.1.5~opt").concretized()
first_spec.package.do_install(fake=True, explicit=True)
with spack.config.override("concretizer:reuse", True):
# reproducer of the issue is that spack will solve when there is a change to the base spec
second_spec = spack.spec.Spec("git-ref-package@git.2.1.5=2.1.5+opt").concretized()
assert second_spec.dag_hash() != first_spec.dag_hash()
# we also want to confirm that reuse actually works so leave variant off to
# let solver reuse
third_spec = spack.spec.Spec("git-ref-package@git.2.1.5=2.1.5")
assert first_spec.satisfies(third_spec)
third_spec.concretize()
assert third_spec.dag_hash() == first_spec.dag_hash()
@pytest.mark.only_clingo("clingo only reuse feature being tested")
@pytest.mark.parametrize("standard_version", ["2.0.0", "2.1.5", "2.1.6"])
def test_reuse_prefers_standard_over_git_versions(
standard_version, install_mockery_mutable_config, do_not_check_runtimes_on_reuse
):
"""
order matters in this test. typically reuse would pick the highest versioned installed match
but we want to prefer the standard version over git ref based versions
so install git ref last and ensure it is not picked up by reuse
"""
standard_spec = spack.spec.Spec(f"git-ref-package@{standard_version}").concretized()
standard_spec.package.do_install(fake=True, explicit=True)
git_spec = spack.spec.Spec("git-ref-package@git.2.1.5=2.1.5").concretized()
git_spec.package.do_install(fake=True, explicit=True)
with spack.config.override("concretizer:reuse", True):
test_spec = spack.spec.Spec("git-ref-package@2").concretized()
assert git_spec.dag_hash() != test_spec.dag_hash()
assert standard_spec.dag_hash() == test_spec.dag_hash()

41
var/spack/repos/builtin.mock/packages/git-ref-package/package.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
class GitRefPackage(AutotoolsPackage):
"""
dummy package copied from zlib-ng
"""
homepage = "https://github.com/dummy/dummy"
url = "https://github.com/dummy/dummy/archive/2.0.0.tar.gz"
git = "https://github.com/dummy/dummy.git"
version("2.1.6", sha256="a5d504c0d52e2e2721e7e7d86988dec2e290d723ced2307145dedd06aeb6fef2")
version("2.1.5", sha256="3f6576971397b379d4205ae5451ff5a68edf6c103b2f03c4188ed7075fbb5f04")
version("2.1.4", sha256="a0293475e6a44a3f6c045229fe50f69dc0eebc62a42405a51f19d46a5541e77a")
version(
"2.1.3",
sha256="d20e55f89d71991c59f1c5ad1ef944815e5850526c0d9cd8e504eaed5b24491a",
deprecated=True,
)
version(
"2.1.2",
sha256="383560d6b00697c04e8878e26c0187b480971a8bce90ffd26a5a7b0f7ecf1a33",
deprecated=True,
)
version("2.0.7", sha256="6c0853bb27738b811f2b4d4af095323c3d5ce36ceed6b50e5f773204fb8f7200")
version("2.0.0", sha256="86993903527d9b12fc543335c19c1d33a93797b3d4d37648b5addae83679ecd8")
variant("compat", default=True, description="Enable compatibility API")
variant("opt", default=True, description="Enable optimizations")
variant("shared", default=True, description="Build shared library")
variant("pic", default=True, description="Enable position-independent code (PIC)")
conflicts("+shared~pic")
variant("new_strategies", default=True, description="Enable new deflate strategies")