zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reuse concretization: allow externals from remote when locally configured (#35975)

Browse Source 
This looks to me like the best compromise regarding externals in a
build cache. I wouldn't want `spack install` on my machine to install
specs that were marked external on another. At the same time there are
centers that control the target systems on which spack is used, and
would want to use external in buildcaches.

As a solution, reuse concretization will now consider those externals
used in buildcaches that match a locally configured external in
packages.yaml.

So for example person A installs and pushes specs with this config:

```yaml
packages:
  ncurses:
    externals:
    - spec: ncurses@6.0.12345 +feature
      prefix: /usr
```

and person B concretizes and installs using that buildcache with the
following config:

```yaml
packages:
  ncurses:
    externals:
    - spec: ncurses@6
    prefix: /usr
```

the spec will be reused (or rather, will be considered for reuse...)
This commit is contained in:
Harmen Stoppels
2023-11-30 18:38:05 +01:00
committed by GitHub
parent f3983d60c2
commit d436e97fc6
3 changed files with 114 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lib/spack/spack/binary_distribution.py
View File

@@ -230,7 +230,11 @@ def _associate_built_specs_with_mirror(self, cache_key, mirror_url):
)
return
spec_list = db.query_local(installed=False, in_buildcache=True)
spec_list = [
s
for s in db.query_local(installed=any, in_buildcache=any)
if s.external or db.query_local_by_spec_hash(s.dag_hash()).in_buildcache
]
for indexed_spec in spec_list:
dag_hash = indexed_spec.dag_hash()

33
lib/spack/spack/solver/asp.py
View File

@@ -3134,6 +3134,25 @@ def _develop_specs_from_env(spec, env):
spec.constrain(dev_info["spec"])
def _is_reusable_external(packages, spec: spack.spec.Spec) -> bool:
"""Returns true iff spec is an external that can be reused.
Arguments:
packages: the packages configuration
spec: the spec to check
"""
for name in {spec.name, *(p.name for p in spec.package.provided)}:
for entry in packages.get(name, {}).get("externals", []):
if (
spec.satisfies(entry["spec"])
and spec.external_path == entry.get("prefix")
and spec.external_modules == entry.get("modules")
):
return True
return False
class Solver:
"""This is the main external interface class for solving.
@@ -3181,8 +3200,18 @@ def _reusable_specs(self, specs):
# Specs from buildcaches
try:
index = spack.binary_distribution.update_cache_and_get_specs()
reusable_specs.extend(index)
# Specs in a build cache that depend on externals are reusable as long as local
# config has matching externals. This should guard against picking up binaries
# linked against externals not available locally, while still supporting the use
# case of distributing binaries across machines with similar externals.
packages = spack.config.get("packages")
reusable_specs.extend(
[
s
for s in spack.binary_distribution.update_cache_and_get_specs()
if not s.external or _is_reusable_external(packages, s)
]
)
except (spack.binary_distribution.FetchCacheError, IndexError):
# this is raised when no mirrors had indices.
# TODO: update mirror configuration so it can indicate that the

78
lib/spack/spack/test/concretize.py
View File

@@ -2427,3 +2427,81 @@ def test_virtuals_provided_together_but_only_one_required_in_dag(self):
s = Spec("blas-only-client ^openblas").concretized()
assert s.satisfies("^[virtuals=blas] openblas")
assert not s.satisfies("^[virtuals=blas,lapack] openblas")
def test_reusable_externals_match(mock_packages, tmpdir):
spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
spec.external_path = tmpdir.strpath
spec.external_modules = ["mpich/4.1"]
spec._mark_concrete()
assert spack.solver.asp._is_reusable_external(
{
"mpich": {
"externals": [
{"spec": "mpich@4.1", "prefix": tmpdir.strpath, "modules": ["mpich/4.1"]}
]
}
},
spec,
)
def test_reusable_externals_match_virtual(mock_packages, tmpdir):
spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
spec.external_path = tmpdir.strpath
spec.external_modules = ["mpich/4.1"]
spec._mark_concrete()
assert spack.solver.asp._is_reusable_external(
{
"mpi": {
"externals": [
{"spec": "mpich@4.1", "prefix": tmpdir.strpath, "modules": ["mpich/4.1"]}
]
}
},
spec,
)
def test_reusable_externals_different_prefix(mock_packages, tmpdir):
spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
spec.external_path = "/other/path"
spec.external_modules = ["mpich/4.1"]
spec._mark_concrete()
assert not spack.solver.asp._is_reusable_external(
{
"mpich": {
"externals": [
{"spec": "mpich@4.1", "prefix": tmpdir.strpath, "modules": ["mpich/4.1"]}
]
}
},
spec,
)
@pytest.mark.parametrize("modules", [None, ["mpich/4.1", "libfabric/1.19"]])
def test_reusable_externals_different_modules(mock_packages, tmpdir, modules):
spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
spec.external_path = tmpdir.strpath
spec.external_modules = modules
spec._mark_concrete()
assert not spack.solver.asp._is_reusable_external(
{
"mpich": {
"externals": [
{"spec": "mpich@4.1", "prefix": tmpdir.strpath, "modules": ["mpich/4.1"]}
]
}
},
spec,
)
def test_reusable_externals_different_spec(mock_packages, tmpdir):
spec = Spec("mpich@4.1%gcc@13.1.0~debug build_system=generic arch=linux-ubuntu23.04-zen2")
spec.external_path = tmpdir.strpath
spec._mark_concrete()
assert not spack.solver.asp._is_reusable_external(
{"mpich": {"externals": [{"spec": "mpich@4.1 +debug", "prefix": tmpdir.strpath}]}}, spec
)