zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

spec.to_dict: round trip whether deps are direct

Browse Source 
Signed-off-by: Gregory Becker <becker33@llnl.gov>
This commit is contained in:
Gregory Becker 2025-03-31 11:54:39 -05:00 committed by Harmen Stoppels
parent b932c14008
commit 3b2563c561
5 changed files with 58 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/spack/spack/database.py
View File

@ -778,7 +778,7 @@ def _assign_dependencies(
spec_node_dict = spec_node_dict[spec.name]
if "dependencies" in spec_node_dict:
yaml_deps = spec_node_dict["dependencies"]
for dname, dhash, dtypes, _, virtuals in spec_reader.read_specfile_dep_specs(
for dname, dhash, dtypes, _, virtuals, direct in spec_reader.read_specfile_dep_specs(
yaml_deps
):
# It is important that we always check upstream installations in the same order,
@ -797,7 +797,9 @@ def _assign_dependencies(
)
continue
spec._add_dependency(child, depflag=dt.canonicalize(dtypes), virtuals=virtuals)
spec._add_dependency(
child, depflag=dt.canonicalize(dtypes), virtuals=virtuals, direct=direct
)
def _read_from_file(self, filename: pathlib.Path, *, reindex: bool = False) -> None:
"""Fill database from file, do not maintain old data.

7
lib/spack/spack/environment/__init__.py
View File

@ -544,6 +544,13 @@
}
}
Version 7
---------
Version 7 does not change the lockfile format itself, but reflects
a change in the specfile format. This adds the ``direct`` key for
direct dependency edges. These are not relevant for environment
behavior and merely require a change of specfile reader.
"""
from .environment import (

12
lib/spack/spack/environment/environment.py
View File

@ -129,7 +129,7 @@ def default_manifest_yaml():
valid_environment_name_re = r"^\w[\w-]*$"
#: version of the lockfile format. Must increase monotonically.
lockfile_format_version = 6
lockfile_format_version = 7
READER_CLS = {
@ -139,6 +139,7 @@ def default_manifest_yaml():
4: spack.spec.SpecfileV3,
5: spack.spec.SpecfileV4,
6: spack.spec.SpecfileV5,
7: spack.spec.SpecfileV6,
}
@ -2230,9 +2231,14 @@ def filter_specs(self, reader, json_specs_by_hash, order_concretized):
# and add them to the spec, including build specs
for lockfile_key, node_dict in json_specs_by_hash.items():
name, data = reader.name_and_data(node_dict)
for _, dep_hash, deptypes, _, virtuals in reader.dependencies_from_node_dict(data):
for _, dep_hash, deptypes, _, virtuals, direct in reader.dependencies_from_node_dict(
data
):
specs_by_hash[lockfile_key]._add_dependency(
specs_by_hash[dep_hash], depflag=dt.canonicalize(deptypes), virtuals=virtuals
specs_by_hash[dep_hash],
depflag=dt.canonicalize(deptypes),
virtuals=virtuals,
direct=direct,
)
if "build_spec" in node_dict:

47
lib/spack/spack/spec.py
View File

@ -187,7 +187,7 @@
)
#: specfile format version. Must increase monotonically
SPECFILE_FORMAT_VERSION = 5
SPECFILE_FORMAT_VERSION = 6
class InstallStatus(enum.Enum):
@ -2383,6 +2383,7 @@ def to_node_dict(self, hash=ht.dag_hash):
"parameters": {
"deptypes": dt.flag_to_tuple(dspec.depflag),
"virtuals": dspec.virtuals,
"direct": dspec.direct,
},
}
for name, edges_for_name in sorted(deps.items())
@ -2683,7 +2684,7 @@ def name_and_dependency_types(s: str) -> Tuple[str, dt.DepFlag]:
return name, depflag
def spec_and_dependency_types(
s: Union[Spec, Tuple[Spec, str]],
s: Union[Spec, Tuple[Spec, str]]
) -> Tuple[Spec, dt.DepFlag]:
"""Given a non-string key in the literal, extracts the spec
and its dependency types.
@ -2727,8 +2728,10 @@ def from_dict(data) -> "Spec":
spec = SpecfileV3.load(data)
elif int(data["spec"]["_meta"]["version"]) == 4:
spec = SpecfileV4.load(data)
else:
elif int(data["spec"]["_meta"]["version"]) == 5:
spec = SpecfileV5.load(data)
else:
spec = SpecfileV6.load(data)
# Any git version should
for s in spec.traverse():
@ -4890,7 +4893,7 @@ def _load(cls, data):
# Pass 0: Determine hash type
for node in nodes:
for _, _, _, dhash_type, _ in cls.dependencies_from_node_dict(node):
for _, _, _, dhash_type, _, _ in cls.dependencies_from_node_dict(node):
any_deps = True
if dhash_type:
hash_type = dhash_type
@ -4921,11 +4924,12 @@ def _load(cls, data):
# Pass 2: Finish construction of all DAG edges (including build specs)
for node_hash, node in hash_dict.items():
node_spec = node["node_spec"]
for _, dhash, dtype, _, virtuals in cls.dependencies_from_node_dict(node):
for _, dhash, dtype, _, virtuals, direct in cls.dependencies_from_node_dict(node):
node_spec._add_dependency(
hash_dict[dhash]["node_spec"],
depflag=dt.canonicalize(dtype),
virtuals=virtuals,
direct=direct,
)
if "build_spec" in node.keys():
_, bhash, _ = cls.extract_build_spec_info_from_node_dict(node, hash_type=hash_type)
@ -4966,9 +4970,9 @@ def load(cls, data):
for node in nodes:
# get dependency dict from the node.
name, data = cls.name_and_data(node)
for dname, _, dtypes, _, virtuals in cls.dependencies_from_node_dict(data):
for dname, _, dtypes, _, virtuals, direct in cls.dependencies_from_node_dict(data):
deps[name]._add_dependency(
deps[dname], depflag=dt.canonicalize(dtypes), virtuals=virtuals
deps[dname], depflag=dt.canonicalize(dtypes), virtuals=virtuals, direct=direct
)
reconstruct_virtuals_on_edges(result)
@ -5006,7 +5010,7 @@ def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):
raise spack.error.SpecError("Couldn't parse dependency spec.")
else:
raise spack.error.SpecError("Couldn't parse dependency types in spec.")
yield dep_name, dep_hash, list(deptypes), hash_type, list(virtuals)
yield dep_name, dep_hash, list(deptypes), hash_type, list(virtuals), True
class SpecfileV2(SpecfileReaderBase):
@ -5043,13 +5047,15 @@ def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):
# new format: elements of dependency spec are keyed.
for h in ht.HASHES:
if h.name in elt:
dep_hash, deptypes, hash_type, virtuals = cls.extract_info_from_dep(elt, h)
dep_hash, deptypes, hash_type, virtuals, direct = (
cls.extract_info_from_dep(elt, h)
)
break
else: # We never determined a hash type...
raise spack.error.SpecError("Couldn't parse dependency spec.")
else:
raise spack.error.SpecError("Couldn't parse dependency types in spec.")
result.append((dep_name, dep_hash, list(deptypes), hash_type, list(virtuals)))
result.append((dep_name, dep_hash, list(deptypes), hash_type, list(virtuals), direct))
return result
@classmethod
@ -5057,7 +5063,8 @@ def extract_info_from_dep(cls, elt, hash):
dep_hash, deptypes = elt[hash.name], elt["type"]
hash_type = hash.name
virtuals = []
return dep_hash, deptypes, hash_type, virtuals
direct = True
return dep_hash, deptypes, hash_type, virtuals, direct
@classmethod
def extract_build_spec_info_from_node_dict(cls, node, hash_type=ht.dag_hash.name):
@ -5078,7 +5085,8 @@ def extract_info_from_dep(cls, elt, hash):
deptypes = elt["parameters"]["deptypes"]
hash_type = hash.name
virtuals = elt["parameters"]["virtuals"]
return dep_hash, deptypes, hash_type, virtuals
direct = True
return dep_hash, deptypes, hash_type, virtuals, direct
@classmethod
def load(cls, data):
@ -5093,8 +5101,21 @@ def legacy_compiler(cls, node):
raise RuntimeError("The 'compiler' option is unexpected in specfiles at v5 or greater")
class SpecfileV6(SpecfileV5):
SPEC_VERSION = 6
@classmethod
def extract_info_from_dep(cls, elt, hash):
dep_hash = elt[hash.name]
deptypes = elt["parameters"]["deptypes"]
hash_type = hash.name
virtuals = elt["parameters"]["virtuals"]
direct = elt["parameters"]["direct"]
return dep_hash, deptypes, hash_type, virtuals, direct
#: Alias to the latest version of specfiles
SpecfileLatest = SpecfileV5
SpecfileLatest = SpecfileV6
class LazySpecCache(collections.defaultdict):

4
lib/spack/spack/test/spec_yaml.py
View File

@ -494,6 +494,10 @@ def test_anchorify_2():
"hdf5~~mpi++shared",
"hdf5 cflags==-g foo==bar cxxflags==-O3",
"hdf5 cflags=-g foo==bar cxxflags==-O3",
"hdf5%gcc",
"hdf5%cmake",
"hdf5^gcc",
"hdf5^cmake",
],
)
def test_pickle_roundtrip_for_abstract_specs(spec_str):