Fix pickle round-trip of specs propagating variants (#47351)
This changes `Spec` serialization to include information about propagation for abstract specs. This was previously not included in the JSON representation for abstract specs, and couldn't be stored. Now, there is a separate `propagate` dictionary alongside the `parameters` dictionary. This isn't beautiful, but when we bump the spec version for Spack `v0.24`, we can clean up this and other aspects of the schema.
This commit is contained in:
parent
88b47d2714
commit
e0d246210f
@ -2045,6 +2045,18 @@ def to_node_dict(self, hash=ht.dag_hash):
|
||||
if params:
|
||||
d["parameters"] = params
|
||||
|
||||
if params and not self.concrete:
|
||||
flag_names = [
|
||||
name
|
||||
for name, flags in self.compiler_flags.items()
|
||||
if any(x.propagate for x in flags)
|
||||
]
|
||||
d["propagate"] = sorted(
|
||||
itertools.chain(
|
||||
[v.name for v in self.variants.values() if v.propagate], flag_names
|
||||
)
|
||||
)
|
||||
|
||||
if self.external:
|
||||
d["external"] = syaml.syaml_dict(
|
||||
[
|
||||
@ -2217,16 +2229,10 @@ def node_dict_with_hashes(self, hash=ht.dag_hash):
|
||||
spec is concrete, the full hash is added as well. If 'build' is in
|
||||
the hash_type, the build hash is also added."""
|
||||
node = self.to_node_dict(hash)
|
||||
# All specs have at least a DAG hash
|
||||
node[ht.dag_hash.name] = self.dag_hash()
|
||||
|
||||
# dag_hash is lazily computed -- but if we write a spec out, we want it
|
||||
# to be included. This is effectively the last chance we get to compute
|
||||
# it accurately.
|
||||
if self.concrete:
|
||||
# all specs have at least a DAG hash
|
||||
node[ht.dag_hash.name] = self.dag_hash()
|
||||
|
||||
else:
|
||||
if not self.concrete:
|
||||
node["concrete"] = False
|
||||
|
||||
# we can also give them other hash types if we want
|
||||
@ -4999,13 +5005,17 @@ def from_node_dict(cls, node):
|
||||
else:
|
||||
spec.compiler = None
|
||||
|
||||
propagated_names = node.get("propagate", [])
|
||||
for name, values in node.get("parameters", {}).items():
|
||||
propagate = name in propagated_names
|
||||
if name in _valid_compiler_flags:
|
||||
spec.compiler_flags[name] = []
|
||||
for val in values:
|
||||
spec.compiler_flags.add_flag(name, val, False)
|
||||
spec.compiler_flags.add_flag(name, val, propagate)
|
||||
else:
|
||||
spec.variants[name] = vt.MultiValuedVariant.from_node_dict(name, values)
|
||||
spec.variants[name] = vt.MultiValuedVariant.from_node_dict(
|
||||
name, values, propagate=propagate
|
||||
)
|
||||
|
||||
spec.external_path = None
|
||||
spec.external_modules = None
|
||||
|
@ -16,6 +16,7 @@
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
|
||||
import pytest
|
||||
import ruamel.yaml
|
||||
@ -554,3 +555,26 @@ def test_anchorify_2():
|
||||
e: *id002
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"spec_str",
|
||||
[
|
||||
"hdf5 ++mpi",
|
||||
"hdf5 cflags==-g",
|
||||
"hdf5 foo==bar",
|
||||
"hdf5~~mpi++shared",
|
||||
"hdf5 cflags==-g foo==bar cxxflags==-O3",
|
||||
"hdf5 cflags=-g foo==bar cxxflags==-O3",
|
||||
],
|
||||
)
|
||||
def test_pickle_roundtrip_for_abstract_specs(spec_str):
|
||||
"""Tests that abstract specs correctly round trip when pickled.
|
||||
|
||||
This test compares both spec objects and their string representation, due to some
|
||||
inconsistencies in how `Spec.__eq__` is implemented.
|
||||
"""
|
||||
s = spack.spec.Spec(spec_str)
|
||||
t = pickle.loads(pickle.dumps(s))
|
||||
assert s == t
|
||||
assert str(s) == str(t)
|
||||
|
@ -12,6 +12,7 @@
|
||||
import io
|
||||
import itertools
|
||||
import re
|
||||
from typing import List, Union
|
||||
|
||||
import llnl.util.lang as lang
|
||||
import llnl.util.tty.color
|
||||
@ -255,19 +256,21 @@ def __init__(self, name, value, propagate=False):
|
||||
self.value = value
|
||||
|
||||
@staticmethod
|
||||
def from_node_dict(name, value):
|
||||
def from_node_dict(
|
||||
name: str, value: Union[str, List[str]], *, propagate: bool = False
|
||||
) -> "AbstractVariant":
|
||||
"""Reconstruct a variant from a node dict."""
|
||||
if isinstance(value, list):
|
||||
# read multi-value variants in and be faithful to the YAML
|
||||
mvar = MultiValuedVariant(name, ())
|
||||
mvar = MultiValuedVariant(name, (), propagate=propagate)
|
||||
mvar._value = tuple(value)
|
||||
mvar._original_value = mvar._value
|
||||
return mvar
|
||||
|
||||
elif str(value).upper() == "TRUE" or str(value).upper() == "FALSE":
|
||||
return BoolValuedVariant(name, value)
|
||||
return BoolValuedVariant(name, value, propagate=propagate)
|
||||
|
||||
return SingleValuedVariant(name, value)
|
||||
return SingleValuedVariant(name, value, propagate=propagate)
|
||||
|
||||
def yaml_entry(self):
|
||||
"""Returns a key, value tuple suitable to be an entry in a yaml dict.
|
||||
|
Loading…
Reference in New Issue
Block a user