zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update vendored ruamel.yaml to v0.17.21 (#37008)

Browse Source 
* Vendor ruamel.yaml v0.17.21

* Add unit test for whitespace regression

* Add an abstraction layer in Spack to wrap ruamel.yaml

All YAML operations are routed through spack.util.spack_yaml

The custom classes have been adapted to the new ruamel.yaml
class hierarchy.

Fixed line annotation issue in "spack config blame"
This commit is contained in:
Massimiliano Culpo
2023-05-04 17:00:38 +02:00
committed by GitHub
parent 95e61f2fdf
commit 600955edd4
73 changed files with 15046 additions and 8603 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lib/spack/spack/binary_distribution.py
View File

@@ -27,8 +27,6 @@
from typing import List, NamedTuple, Optional, Union
from urllib.error import HTTPError, URLError
import ruamel.yaml as yaml
import llnl.util.filesystem as fsys
import llnl.util.lang
import llnl.util.tty as tty
@@ -616,7 +614,7 @@ def read_buildinfo_file(prefix):
filename = buildinfo_file_name(prefix)
with open(filename, "r") as inputfile:
content = inputfile.read()
buildinfo = yaml.load(content)
buildinfo = syaml.load(content)
return buildinfo

8
lib/spack/spack/cmd/__init__.py
View File

@@ -13,9 +13,6 @@
from textwrap import dedent
from typing import List, Match, Tuple
import ruamel.yaml as yaml
from ruamel.yaml.error import MarkedYAMLError
import llnl.util.tty as tty
from llnl.util.filesystem import join_path
from llnl.util.lang import attr_setdefault, index_by
@@ -33,6 +30,7 @@
import spack.traverse as traverse
import spack.user_environment as uenv
import spack.util.spack_json as sjson
import spack.util.spack_yaml as syaml
import spack.util.string
# cmd has a submodule called "list" so preserve the python list module
@@ -537,9 +535,9 @@ def is_git_repo(path):
# we might be in a git worktree
try:
with open(dotgit_path, "rb") as f:
dotgit_content = yaml.load(f)
dotgit_content = syaml.load(f)
return os.path.isdir(dotgit_content.get("gitdir", dotgit_path))
except MarkedYAMLError:
except syaml.SpackYAMLError:
pass
return False

54
lib/spack/spack/config.py
View File

@@ -38,10 +38,6 @@
from contextlib import contextmanager
from typing import Dict, List, Optional, Union
import ruamel.yaml as yaml
from ruamel.yaml.comments import Comment
from ruamel.yaml.error import MarkedYAMLError
import llnl.util.lang
import llnl.util.tty as tty
from llnl.util.filesystem import mkdirp, rename
@@ -163,8 +159,8 @@ def _write_section(self, section):
mkdirp(self.path)
with open(filename, "w") as f:
syaml.dump_config(data, stream=f, default_flow_style=False)
except (yaml.YAMLError, IOError) as e:
raise ConfigFileError("Error writing to config file: '%s'" % str(e))
except (syaml.SpackYAMLError, IOError) as e:
raise ConfigFileError(f"cannot write to '{filename}'") from e
def clear(self):
"""Empty cached config information."""
@@ -293,8 +289,8 @@ def _write_section(self, section):
syaml.dump_config(data_to_write, stream=f, default_flow_style=False)
rename(tmp, self.path)
except (yaml.YAMLError, IOError) as e:
raise ConfigFileError("Error writing to config file: '%s'" % str(e))
except (syaml.SpackYAMLError, IOError) as e:
raise ConfigFileError(f"cannot write to config file {str(e)}") from e
def __repr__(self):
return "<SingleFileScope: %s: %s>" % (self.name, self.path)
@@ -546,12 +542,12 @@ def update_config(
# manually preserve comments
need_comment_copy = section in scope.sections and scope.sections[section]
if need_comment_copy:
comments = getattr(scope.sections[section][section], Comment.attrib, None)
comments = syaml.extract_comments(scope.sections[section][section])
# read only the requested section's data.
scope.sections[section] = syaml.syaml_dict({section: update_data})
if need_comment_copy and comments:
setattr(scope.sections[section][section], Comment.attrib, comments)
syaml.set_comments(scope.sections[section][section], data_comments=comments)
scope._write_section(section)
@@ -704,8 +700,8 @@ def print_section(self, section, blame=False):
data = syaml.syaml_dict()
data[section] = self.get_config(section)
syaml.dump_config(data, stream=sys.stdout, default_flow_style=False, blame=blame)
except (yaml.YAMLError, IOError):
raise ConfigError("Error reading configuration: %s" % section)
except (syaml.SpackYAMLError, IOError) as e:
raise ConfigError(f"cannot read '{section}' configuration") from e
@contextmanager
@@ -959,19 +955,9 @@ def validate(data, schema, filename=None):
"""
import jsonschema
# validate a copy to avoid adding defaults
# Validate a copy to avoid adding defaults
# This allows us to round-trip data without adding to it.
test_data = copy.deepcopy(data)
if isinstance(test_data, yaml.comments.CommentedMap):
# HACK to fully copy ruamel CommentedMap that doesn't provide copy
# method. Especially necessary for environments
setattr(
test_data,
yaml.comments.Comment.attrib,
getattr(data, yaml.comments.Comment.attrib, yaml.comments.Comment()),
)
test_data = syaml.deepcopy(data)
try:
spack.schema.Validator(schema).validate(test_data)
except jsonschema.ValidationError as e:
@@ -1019,21 +1005,13 @@ def read_config_file(filename, schema=None):
return data
except StopIteration:
raise ConfigFileError("Config file is empty or is not a valid YAML dict: %s" % filename)
raise ConfigFileError(f"Config file is empty or is not a valid YAML dict: {filename}")
except MarkedYAMLError as e:
msg = "Error parsing yaml"
mark = e.context_mark if e.context_mark else e.problem_mark
if mark:
line, column = mark.line, mark.column
msg += ": near %s, %s, %s" % (mark.name, str(line), str(column))
else:
msg += ": %s" % (filename)
msg += ": %s" % (e.problem)
raise ConfigFileError(msg)
except syaml.SpackYAMLError as e:
raise ConfigFileError(str(e)) from e
except IOError as e:
raise ConfigFileError("Error reading configuration file %s: %s" % (filename, str(e)))
raise ConfigFileError(f"Error reading configuration file {filename}: {str(e)}") from e
def _override(string):
@@ -1089,8 +1067,8 @@ def _mark_internal(data, name):
d = syaml.syaml_type(data)
if syaml.markable(d):
d._start_mark = yaml.Mark(name, None, None, None, None, None)
d._end_mark = yaml.Mark(name, None, None, None, None, None)
d._start_mark = syaml.name_mark(name)
d._end_mark = syaml.name_mark(name)
return d

7
lib/spack/spack/environment/environment.py
View File

@@ -18,8 +18,6 @@
import warnings
from typing import Any, Dict, List, Optional, Union
import ruamel.yaml as yaml
import llnl.util.filesystem as fs
import llnl.util.tty as tty
import llnl.util.tty.color as clr
@@ -526,11 +524,6 @@ def __eq__(self, other):
def to_dict(self):
ret = syaml.syaml_dict([("root", self.raw_root)])
if self.projections:
# projections guaranteed to be ordered dict if true-ish
# for python2.6, may be syaml or ruamel.yaml implementation
# so we have to check for both
types = (collections.OrderedDict, syaml.syaml_dict, yaml.comments.CommentedMap)
assert isinstance(self.projections, types)
ret["projections"] = self.projections
if self.select:
ret["select"] = self.select

16
lib/spack/spack/mirror.py
View File

@@ -19,8 +19,6 @@
import traceback
import urllib.parse
import ruamel.yaml.error as yaml_error
import llnl.util.tty as tty
from llnl.util.filesystem import mkdirp
@@ -89,11 +87,8 @@ def to_yaml(self, stream=None):
@staticmethod
def from_yaml(stream, name=None):
try:
data = syaml.load(stream)
return Mirror.from_dict(data, name)
except yaml_error.MarkedYAMLError as e:
raise syaml.SpackYAMLError("error parsing YAML mirror:", str(e)) from e
data = syaml.load(stream)
return Mirror.from_dict(data, name)
@staticmethod
def from_json(stream, name=None):
@@ -288,11 +283,8 @@ def to_yaml(self, stream=None):
# TODO: this isn't called anywhere
@staticmethod
def from_yaml(stream, name=None):
try:
data = syaml.load(stream)
return MirrorCollection(data)
except yaml_error.MarkedYAMLError as e:
raise syaml.SpackYAMLError("error parsing YAML mirror collection:", str(e)) from e
data = syaml.load(stream)
return MirrorCollection(data)
@staticmethod
def from_json(stream, name=None):

5
lib/spack/spack/repo.py
View File

@@ -26,8 +26,6 @@
import uuid
from typing import Dict, Union
import ruamel.yaml as yaml
import llnl.util.filesystem as fs
import llnl.util.lang
import llnl.util.tty as tty
@@ -44,6 +42,7 @@
import spack.util.git
import spack.util.naming as nm
import spack.util.path
import spack.util.spack_yaml as syaml
#: Package modules are imported as spack.pkg.<repo-namespace>.<pkg-name>
ROOT_PYTHON_NAMESPACE = "spack.pkg"
@@ -1008,7 +1007,7 @@ def _read_config(self):
"""Check for a YAML config file in this db's root directory."""
try:
with open(self.config_file) as reponame_file:
yaml_data = yaml.load(reponame_file)
yaml_data = syaml.load(reponame_file)
if (
not yaml_data

9
lib/spack/spack/spec.py
View File

@@ -57,8 +57,6 @@
import warnings
from typing import Tuple
import ruamel.yaml as yaml
import llnl.util.filesystem as fs
import llnl.util.lang as lang
import llnl.util.tty as tty
@@ -2305,11 +2303,8 @@ def from_yaml(stream):
Args:
stream: string or file object to read from.
"""
try:
data = yaml.load(stream)
return Spec.from_dict(data)
except yaml.error.MarkedYAMLError as e:
raise syaml.SpackYAMLError("error parsing YAML spec:", str(e)) from e
data = syaml.load(stream)
return Spec.from_dict(data)
@staticmethod
def from_json(stream):

2
lib/spack/spack/test/cmd/config.py
View File

@@ -483,7 +483,7 @@ def test_config_add_to_env_preserve_comments(mutable_empty_config, mutable_mock_
spack: # comment
# comment
specs: # comment
- foo # comment
- foo # comment
# comment
view: true # comment
packages: # comment

2
lib/spack/spack/test/config.py
View File

@@ -1414,5 +1414,5 @@ def test_config_file_read_invalid_yaml(tmpdir, mutable_empty_config):
with open(filename, "w") as f:
f.write("spack:\nview")
with pytest.raises(spack.config.ConfigFileError, match="parsing yaml"):
with pytest.raises(spack.config.ConfigFileError, match="parsing YAML"):
spack.config.read_config_file(filename)

34
lib/spack/spack/test/env.py
View File

@@ -385,3 +385,37 @@ def test_can_add_specs_to_environment_without_specs_attribute(tmp_path, mock_pac
assert len(env.user_specs) == 1
assert env.manifest.pristine_yaml_content["spack"]["specs"] == ["a"]
@pytest.mark.parametrize(
"original_yaml,new_spec,expected_yaml",
[
(
"""spack:
specs:
# baz
- zlib
""",
"libpng",
"""spack:
specs:
# baz
- zlib
- libpng
""",
)
],
)
def test_preserving_comments_when_adding_specs(
original_yaml, new_spec, expected_yaml, config, tmp_path
):
"""Ensure that round-tripping a spack.yaml file doesn't change its content."""
spack_yaml = tmp_path / "spack.yaml"
spack_yaml.write_text(original_yaml)
e = ev.Environment(str(tmp_path))
e.add(new_spec)
e.write()
content = spack_yaml.read_text()
assert content == expected_yaml

12
lib/spack/spack/test/mirror.py
View File

@@ -145,11 +145,9 @@ def test_roundtrip_mirror(mirror):
"invalid_yaml", ["playing_playlist: {{ action }} playlist {{ playlist_name }}"]
)
def test_invalid_yaml_mirror(invalid_yaml):
with pytest.raises(SpackYAMLError) as e:
with pytest.raises(SpackYAMLError, match="error parsing YAML") as e:
spack.mirror.Mirror.from_yaml(invalid_yaml)
exc_msg = str(e.value)
assert exc_msg.startswith("error parsing YAML mirror:")
assert invalid_yaml in exc_msg
assert invalid_yaml in str(e.value)
@pytest.mark.parametrize("invalid_json, error_message", [("{13:", "Expecting property name")])
@@ -184,11 +182,9 @@ def test_roundtrip_mirror_collection(mirror_collection):
"invalid_yaml", ["playing_playlist: {{ action }} playlist {{ playlist_name }}"]
)
def test_invalid_yaml_mirror_collection(invalid_yaml):
with pytest.raises(SpackYAMLError) as e:
with pytest.raises(SpackYAMLError, match="error parsing YAML") as e:
spack.mirror.MirrorCollection.from_yaml(invalid_yaml)
exc_msg = str(e.value)
assert exc_msg.startswith("error parsing YAML mirror collection:")
assert invalid_yaml in exc_msg
assert invalid_yaml in str(e.value)
@pytest.mark.parametrize("invalid_json, error_message", [("{13:", "Expecting property name")])

52
lib/spack/spack/test/spack_yaml.py
View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Test Spack's custom YAML format."""
import io
import sys
import pytest
@@ -88,7 +90,53 @@ def test_yaml_aliases():
"e": aliased_list_2,
"f": aliased_list_2,
}
string = syaml.dump(dict_with_aliases)
stringio = io.StringIO()
syaml.dump(dict_with_aliases, stream=stringio)
# ensure no YAML aliases appear in syaml dumps.
assert "*id" not in string
assert "*id" not in stringio.getvalue()
@pytest.mark.parametrize(
"initial_content,expected_final_content",
[
# List are dumped indented as the outer attribute
(
"""spack:
#foo
specs:
# bar
- zlib
""",
None,
),
(
"""spack:
#foo
specs:
# bar
- zlib
""",
"""spack:
#foo
specs:
# bar
- zlib
""",
),
],
)
@pytest.mark.xfail(sys.platform == "win32", reason="fails on Windows")
def test_round_trip_configuration(initial_content, expected_final_content, tmp_path):
"""Test that configuration can be loaded and dumped without too many changes"""
file = tmp_path / "test.yaml"
file.write_text(initial_content)
final_content = io.StringIO()
data = syaml.load_config(file)
syaml.dump_config(data, stream=final_content)
if expected_final_content is None:
expected_final_content = initial_content
assert final_content.getvalue() == expected_final_content

6
lib/spack/spack/test/spec_yaml.py
View File

@@ -81,11 +81,9 @@ def test_normal_spec(mock_packages):
"invalid_yaml", ["playing_playlist: {{ action }} playlist {{ playlist_name }}"]
)
def test_invalid_yaml_spec(invalid_yaml):
with pytest.raises(SpackYAMLError) as e:
with pytest.raises(SpackYAMLError, match="error parsing YAML") as e:
Spec.from_yaml(invalid_yaml)
exc_msg = str(e.value)
assert exc_msg.startswith("error parsing YAML spec:")
assert invalid_yaml in exc_msg
assert invalid_yaml in str(e)
@pytest.mark.parametrize("invalid_json, error_message", [("{13:", "Expecting property name")])

2
lib/spack/spack/test/util/spack_yaml.py
View File

@@ -92,5 +92,5 @@ def get_file_lines(filename):
val = val.lower()
lines = get_file_lines(filename)
assert key in lines[line]
assert key in lines[line], filename
assert val in lines[line]

277
lib/spack/spack/util/spack_yaml.py
View File

@@ -14,13 +14,16 @@
"""
import collections
import collections.abc
import copy
import ctypes
import enum
import functools
import io
import re
from typing import List
from typing import IO, List, Optional
import ruamel.yaml as yaml
from ruamel.yaml import RoundTripDumper, RoundTripLoader
import ruamel.yaml
from ruamel.yaml import comments, constructor, emitter, error, representer
from llnl.util.tty.color import cextra, clen, colorize
@@ -34,7 +37,7 @@
# Also, use OrderedDict instead of just dict.
class syaml_dict(collections.OrderedDict):
def __repr__(self):
mappings = ("%r: %r" % (k, v) for k, v in self.items())
mappings = (f"{k!r}: {v!r}" for k, v in self.items())
return "{%s}" % ", ".join(mappings)
@@ -54,7 +57,7 @@ class syaml_int(int):
syaml_types = {syaml_str: str, syaml_int: int, syaml_dict: dict, syaml_list: list}
markable_types = set(syaml_types) | set([yaml.comments.CommentedSeq, yaml.comments.CommentedMap])
markable_types = set(syaml_types) | {comments.CommentedSeq, comments.CommentedMap}
def syaml_type(obj):
@@ -96,7 +99,7 @@ def marked(obj):
)
class OrderedLineLoader(RoundTripLoader):
class OrderedLineConstructor(constructor.RoundTripConstructor):
"""YAML loader specifically intended for reading Spack configuration
files. It preserves order and line numbers. It also has special-purpose
logic for handling dictionary keys that indicate a Spack config
@@ -120,7 +123,7 @@ class OrderedLineLoader(RoundTripLoader):
#
def construct_yaml_str(self, node):
value = super(OrderedLineLoader, self).construct_yaml_str(node)
value = super().construct_yaml_str(node)
# There is no specific marker to indicate that we are parsing a key,
# so this assumes we are talking about a Spack config override key if
# it ends with a ':' and does not contain a '@' (which can appear
@@ -134,7 +137,7 @@ def construct_yaml_str(self, node):
return value
def construct_yaml_seq(self, node):
gen = super(OrderedLineLoader, self).construct_yaml_seq(node)
gen = super().construct_yaml_seq(node)
data = next(gen)
if markable(data):
mark(data, node)
@@ -143,7 +146,7 @@ def construct_yaml_seq(self, node):
pass
def construct_yaml_map(self, node):
gen = super(OrderedLineLoader, self).construct_yaml_map(node)
gen = super().construct_yaml_map(node)
data = next(gen)
if markable(data):
mark(data, node)
@@ -153,19 +156,24 @@ def construct_yaml_map(self, node):
# register above new constructors
OrderedLineLoader.add_constructor("tag:yaml.org,2002:map", OrderedLineLoader.construct_yaml_map)
OrderedLineLoader.add_constructor("tag:yaml.org,2002:seq", OrderedLineLoader.construct_yaml_seq)
OrderedLineLoader.add_constructor("tag:yaml.org,2002:str", OrderedLineLoader.construct_yaml_str)
OrderedLineConstructor.add_constructor(
"tag:yaml.org,2002:map", OrderedLineConstructor.construct_yaml_map
)
OrderedLineConstructor.add_constructor(
"tag:yaml.org,2002:seq", OrderedLineConstructor.construct_yaml_seq
)
OrderedLineConstructor.add_constructor(
"tag:yaml.org,2002:str", OrderedLineConstructor.construct_yaml_str
)
class OrderedLineDumper(RoundTripDumper):
"""Dumper that preserves ordering and formats ``syaml_*`` objects.
class OrderedLineRepresenter(representer.RoundTripRepresenter):
"""Representer that preserves ordering and formats ``syaml_*`` objects.
This dumper preserves insertion ordering ``syaml_dict`` objects
This representer preserves insertion ordering ``syaml_dict`` objects
when they're written out. It also has some custom formatters
for ``syaml_*`` objects so that they are formatted like their
regular Python equivalents, instead of ugly YAML pyobjects.
"""
def ignore_aliases(self, _data):
@@ -173,7 +181,7 @@ def ignore_aliases(self, _data):
return True
def represent_data(self, data):
result = super(OrderedLineDumper, self).represent_data(data)
result = super().represent_data(data)
if data is None:
result.value = syaml_str("null")
return result
@@ -181,31 +189,53 @@ def represent_data(self, data):
def represent_str(self, data):
if hasattr(data, "override") and data.override:
data = data + ":"
return super(OrderedLineDumper, self).represent_str(data)
return super().represent_str(data)
class SafeDumper(RoundTripDumper):
class SafeRepresenter(representer.RoundTripRepresenter):
def ignore_aliases(self, _data):
"""Make the dumper NEVER print YAML aliases."""
return True
# Make our special objects look like normal YAML ones.
RoundTripDumper.add_representer(syaml_dict, RoundTripDumper.represent_dict)
RoundTripDumper.add_representer(syaml_list, RoundTripDumper.represent_list)
RoundTripDumper.add_representer(syaml_int, RoundTripDumper.represent_int)
RoundTripDumper.add_representer(syaml_str, RoundTripDumper.represent_str)
OrderedLineDumper.add_representer(syaml_str, OrderedLineDumper.represent_str)
representer.RoundTripRepresenter.add_representer(
syaml_dict, representer.RoundTripRepresenter.represent_dict
)
representer.RoundTripRepresenter.add_representer(
syaml_list, representer.RoundTripRepresenter.represent_list
)
representer.RoundTripRepresenter.add_representer(
syaml_int, representer.RoundTripRepresenter.represent_int
)
representer.RoundTripRepresenter.add_representer(
syaml_str, representer.RoundTripRepresenter.represent_str
)
OrderedLineRepresenter.add_representer(syaml_str, OrderedLineRepresenter.represent_str)
#: Max integer helps avoid passing too large a value to cyaml.
maxint = 2 ** (ctypes.sizeof(ctypes.c_int) * 8 - 1) - 1
def dump(obj, default_flow_style=False, stream=None):
return yaml.dump(
obj, default_flow_style=default_flow_style, width=maxint, Dumper=SafeDumper, stream=stream
)
def return_string_when_no_stream(func):
@functools.wraps(func)
def wrapper(data, stream=None, **kwargs):
if stream:
return func(data, stream=stream, **kwargs)
stream = io.StringIO()
func(data, stream=stream, **kwargs)
return stream.getvalue()
return wrapper
@return_string_when_no_stream
def dump(data, stream=None, default_flow_style=False):
handler = ConfigYAML(yaml_type=YAMLType.GENERIC_YAML)
handler.default_flow_style = default_flow_style
handler.width = maxint
return handler.dump(data, stream=stream)
def file_line(mark):
@@ -220,11 +250,11 @@ def file_line(mark):
#: This is nasty but YAML doesn't give us many ways to pass arguments --
#: yaml.dump() takes a class (not an instance) and instantiates the dumper
#: itself, so we can't just pass an instance
_annotations: List[str] = []
_ANNOTATIONS: List[str] = []
class LineAnnotationDumper(OrderedLineDumper):
"""Dumper that generates per-line annotations.
class LineAnnotationRepresenter(OrderedLineRepresenter):
"""Representer that generates per-line annotations.
Annotations are stored in the ``_annotations`` global. After one
dump pass, the strings in ``_annotations`` will correspond one-to-one
@@ -240,22 +270,9 @@ class LineAnnotationDumper(OrderedLineDumper):
annotations.
"""
saved = None
def __init__(self, *args, **kwargs):
super(LineAnnotationDumper, self).__init__(*args, **kwargs)
del _annotations[:]
self.colors = "KgrbmcyGRBMCY"
self.filename_colors = {}
def process_scalar(self):
super(LineAnnotationDumper, self).process_scalar()
if marked(self.event.value):
self.saved = self.event.value
def represent_data(self, data):
"""Force syaml_str to be passed through with marks."""
result = super(LineAnnotationDumper, self).represent_data(data)
result = super().represent_data(data)
if data is None:
result.value = syaml_str("null")
elif isinstance(result.value, str):
@@ -264,10 +281,25 @@ def represent_data(self, data):
mark(result.value, data)
return result
class LineAnnotationEmitter(emitter.Emitter):
saved = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
del _ANNOTATIONS[:]
self.colors = "KgrbmcyGRBMCY"
self.filename_colors = {}
def process_scalar(self):
super().process_scalar()
if marked(self.event.value):
self.saved = self.event.value
def write_line_break(self):
super(LineAnnotationDumper, self).write_line_break()
super().write_line_break()
if self.saved is None:
_annotations.append(colorize("@K{---}"))
_ANNOTATIONS.append(colorize("@K{---}"))
return
# append annotations at the end of each line
@@ -284,37 +316,131 @@ def write_line_break(self):
ann = fmt % mark.name
if mark.line is not None:
ann += ":@c{%s}" % (mark.line + 1)
_annotations.append(colorize(ann))
_ANNOTATIONS.append(colorize(ann))
else:
_annotations.append("")
_ANNOTATIONS.append("")
def write_comment(self, comment, pre=False):
pass
def load_config(*args, **kwargs):
class YAMLType(enum.Enum):
"""YAML configurations handled by Spack"""
#: Generic YAML configuration
GENERIC_YAML = enum.auto()
#: A Spack config file with overrides
SPACK_CONFIG_FILE = enum.auto()
#: A Spack config file with line annotations
ANNOTATED_SPACK_CONFIG_FILE = enum.auto()
class ConfigYAML:
"""Handles the loading and dumping of Spack's YAML files."""
def __init__(self, yaml_type: YAMLType) -> None:
self.yaml = ruamel.yaml.YAML(typ="rt", pure=True)
if yaml_type == YAMLType.GENERIC_YAML:
self.yaml.Representer = SafeRepresenter
elif yaml_type == YAMLType.ANNOTATED_SPACK_CONFIG_FILE:
self.yaml.Representer = LineAnnotationRepresenter
self.yaml.Emitter = LineAnnotationEmitter
self.yaml.Constructor = OrderedLineConstructor
else:
self.yaml.Representer = OrderedLineRepresenter
self.yaml.Constructor = OrderedLineConstructor
def load(self, stream: IO):
"""Loads the YAML data from a stream and returns it.
Args:
stream: stream to load from.
Raises:
SpackYAMLError: if anything goes wrong while loading
"""
try:
return self.yaml.load(stream)
except error.MarkedYAMLError as e:
msg = "error parsing YAML"
error_mark = e.context_mark if e.context_mark else e.problem_mark
if error_mark:
line, column = error_mark.line, error_mark.column
msg += f": near {error_mark.name}, {str(line)}, {str(column)}"
else:
msg += f": {stream.name}"
msg += f": {e.problem}"
raise SpackYAMLError(msg, e) from e
except Exception as e:
msg = "cannot load Spack YAML configuration"
raise SpackYAMLError(msg, e) from e
def dump(self, data, stream: Optional[IO] = None, *, transform=None) -> None:
"""Dumps the YAML data to a stream.
Args:
data: data to be dumped
stream: stream to dump the data into.
Raises:
SpackYAMLError: if anything goes wrong while dumping
"""
try:
return self.yaml.dump(data, stream=stream, transform=transform)
except Exception as e:
msg = "cannot dump Spack YAML configuration"
raise SpackYAMLError(msg, str(e)) from e
def as_string(self, data) -> str:
"""Returns a string representing the YAML data passed as input."""
result = io.StringIO()
self.dump(data, stream=result)
return result.getvalue()
def deepcopy(data):
"""Returns a deepcopy of the input YAML data."""
result = copy.deepcopy(data)
if isinstance(result, comments.CommentedMap):
# HACK to fully copy ruamel CommentedMap that doesn't provide copy
# method. Especially necessary for environments
extracted_comments = extract_comments(data)
if extracted_comments:
set_comments(result, data_comments=extracted_comments)
return result
def load_config(str_or_file):
"""Load but modify the loader instance so that it will add __line__
attributes to the returned object."""
kwargs["Loader"] = OrderedLineLoader
return yaml.load(*args, **kwargs)
handler = ConfigYAML(yaml_type=YAMLType.SPACK_CONFIG_FILE)
return handler.load(str_or_file)
def load(*args, **kwargs):
return yaml.load(*args, **kwargs)
handler = ConfigYAML(yaml_type=YAMLType.GENERIC_YAML)
return handler.load(*args, **kwargs)
def dump_config(*args, **kwargs):
blame = kwargs.pop("blame", False)
@return_string_when_no_stream
def dump_config(data, stream, *, default_flow_style=False, blame=False):
if blame:
return dump_annotated(*args, **kwargs)
else:
kwargs["Dumper"] = OrderedLineDumper
return yaml.dump(*args, **kwargs)
handler = ConfigYAML(yaml_type=YAMLType.ANNOTATED_SPACK_CONFIG_FILE)
handler.yaml.default_flow_style = default_flow_style
return _dump_annotated(handler, data, stream)
handler = ConfigYAML(yaml_type=YAMLType.SPACK_CONFIG_FILE)
handler.yaml.default_flow_style = default_flow_style
return handler.dump(data, stream)
def dump_annotated(data, stream=None, *args, **kwargs):
kwargs["Dumper"] = LineAnnotationDumper
def _dump_annotated(handler, data, stream=None):
sio = io.StringIO()
yaml.dump(data, sio, *args, **kwargs)
handler.dump(data, sio)
# write_line_break() is not called by YAML for empty lines, so we
# skip empty lines here with \n+.
@@ -326,10 +452,10 @@ def dump_annotated(data, stream=None, *args, **kwargs):
getvalue = stream.getvalue
# write out annotations and lines, accounting for color
width = max(clen(a) for a in _annotations)
formats = ["%%-%ds %%s\n" % (width + cextra(a)) for a in _annotations]
width = max(clen(a) for a in _ANNOTATIONS)
formats = ["%%-%ds %%s\n" % (width + cextra(a)) for a in _ANNOTATIONS]
for f, a, l in zip(formats, _annotations, lines):
for f, a, l in zip(formats, _ANNOTATIONS, lines):
stream.write(f % (a, l))
if getvalue:
@@ -352,8 +478,23 @@ def sorted_dict(dict_like):
return result
def extract_comments(data):
"""Extract and returns comments from some YAML data"""
return getattr(data, comments.Comment.attrib, None)
def set_comments(data, *, data_comments):
"""Set comments on some YAML data"""
return setattr(data, comments.Comment.attrib, data_comments)
def name_mark(name):
"""Returns a mark with just a name"""
return error.StringMark(name, None, None, None, None, None)
class SpackYAMLError(spack.error.SpackError):
"""Raised when there are issues with YAML parsing."""
def __init__(self, msg, yaml_error):
super(SpackYAMLError, self).__init__(msg, str(yaml_error))
super().__init__(msg, str(yaml_error))