zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

jsonschema: use draft7 validator and simplify schemas (#48621)

Browse Source
This commit is contained in:
Massimiliano Culpo 2025-01-20 09:51:29 +01:00 committed by GitHub
parent a842332b1b
commit 783eccfbd5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 96 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/spack/spack/config.py
View File

@ -36,6 +36,8 @@
import sys import sys
from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
import jsonschema
from llnl.util import filesystem, lang, tty from llnl.util import filesystem, lang, tty
import spack.error import spack.error
@ -1048,8 +1050,6 @@ def validate(
This leverages the line information (start_mark, end_mark) stored This leverages the line information (start_mark, end_mark) stored
on Spack YAML structures. on Spack YAML structures.
""" """
import jsonschema
try: try:
spack.schema.Validator(schema).validate(data) spack.schema.Validator(schema).validate(data)
except jsonschema.ValidationError as e: except jsonschema.ValidationError as e:

4
lib/spack/spack/container/__init__.py
View File

@ -6,6 +6,8 @@
""" """
import warnings import warnings
import jsonschema
import spack.environment as ev import spack.environment as ev
import spack.schema.env as env import spack.schema.env as env
import spack.util.spack_yaml as syaml import spack.util.spack_yaml as syaml
@ -30,8 +32,6 @@ def validate(configuration_file):
Returns: Returns:
A sanitized copy of the configuration stored in the input file A sanitized copy of the configuration stored in the input file
""" """
import jsonschema
with open(configuration_file, encoding="utf-8") as f: with open(configuration_file, encoding="utf-8") as f:
config = syaml.load(f) config = syaml.load(f)

4
lib/spack/spack/container/writers.py
View File

@ -9,6 +9,8 @@
from collections import namedtuple from collections import namedtuple
from typing import Optional from typing import Optional
import jsonschema
import spack.environment as ev import spack.environment as ev
import spack.error import spack.error
import spack.schema.env import spack.schema.env
@ -188,8 +190,6 @@ def paths(self):
@tengine.context_property @tengine.context_property
def manifest(self): def manifest(self):
"""The spack.yaml file that should be used in the image""" """The spack.yaml file that should be used in the image"""
import jsonschema
# Copy in the part of spack.yaml prescribed in the configuration file # Copy in the part of spack.yaml prescribed in the configuration file
manifest = copy.deepcopy(self.config) manifest = copy.deepcopy(self.config)
manifest.pop("container") manifest.pop("container")

12
lib/spack/spack/schema/__init__.py
View File

@ -6,6 +6,8 @@
import typing import typing
import warnings import warnings
import jsonschema
import llnl.util.lang import llnl.util.lang
from spack.error import SpecSyntaxError from spack.error import SpecSyntaxError
@ -19,12 +21,8 @@ class DeprecationMessage(typing.NamedTuple):
# jsonschema is imported lazily as it is heavy to import # jsonschema is imported lazily as it is heavy to import
# and increases the start-up time # and increases the start-up time
def _make_validator(): def _make_validator():
import jsonschema
def _validate_spec(validator, is_spec, instance, schema): def _validate_spec(validator, is_spec, instance, schema):
"""Check if the attributes on instance are valid specs.""" """Check if the attributes on instance are valid specs."""
import jsonschema
import spack.spec_parser import spack.spec_parser
if not validator.is_type(instance, "object"): if not validator.is_type(instance, "object"):
@ -33,8 +31,8 @@ def _validate_spec(validator, is_spec, instance, schema):
for spec_str in instance: for spec_str in instance:
try: try:
spack.spec_parser.parse(spec_str) spack.spec_parser.parse(spec_str)
except SpecSyntaxError as e: except SpecSyntaxError:
yield jsonschema.ValidationError(str(e)) yield jsonschema.ValidationError(f"the key '{spec_str}' is not a valid spec")
def _deprecated_properties(validator, deprecated, instance, schema): def _deprecated_properties(validator, deprecated, instance, schema):
if not (validator.is_type(instance, "object") or validator.is_type(instance, "array")): if not (validator.is_type(instance, "object") or validator.is_type(instance, "array")):
@ -67,7 +65,7 @@ def _deprecated_properties(validator, deprecated, instance, schema):
yield jsonschema.ValidationError("\n".join(errors)) yield jsonschema.ValidationError("\n".join(errors))
return jsonschema.validators.extend( return jsonschema.validators.extend(
jsonschema.Draft4Validator, jsonschema.Draft7Validator,
{"validate_spec": _validate_spec, "deprecatedProperties": _deprecated_properties}, {"validate_spec": _validate_spec, "deprecatedProperties": _deprecated_properties},
) )

2
lib/spack/spack/schema/definitions.py
View File

@ -19,7 +19,7 @@
"items": { "items": {
"type": "object", "type": "object",
"properties": {"when": {"type": "string"}}, "properties": {"when": {"type": "string"}},
"patternProperties": {r"^(?!when$)\w*": spec_list_schema}, "additionalProperties": spec_list_schema,
}, },
} }
} }

4
lib/spack/spack/schema/mirrors.py
View File

@ -9,6 +9,8 @@
""" """
from typing import Any, Dict from typing import Any, Dict
import jsonschema
#: Common properties for connection specification #: Common properties for connection specification
connection = { connection = {
"url": {"type": "string"}, "url": {"type": "string"},
@ -102,8 +104,6 @@
def update(data): def update(data):
import jsonschema
errors = [] errors = []
def check_access_pair(name, section): def check_access_pair(name, section):

112
lib/spack/spack/schema/modules.py
View File

@ -12,22 +12,6 @@
import spack.schema.environment import spack.schema.environment
import spack.schema.projections import spack.schema.projections
#: Matches a spec or a multi-valued variant but not another
#: valid keyword.
#:
#: THIS NEEDS TO BE UPDATED FOR EVERY NEW KEYWORD THAT
#: IS ADDED IMMEDIATELY BELOW THE MODULE TYPE ATTRIBUTE
spec_regex = (
r"(?!hierarchy|core_specs|verbose|hash_length|defaults|filter_hierarchy_specs|hide|"
r"include|exclude|projections|naming_scheme|core_compilers|all)(^\w[\w-]*)"
)
#: Matches a valid name for a module set
valid_module_set_name = r"^(?!prefix_inspections$)\w[\w-]*$"
#: Matches an anonymous spec, i.e. a spec without a root name
anonymous_spec_regex = r"^[\^@%+~]"
#: Definitions for parts of module schema #: Definitions for parts of module schema
array_of_strings = {"type": "array", "default": [], "items": {"type": "string"}} array_of_strings = {"type": "array", "default": [], "items": {"type": "string"}}
@ -56,7 +40,7 @@
"suffixes": { "suffixes": {
"type": "object", "type": "object",
"validate_spec": True, "validate_spec": True,
"patternProperties": {r"\w[\w-]*": {"type": "string"}}, # key "additionalProperties": {"type": "string"}, # key
}, },
"environment": spack.schema.environment.definition, "environment": spack.schema.environment.definition,
}, },
@ -64,34 +48,40 @@
projections_scheme = spack.schema.projections.properties["projections"] projections_scheme = spack.schema.projections.properties["projections"]
module_type_configuration = { module_type_configuration: Dict = {
"type": "object", "type": "object",
"default": {}, "default": {},
"allOf": [ "validate_spec": True,
{ "properties": {
"properties": { "verbose": {"type": "boolean", "default": False},
"verbose": {"type": "boolean", "default": False}, "hash_length": {"type": "integer", "minimum": 0, "default": 7},
"hash_length": {"type": "integer", "minimum": 0, "default": 7}, "include": array_of_strings,
"include": array_of_strings, "exclude": array_of_strings,
"exclude": array_of_strings, "exclude_implicits": {"type": "boolean", "default": False},
"exclude_implicits": {"type": "boolean", "default": False}, "defaults": array_of_strings,
"defaults": array_of_strings, "hide_implicits": {"type": "boolean", "default": False},
"hide_implicits": {"type": "boolean", "default": False}, "naming_scheme": {"type": "string"},
"naming_scheme": {"type": "string"}, # Can we be more specific here? "projections": projections_scheme,
"projections": projections_scheme, "all": module_file_configuration,
"all": module_file_configuration, },
} "additionalProperties": module_file_configuration,
},
{
"validate_spec": True,
"patternProperties": {
spec_regex: module_file_configuration,
anonymous_spec_regex: module_file_configuration,
},
},
],
} }
tcl_configuration = module_type_configuration.copy()
lmod_configuration = module_type_configuration.copy()
lmod_configuration["properties"].update(
{
"core_compilers": array_of_strings,
"hierarchy": array_of_strings,
"core_specs": array_of_strings,
"filter_hierarchy_specs": {
"type": "object",
"validate_spec": True,
"additionalProperties": array_of_strings,
},
}
)
module_config_properties = { module_config_properties = {
"use_view": {"anyOf": [{"type": "string"}, {"type": "boolean"}]}, "use_view": {"anyOf": [{"type": "string"}, {"type": "boolean"}]},
@ -105,31 +95,8 @@
"default": [], "default": [],
"items": {"type": "string", "enum": ["tcl", "lmod"]}, "items": {"type": "string", "enum": ["tcl", "lmod"]},
}, },
"lmod": { "lmod": lmod_configuration,
"allOf": [ "tcl": tcl_configuration,
# Base configuration
module_type_configuration,
{
"type": "object",
"properties": {
"core_compilers": array_of_strings,
"hierarchy": array_of_strings,
"core_specs": array_of_strings,
"filter_hierarchy_specs": {
"type": "object",
"patternProperties": {spec_regex: array_of_strings},
},
},
}, # Specific lmod extensions
]
},
"tcl": {
"allOf": [
# Base configuration
module_type_configuration,
{}, # Specific tcl extensions
]
},
"prefix_inspections": { "prefix_inspections": {
"type": "object", "type": "object",
"additionalProperties": False, "additionalProperties": False,
@ -145,7 +112,6 @@
properties: Dict[str, Any] = { properties: Dict[str, Any] = {
"modules": { "modules": {
"type": "object", "type": "object",
"additionalProperties": False,
"properties": { "properties": {
"prefix_inspections": { "prefix_inspections": {
"type": "object", "type": "object",
@ -156,13 +122,11 @@
}, },
} }
}, },
"patternProperties": { "additionalProperties": {
valid_module_set_name: { "type": "object",
"type": "object", "default": {},
"default": {}, "additionalProperties": False,
"additionalProperties": False, "properties": module_config_properties,
"properties": module_config_properties,
}
}, },
} }
} }

89
lib/spack/spack/schema/packages.py
View File

@ -98,7 +98,6 @@
"packages": { "packages": {
"type": "object", "type": "object",
"default": {}, "default": {},
"additionalProperties": False,
"properties": { "properties": {
"all": { # package name "all": { # package name
"type": "object", "type": "object",
@ -140,58 +139,54 @@
}, },
} }
}, },
"patternProperties": { "additionalProperties": { # package name
r"(?!^all$)(^\w[\w-]*)": { # package name "type": "object",
"type": "object", "default": {},
"default": {}, "additionalProperties": False,
"additionalProperties": False, "properties": {
"properties": { "require": requirements,
"require": requirements, "prefer": prefer_and_conflict,
"prefer": prefer_and_conflict, "conflict": prefer_and_conflict,
"conflict": prefer_and_conflict, "version": {
"version": { "type": "array",
"type": "array", "default": [],
"default": [], # version strings
# version strings "items": {"anyOf": [{"type": "string"}, {"type": "number"}]},
"items": {"anyOf": [{"type": "string"}, {"type": "number"}]}, },
}, "buildable": {"type": "boolean", "default": True},
"buildable": {"type": "boolean", "default": True}, "permissions": permissions,
"permissions": permissions, # If 'get_full_repo' is promoted to a Package-level
# If 'get_full_repo' is promoted to a Package-level # attribute, it could be useful to set it here
# attribute, it could be useful to set it here "package_attributes": package_attributes,
"package_attributes": package_attributes, "variants": variants,
"variants": variants, "externals": {
"externals": { "type": "array",
"type": "array", "items": {
"items": { "type": "object",
"type": "object", "properties": {
"properties": { "spec": {"type": "string"},
"spec": {"type": "string"}, "prefix": {"type": "string"},
"prefix": {"type": "string"}, "modules": {"type": "array", "items": {"type": "string"}},
"modules": {"type": "array", "items": {"type": "string"}}, "extra_attributes": {
"extra_attributes": { "type": "object",
"type": "object", "additionalProperties": {"type": "string"},
"additionalProperties": True, "properties": {
"properties": { "compilers": {
"compilers": { "type": "object",
"type": "object", "patternProperties": {r"(^\w[\w-]*)": {"type": "string"}},
"patternProperties": {
r"(^\w[\w-]*)": {"type": "string"}
},
},
"environment": spack.schema.environment.definition,
"extra_rpaths": extra_rpaths,
"implicit_rpaths": implicit_rpaths,
"flags": flags,
}, },
"environment": spack.schema.environment.definition,
"extra_rpaths": extra_rpaths,
"implicit_rpaths": implicit_rpaths,
"flags": flags,
}, },
}, },
"additionalProperties": True,
"required": ["spec"],
}, },
"additionalProperties": True,
"required": ["spec"],
}, },
}, },
} },
}, },
} }
} }

4
lib/spack/spack/test/schema.py
View File

@ -64,7 +64,7 @@ def test_validate_spec(validate_spec_schema):
# Check that invalid data throws # Check that invalid data throws
data["^python@3.7@"] = "baz" data["^python@3.7@"] = "baz"
with pytest.raises(jsonschema.ValidationError, match="unexpected characters"): with pytest.raises(jsonschema.ValidationError, match="is not a valid spec"):
v.validate(data) v.validate(data)
@ -73,7 +73,7 @@ def test_module_suffixes(module_suffixes_schema):
v = spack.schema.Validator(module_suffixes_schema) v = spack.schema.Validator(module_suffixes_schema)
data = {"tcl": {"all": {"suffixes": {"^python@2.7@": "py2.7"}}}} data = {"tcl": {"all": {"suffixes": {"^python@2.7@": "py2.7"}}}}
with pytest.raises(jsonschema.ValidationError, match="unexpected characters"): with pytest.raises(jsonschema.ValidationError, match="is not a valid spec"):
v.validate(data) v.validate(data)