diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index ab18ed507e4..c15bb5206ac 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -36,6 +36,8 @@ import sys from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union +import jsonschema + from llnl.util import filesystem, lang, tty import spack.error @@ -1048,8 +1050,6 @@ def validate( This leverages the line information (start_mark, end_mark) stored on Spack YAML structures. """ - import jsonschema - try: spack.schema.Validator(schema).validate(data) except jsonschema.ValidationError as e: diff --git a/lib/spack/spack/container/__init__.py b/lib/spack/spack/container/__init__.py index cbeb5974a97..2b2d89d5813 100644 --- a/lib/spack/spack/container/__init__.py +++ b/lib/spack/spack/container/__init__.py @@ -6,6 +6,8 @@ """ import warnings +import jsonschema + import spack.environment as ev import spack.schema.env as env import spack.util.spack_yaml as syaml @@ -30,8 +32,6 @@ def validate(configuration_file): Returns: A sanitized copy of the configuration stored in the input file """ - import jsonschema - with open(configuration_file, encoding="utf-8") as f: config = syaml.load(f) diff --git a/lib/spack/spack/container/writers.py b/lib/spack/spack/container/writers.py index 92fddd7b86f..e521527be6c 100644 --- a/lib/spack/spack/container/writers.py +++ b/lib/spack/spack/container/writers.py @@ -9,6 +9,8 @@ from collections import namedtuple from typing import Optional +import jsonschema + import spack.environment as ev import spack.error import spack.schema.env @@ -188,8 +190,6 @@ def paths(self): @tengine.context_property def manifest(self): """The spack.yaml file that should be used in the image""" - import jsonschema - # Copy in the part of spack.yaml prescribed in the configuration file manifest = copy.deepcopy(self.config) manifest.pop("container") diff --git a/lib/spack/spack/schema/__init__.py b/lib/spack/spack/schema/__init__.py index 298d4ef7c52..83e8c06b246 100644 --- a/lib/spack/spack/schema/__init__.py +++ b/lib/spack/spack/schema/__init__.py @@ -6,6 +6,8 @@ import typing import warnings +import jsonschema + import llnl.util.lang from spack.error import SpecSyntaxError @@ -19,12 +21,8 @@ class DeprecationMessage(typing.NamedTuple): # jsonschema is imported lazily as it is heavy to import # and increases the start-up time def _make_validator(): - import jsonschema - def _validate_spec(validator, is_spec, instance, schema): """Check if the attributes on instance are valid specs.""" - import jsonschema - import spack.spec_parser if not validator.is_type(instance, "object"): @@ -33,8 +31,8 @@ def _validate_spec(validator, is_spec, instance, schema): for spec_str in instance: try: spack.spec_parser.parse(spec_str) - except SpecSyntaxError as e: - yield jsonschema.ValidationError(str(e)) + except SpecSyntaxError: + yield jsonschema.ValidationError(f"the key '{spec_str}' is not a valid spec") def _deprecated_properties(validator, deprecated, instance, schema): if not (validator.is_type(instance, "object") or validator.is_type(instance, "array")): @@ -67,7 +65,7 @@ def _deprecated_properties(validator, deprecated, instance, schema): yield jsonschema.ValidationError("\n".join(errors)) return jsonschema.validators.extend( - jsonschema.Draft4Validator, + jsonschema.Draft7Validator, {"validate_spec": _validate_spec, "deprecatedProperties": _deprecated_properties}, ) diff --git a/lib/spack/spack/schema/definitions.py b/lib/spack/spack/schema/definitions.py index 504284b04ba..e4d65cd9b8e 100644 --- a/lib/spack/spack/schema/definitions.py +++ b/lib/spack/spack/schema/definitions.py @@ -19,7 +19,7 @@ "items": { "type": "object", "properties": {"when": {"type": "string"}}, - "patternProperties": {r"^(?!when$)\w*": spec_list_schema}, + "additionalProperties": spec_list_schema, }, } } diff --git a/lib/spack/spack/schema/mirrors.py b/lib/spack/spack/schema/mirrors.py index b0844ed87b2..45e1f5adbf9 100644 --- a/lib/spack/spack/schema/mirrors.py +++ b/lib/spack/spack/schema/mirrors.py @@ -9,6 +9,8 @@ """ from typing import Any, Dict +import jsonschema + #: Common properties for connection specification connection = { "url": {"type": "string"}, @@ -102,8 +104,6 @@ def update(data): - import jsonschema - errors = [] def check_access_pair(name, section): diff --git a/lib/spack/spack/schema/modules.py b/lib/spack/spack/schema/modules.py index 415273b40b0..2ea8d0184a9 100644 --- a/lib/spack/spack/schema/modules.py +++ b/lib/spack/spack/schema/modules.py @@ -12,22 +12,6 @@ import spack.schema.environment import spack.schema.projections -#: Matches a spec or a multi-valued variant but not another -#: valid keyword. -#: -#: THIS NEEDS TO BE UPDATED FOR EVERY NEW KEYWORD THAT -#: IS ADDED IMMEDIATELY BELOW THE MODULE TYPE ATTRIBUTE -spec_regex = ( - r"(?!hierarchy|core_specs|verbose|hash_length|defaults|filter_hierarchy_specs|hide|" - r"include|exclude|projections|naming_scheme|core_compilers|all)(^\w[\w-]*)" -) - -#: Matches a valid name for a module set -valid_module_set_name = r"^(?!prefix_inspections$)\w[\w-]*$" - -#: Matches an anonymous spec, i.e. a spec without a root name -anonymous_spec_regex = r"^[\^@%+~]" - #: Definitions for parts of module schema array_of_strings = {"type": "array", "default": [], "items": {"type": "string"}} @@ -56,7 +40,7 @@ "suffixes": { "type": "object", "validate_spec": True, - "patternProperties": {r"\w[\w-]*": {"type": "string"}}, # key + "additionalProperties": {"type": "string"}, # key }, "environment": spack.schema.environment.definition, }, @@ -64,34 +48,40 @@ projections_scheme = spack.schema.projections.properties["projections"] -module_type_configuration = { +module_type_configuration: Dict = { "type": "object", "default": {}, - "allOf": [ - { - "properties": { - "verbose": {"type": "boolean", "default": False}, - "hash_length": {"type": "integer", "minimum": 0, "default": 7}, - "include": array_of_strings, - "exclude": array_of_strings, - "exclude_implicits": {"type": "boolean", "default": False}, - "defaults": array_of_strings, - "hide_implicits": {"type": "boolean", "default": False}, - "naming_scheme": {"type": "string"}, # Can we be more specific here? - "projections": projections_scheme, - "all": module_file_configuration, - } - }, - { - "validate_spec": True, - "patternProperties": { - spec_regex: module_file_configuration, - anonymous_spec_regex: module_file_configuration, - }, - }, - ], + "validate_spec": True, + "properties": { + "verbose": {"type": "boolean", "default": False}, + "hash_length": {"type": "integer", "minimum": 0, "default": 7}, + "include": array_of_strings, + "exclude": array_of_strings, + "exclude_implicits": {"type": "boolean", "default": False}, + "defaults": array_of_strings, + "hide_implicits": {"type": "boolean", "default": False}, + "naming_scheme": {"type": "string"}, + "projections": projections_scheme, + "all": module_file_configuration, + }, + "additionalProperties": module_file_configuration, } +tcl_configuration = module_type_configuration.copy() + +lmod_configuration = module_type_configuration.copy() +lmod_configuration["properties"].update( + { + "core_compilers": array_of_strings, + "hierarchy": array_of_strings, + "core_specs": array_of_strings, + "filter_hierarchy_specs": { + "type": "object", + "validate_spec": True, + "additionalProperties": array_of_strings, + }, + } +) module_config_properties = { "use_view": {"anyOf": [{"type": "string"}, {"type": "boolean"}]}, @@ -105,31 +95,8 @@ "default": [], "items": {"type": "string", "enum": ["tcl", "lmod"]}, }, - "lmod": { - "allOf": [ - # Base configuration - module_type_configuration, - { - "type": "object", - "properties": { - "core_compilers": array_of_strings, - "hierarchy": array_of_strings, - "core_specs": array_of_strings, - "filter_hierarchy_specs": { - "type": "object", - "patternProperties": {spec_regex: array_of_strings}, - }, - }, - }, # Specific lmod extensions - ] - }, - "tcl": { - "allOf": [ - # Base configuration - module_type_configuration, - {}, # Specific tcl extensions - ] - }, + "lmod": lmod_configuration, + "tcl": tcl_configuration, "prefix_inspections": { "type": "object", "additionalProperties": False, @@ -145,7 +112,6 @@ properties: Dict[str, Any] = { "modules": { "type": "object", - "additionalProperties": False, "properties": { "prefix_inspections": { "type": "object", @@ -156,13 +122,11 @@ }, } }, - "patternProperties": { - valid_module_set_name: { - "type": "object", - "default": {}, - "additionalProperties": False, - "properties": module_config_properties, - } + "additionalProperties": { + "type": "object", + "default": {}, + "additionalProperties": False, + "properties": module_config_properties, }, } } diff --git a/lib/spack/spack/schema/packages.py b/lib/spack/spack/schema/packages.py index e16ba1f14bb..8c92b1f89eb 100644 --- a/lib/spack/spack/schema/packages.py +++ b/lib/spack/spack/schema/packages.py @@ -98,7 +98,6 @@ "packages": { "type": "object", "default": {}, - "additionalProperties": False, "properties": { "all": { # package name "type": "object", @@ -140,58 +139,54 @@ }, } }, - "patternProperties": { - r"(?!^all$)(^\w[\w-]*)": { # package name - "type": "object", - "default": {}, - "additionalProperties": False, - "properties": { - "require": requirements, - "prefer": prefer_and_conflict, - "conflict": prefer_and_conflict, - "version": { - "type": "array", - "default": [], - # version strings - "items": {"anyOf": [{"type": "string"}, {"type": "number"}]}, - }, - "buildable": {"type": "boolean", "default": True}, - "permissions": permissions, - # If 'get_full_repo' is promoted to a Package-level - # attribute, it could be useful to set it here - "package_attributes": package_attributes, - "variants": variants, - "externals": { - "type": "array", - "items": { - "type": "object", - "properties": { - "spec": {"type": "string"}, - "prefix": {"type": "string"}, - "modules": {"type": "array", "items": {"type": "string"}}, - "extra_attributes": { - "type": "object", - "additionalProperties": True, - "properties": { - "compilers": { - "type": "object", - "patternProperties": { - r"(^\w[\w-]*)": {"type": "string"} - }, - }, - "environment": spack.schema.environment.definition, - "extra_rpaths": extra_rpaths, - "implicit_rpaths": implicit_rpaths, - "flags": flags, + "additionalProperties": { # package name + "type": "object", + "default": {}, + "additionalProperties": False, + "properties": { + "require": requirements, + "prefer": prefer_and_conflict, + "conflict": prefer_and_conflict, + "version": { + "type": "array", + "default": [], + # version strings + "items": {"anyOf": [{"type": "string"}, {"type": "number"}]}, + }, + "buildable": {"type": "boolean", "default": True}, + "permissions": permissions, + # If 'get_full_repo' is promoted to a Package-level + # attribute, it could be useful to set it here + "package_attributes": package_attributes, + "variants": variants, + "externals": { + "type": "array", + "items": { + "type": "object", + "properties": { + "spec": {"type": "string"}, + "prefix": {"type": "string"}, + "modules": {"type": "array", "items": {"type": "string"}}, + "extra_attributes": { + "type": "object", + "additionalProperties": {"type": "string"}, + "properties": { + "compilers": { + "type": "object", + "patternProperties": {r"(^\w[\w-]*)": {"type": "string"}}, }, + "environment": spack.schema.environment.definition, + "extra_rpaths": extra_rpaths, + "implicit_rpaths": implicit_rpaths, + "flags": flags, }, }, - "additionalProperties": True, - "required": ["spec"], }, + "additionalProperties": True, + "required": ["spec"], }, }, - } + }, }, } } diff --git a/lib/spack/spack/test/schema.py b/lib/spack/spack/test/schema.py index 010afde80c8..fcbe2eb59de 100644 --- a/lib/spack/spack/test/schema.py +++ b/lib/spack/spack/test/schema.py @@ -64,7 +64,7 @@ def test_validate_spec(validate_spec_schema): # Check that invalid data throws data["^python@3.7@"] = "baz" - with pytest.raises(jsonschema.ValidationError, match="unexpected characters"): + with pytest.raises(jsonschema.ValidationError, match="is not a valid spec"): v.validate(data) @@ -73,7 +73,7 @@ def test_module_suffixes(module_suffixes_schema): v = spack.schema.Validator(module_suffixes_schema) data = {"tcl": {"all": {"suffixes": {"^python@2.7@": "py2.7"}}}} - with pytest.raises(jsonschema.ValidationError, match="unexpected characters"): + with pytest.raises(jsonschema.ValidationError, match="is not a valid spec"): v.validate(data)