schemas: rework schemas so that they can be included from other files

- Each schema now has a top-level `properties` and `schema` attribute.

- The `properties` is a fragment that can be included in other
  jsonschemas, via Python, not via '$ref'

- Th `schema` is a complete `jsonschema` with `title` and `$schema`
  properties.

lib/spack/spack/config.py

@@ -51,7 +51,12 @@
 
 import spack.paths
 import spack.architecture
-import spack.schema
+import spack.schema.compilers
+import spack.schema.mirrors
+import spack.schema.repos
+import spack.schema.packages
+import spack.schema.modules
+import spack.schema.config
 from spack.error import SpackError
 
 # Hacked yaml for configuration files preserves line numbers.
@@ -175,7 +180,7 @@ def write_section(self, section):
         try:
             mkdirp(self.path)
             with open(filename, 'w') as f:
-                _validate_section(data, section_schemas[section])
+                _validate(data, section_schemas[section])
                 syaml.dump(data, stream=f, default_flow_style=False)
         except (yaml.YAMLError, IOError) as e:
             raise ConfigFileError(
@@ -216,7 +221,7 @@ def __init__(self, name, data=None):
         if data:
             for section in data:
                 dsec = data[section]
-                _validate_section({section: dsec}, section_schemas[section])
+                _validate({section: dsec}, section_schemas[section])
                 self.sections[section] = _mark_internal(
                     syaml.syaml_dict({section: dsec}), name)
 
@@ -234,7 +239,7 @@ def write_section(self, section):
         """This only validates, as the data is already in memory."""
         data = self.get_section(section)
         if data is not None:
-            _validate_section(data, section_schemas[section])
+            _validate(data, section_schemas[section])
         self.sections[section] = _mark_internal(data, self.name)
 
     def __repr__(self):
@@ -585,7 +590,7 @@ def _validate_section_name(section):
             % (section, " ".join(section_schemas.keys())))
 
 
-def _validate_section(data, schema):
+def _validate(data, schema):
     """Validate data read in from a Spack YAML file.
 
     This leverages the line information (start_mark, end_mark) stored
@@ -593,13 +598,13 @@ def _validate_section(data, schema):
 
     """
     import jsonschema
-    if not hasattr(_validate_section, 'validator'):
+    if not hasattr(_validate, 'validator'):
         default_setting_validator = _extend_with_default(
             jsonschema.Draft4Validator)
-        _validate_section.validator = default_setting_validator
+        _validate.validator = default_setting_validator
 
     try:
-        _validate_section.validator(schema).validate(data)
+        _validate.validator(schema).validate(data)
     except jsonschema.ValidationError as e:
         raise ConfigFormatError(e, data)
 
@@ -623,7 +628,7 @@ def _read_config_file(filename, schema):
             data = _mark_overrides(syaml.load(f))
 
         if data:
-            _validate_section(data, schema)
+            _validate(data, schema)
         return data
 
     except MarkedYAMLError as e:

lib/spack/spack/environment.py

@@ -85,7 +85,6 @@ def deactivate():
         return
 
 
-
 def root(name):
     """Get the root directory for an environment by name."""
     return fs.join_path(env_path, name)

lib/spack/spack/schema/__init__.py

@@ -3,12 +3,4 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
-"""This module contains jsonschema files for all of Spack's YAML formats.
+"""This module contains jsonschema files for all of Spack's YAML formats."""
-"""
-from llnl.util.lang import list_modules
-
-# Automatically bring in all sub-modules
-__all__ = []
-for mod in list_modules(__path__[0]):
-    __import__('%s.%s' % (__name__, mod))
-    __all__.append(mod)

lib/spack/spack/schema/compilers.py

@@ -10,15 +10,11 @@
 """
 
 
-schema = {
+#: Properties for inclusion in other schemas
-    '$schema': 'http://json-schema.org/schema#',
+properties = {
-    'title': 'Spack compiler configuration file schema',
-    'type': 'object',
-    'additionalProperties': False,
-    'properties': {
     'compilers': {
         'type': 'array',
-            'items': {
+        'items': [{
             'type': 'object',
             'additionalProperties': False,
             'properties': {
@@ -117,7 +113,16 @@
                     }
                 }
             }
+        }]
     }
 }
-    }
+
+
+#: Full schema with metadata
+schema = {
+    '$schema': 'http://json-schema.org/schema#',
+    'title': 'Spack compiler configuration file schema',
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': properties,
 }

lib/spack/spack/schema/config.py

@@ -10,12 +10,8 @@
 """
 
 
-schema = {
+#: Properties for inclusion in other schemas
-    '$schema': 'http://json-schema.org/schema#',
+properties = {
-    'title': 'Spack core configuration file schema',
-    'type': 'object',
-    'additionalProperties': False,
-    'patternProperties': {
     'config': {
         'type': 'object',
         'default': {},
@@ -59,7 +55,16 @@
                     {'type': 'null'}
                 ],
             },
-            }
         },
     },
 }
+
+
+#: Full schema with metadata
+schema = {
+    '$schema': 'http://json-schema.org/schema#',
+    'title': 'Spack core configuration file schema',
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': properties,
+}

lib/spack/spack/schema/env.py

@@ -12,19 +12,22 @@
 
 schema = {
     '$schema': 'http://json-schema.org/schema#',
-    'title': 'Spack Environments user configuration file schema',
+    'title': 'Spack environment file schema',
     'type': 'object',
     'additionalProperties': False,
-    'properties': {
+    'patternProperties': {
-        'env': {
+        '^env|spack$': {
             'type': 'object',
             'default': {},
             'properties': {
-                'configs': {
+                'include': {
                     'type': 'array',
-                    'default': [],
+                    'items': {
-                    'items': {'type': 'string'}
+                        'type': 'string'
                     },
+                },
+
+
                 'specs': {
                     'type': 'object',
                     'default': {},

lib/spack/spack/schema/mirrors.py

@@ -10,20 +10,24 @@
 """
 
 
+#: Properties for inclusion in other schemas
+properties = {
+    'mirrors': {
+        'type': 'object',
+        'default': {},
+        'additionalProperties': False,
+        'patternProperties': {
+            r'\w[\w-]*': {'type': 'string'},
+        },
+    },
+}
+
+
+#: Full schema with metadata
 schema = {
     '$schema': 'http://json-schema.org/schema#',
     'title': 'Spack mirror configuration file schema',
     'type': 'object',
     'additionalProperties': False,
-    'patternProperties': {
+    'properties': properties,
-        r'mirrors': {
-            'type': 'object',
-            'default': {},
-            'additionalProperties': False,
-            'patternProperties': {
-                r'\w[\w-]*': {
-                    'type': 'string'},
-            },
-        },
-    },
 }

lib/spack/spack/schema/modules.py

@@ -10,12 +10,8 @@
 """
 
 
-schema = {
+#: Definitions for parts of module schema
-    '$schema': 'http://json-schema.org/schema#',
+definitions = {
-    'title': 'Spack module file configuration file schema',
-    'type': 'object',
-    'additionalProperties': False,
-    'definitions': {
     'array_of_strings': {
         'type': 'array',
         'default': [],
@@ -117,9 +113,12 @@
             }}
         ]
     }
-    },
+}
-    'patternProperties': {
+
-        r'modules': {
+
+# Properties for inclusion into other schemas (requires definitions)
+properties = {
+    'modules': {
         'type': 'object',
         'default': {},
         'additionalProperties': False,
@@ -148,20 +147,33 @@
                             '$ref': '#/definitions/array_of_strings'
                         }
                     }  # Specific lmod extensions
-                    ]},
+                ]
+            },
             'tcl': {
                 'allOf': [
                     # Base configuration
                     {'$ref': '#/definitions/module_type_configuration'},
                     {}  # Specific tcl extensions
-                    ]},
+                ]
+            },
             'dotkit': {
                 'allOf': [
                     # Base configuration
                     {'$ref': '#/definitions/module_type_configuration'},
                     {}  # Specific dotkit extensions
-                    ]},
+                ]
-            }
+            },
         },
     },
 }
+
+
+#: Full schema with metadata
+schema = {
+    '$schema': 'http://json-schema.org/schema#',
+    'title': 'Spack module file configuration file schema',
+    'definitions': definitions,
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': properties,
+}

lib/spack/spack/schema/packages.py

@@ -10,13 +10,9 @@
 """
 
 
-schema = {
+#: Properties for inclusion in other schemas
-    '$schema': 'http://json-schema.org/schema#',
+properties = {
-    'title': 'Spack package configuration file schema',
+    'packages': {
-    'type': 'object',
-    'additionalProperties': False,
-    'patternProperties': {
-        r'packages': {
         'type': 'object',
         'default': {},
         'additionalProperties': False,
@@ -84,5 +80,14 @@
             },
         },
     },
-    },
+}
+
+
+#: Full schema with metadata
+schema = {
+    '$schema': 'http://json-schema.org/schema#',
+    'title': 'Spack package configuration file schema',
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': properties,
 }

lib/spack/spack/schema/repos.py

@@ -10,17 +10,21 @@
 """
 
 
+#: Properties for inclusion in other schemas
+properties = {
+    'repos': {
+        'type': 'array',
+        'default': [],
+        'items': {'type': 'string'},
+    },
+}
+
+
+#: Full schema with metadata
 schema = {
     '$schema': 'http://json-schema.org/schema#',
     'title': 'Spack repository configuration file schema',
     'type': 'object',
     'additionalProperties': False,
-    'patternProperties': {
+    'properties': properties,
-        r'repos': {
-            'type': 'array',
-            'default': [],
-            'items': {
-                'type': 'string'},
-        },
-    },
 }

lib/spack/spack/test/config.py

@@ -7,6 +7,7 @@
 import collections
 import getpass
 import tempfile
+from six import StringIO
 
 from llnl.util.filesystem import touch, mkdirp
 
@@ -15,6 +16,12 @@
 
 import spack.paths
 import spack.config
+import spack.schema.compilers
+import spack.schema.config
+import spack.schema.packages
+import spack.schema.mirrors
+import spack.schema.repos
+import spack.util.spack_yaml as syaml
 from spack.util.path import canonicalize_path
 
 
@@ -631,7 +638,7 @@ def test_add_command_line_scopes(tmpdir, mutable_config):
     spack.config._add_command_line_scopes(mutable_config, [str(tmpdir)])
 
 
-def test_immuntable_scope(tmpdir):
+def test_immutable_scope(tmpdir):
     config_yaml = str(tmpdir.join('config.yaml'))
     with open(config_yaml, 'w') as f:
         f.write("""\
@@ -645,3 +652,58 @@ def test_immuntable_scope(tmpdir):
 
     with pytest.raises(spack.config.ConfigError):
         scope.write_section('config')
+
+
+def check_schema(name, file_contents):
+    """Check a Spack YAML schema against some data"""
+    f = StringIO(file_contents)
+    data = syaml.load(f)
+    spack.config._validate(data, name)
+
+
+def test_bad_config_yaml(tmpdir):
+    with pytest.raises(spack.config.ConfigFormatError):
+        check_schema(spack.schema.config.schema, """\
+config:
+    verify_ssl: False
+    module_roots:
+        fmod: /some/fake/location
+""")
+
+
+def test_bad_mirrors_yaml(tmpdir):
+    with pytest.raises(spack.config.ConfigFormatError):
+        check_schema(spack.schema.mirrors.schema, """\
+mirrors:
+    local: True
+""")
+
+
+def test_bad_repos_yaml(tmpdir):
+    with pytest.raises(spack.config.ConfigFormatError):
+        check_schema(spack.schema.repos.schema, """\
+repos:
+    True
+""")
+
+
+def test_bad_compilers_yaml(tmpdir):
+    with pytest.raises(spack.config.ConfigFormatError):
+        check_schema(spack.schema.compilers.schema, """\
+compilers:
+    key_instead_of_list: 'value'
+""")
+
+    with pytest.raises(spack.config.ConfigFormatError):
+        check_schema(spack.schema.compilers.schema, """\
+compilers:
+    - shmompiler:
+         environment: /bad/value
+""")
+
+    with pytest.raises(spack.config.ConfigFormatError):
+        check_schema(spack.schema.compilers.schema, """\
+compilers:
+    - compiler:
+         fenfironfent: /bad/value
+""")