add CombinatorialSpecSet class for taking cross-products of Specs.
- add CombinatorialSpecSet in spack.util.spec_set module. - class is iterable and encaspulated YAML parsing and validation. - Adjust YAML format to be more generic - YAML spec-set format now has a `matrix` section, which can contain multiple lists of specs, generated different ways. Including: - specs: a raw list of specs. - packages: a list of package names and versions - compilers: a list of compiler names and versions - All of the elements of `matrix` are dimensions for the build matrix; we take the cartesian product of these lists of specs to generate a build matrix. This means we can add things like [^mpich, ^openmpi] to get builds with different MPI versions. It also means we can multiply the build matrix out with lots of different parameters. - Add a schema format for spec-sets
This commit is contained in:
parent
ad8036e5a2
commit
be4b95ee30
21
lib/spack/docs/example_files/spec_set.yaml
Normal file
21
lib/spack/docs/example_files/spec_set.yaml
Normal file
@ -0,0 +1,21 @@
|
||||
spec-set:
|
||||
include: [ ape, atompaw, transset]
|
||||
exclude: [binutils,tk]
|
||||
packages:
|
||||
ape:
|
||||
versions: [2.2.1]
|
||||
atompaw:
|
||||
versions: [3.1.0.3, 4.0.0.13]
|
||||
binutils:
|
||||
versions: [2.20.1, 2.25, 2.23.2, 2.24, 2.27, 2.26]
|
||||
tk:
|
||||
versions: [8.6.5, 8.6.3]
|
||||
transset:
|
||||
versions: [1.0.1]
|
||||
compilers:
|
||||
gcc:
|
||||
versions: [4.9, 4.8, 4.7]
|
||||
clang:
|
||||
versions: [3.5, 3.6]
|
||||
|
||||
dashboard: ["https://spack.io/cdash/submit.php?project=spack"]
|
@ -81,6 +81,11 @@ or refer to the full manual below.
|
||||
build_systems
|
||||
developer_guide
|
||||
docker_for_developers
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: API Docs
|
||||
|
||||
Spack API Docs <spack>
|
||||
LLNL API Docs <llnl>
|
||||
|
||||
|
110
lib/spack/spack/schema/spec_set.py
Normal file
110
lib/spack/spack/schema/spec_set.py
Normal file
@ -0,0 +1,110 @@
|
||||
# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
"""Schema for Spack spec-set configuration file.
|
||||
|
||||
.. literalinclude:: ../spack/schema/spec_set.py
|
||||
:lines: 32-
|
||||
"""
|
||||
|
||||
|
||||
schema = {
|
||||
'$schema': 'http://json-schema.org/schema#',
|
||||
'title': 'Spack test configuration file schema',
|
||||
'definitions': {
|
||||
# used for include/exclude
|
||||
'list_of_specs': {
|
||||
'type': 'array',
|
||||
'items': {'type': 'string'}
|
||||
},
|
||||
# used for compilers and for packages
|
||||
'objects_with_version_list': {
|
||||
'type': 'object',
|
||||
'additionalProperties': False,
|
||||
'patternProperties': {
|
||||
r'\w[\w-]*': {
|
||||
'type': 'object',
|
||||
'additionalProperties': False,
|
||||
'required': ['versions'],
|
||||
'properties': {
|
||||
'versions': {
|
||||
'type': 'array',
|
||||
'items': {
|
||||
'oneOf': [
|
||||
{'type': 'string'},
|
||||
{'type': 'number'},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
'packages': {
|
||||
'type': 'object',
|
||||
'additionalProperties': False,
|
||||
'properties': {
|
||||
'packages': {
|
||||
'$ref': '#/definitions/objects_with_version_list'
|
||||
},
|
||||
}
|
||||
},
|
||||
'compilers': {
|
||||
'type': 'object',
|
||||
'additionalProperties': False,
|
||||
'properties': {
|
||||
'compilers': {
|
||||
'$ref': '#/definitions/objects_with_version_list'
|
||||
},
|
||||
}
|
||||
},
|
||||
'specs': {
|
||||
'type': 'object',
|
||||
'additionalProperties': False,
|
||||
'properties': {
|
||||
'specs': {'$ref': '#/definitions/list_of_specs'},
|
||||
}
|
||||
},
|
||||
},
|
||||
# this is the actual top level object
|
||||
'type': 'object',
|
||||
'additionalProperties': False,
|
||||
'properties': {
|
||||
'spec-set': {
|
||||
'type': 'object',
|
||||
'additionalProperties': False,
|
||||
'required': ['matrix'],
|
||||
'properties': {
|
||||
# top-level settings are keys and need to be unique
|
||||
'include': {'$ref': '#/definitions/list_of_specs'},
|
||||
'exclude': {'$ref': '#/definitions/list_of_specs'},
|
||||
'cdash': {
|
||||
'oneOf': [
|
||||
{'type': 'string'},
|
||||
{'type': 'array',
|
||||
'items': {'type': 'string'}
|
||||
},
|
||||
],
|
||||
},
|
||||
'project': {
|
||||
'type': 'string',
|
||||
},
|
||||
# things under matrix (packages, compilers, etc.) are a
|
||||
# list so that we can potentiall have multiple of them.
|
||||
'matrix': {
|
||||
'type': 'array',
|
||||
'items': {
|
||||
'type': 'object',
|
||||
'oneOf': [
|
||||
{'$ref': '#/definitions/specs'},
|
||||
{'$ref': '#/definitions/packages'},
|
||||
{'$ref': '#/definitions/compilers'},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
188
lib/spack/spack/spec_set.py
Normal file
188
lib/spack/spack/spec_set.py
Normal file
@ -0,0 +1,188 @@
|
||||
# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import itertools
|
||||
from jsonschema import validate
|
||||
|
||||
import llnl.util.tty as tty
|
||||
from llnl.util.tty.colify import colify
|
||||
|
||||
import spack
|
||||
import spack.compilers
|
||||
import spack.architecture as sarch
|
||||
import spack.schema.spec_set as spec_set_schema
|
||||
import spack.util.spack_yaml as syaml
|
||||
|
||||
from spack.error import SpackError
|
||||
from spack.spec import Spec, ArchSpec
|
||||
|
||||
|
||||
class CombinatorialSpecSet:
|
||||
"""Set of combinatorial Specs constructed from YAML file."""
|
||||
|
||||
def __init__(self, yaml_like, ignore_invalid=True):
|
||||
"""Construct a combinatorial Spec set.
|
||||
|
||||
Args:
|
||||
yaml_like: either raw YAML data as a dict, a file-like object
|
||||
to read the YAML from, or a string containing YAML. In the
|
||||
first case, we assume already-parsed YAML data. In the second
|
||||
two cases, we just run yaml.load() on the data.
|
||||
ignore_invalid (bool): whether to ignore invalid specs when
|
||||
expanding the values of this spec set.
|
||||
"""
|
||||
self.ignore_invalid = ignore_invalid
|
||||
|
||||
if isinstance(yaml_like, dict):
|
||||
# if it's raw data, just assign it to self.data
|
||||
self.data = yaml_like
|
||||
else:
|
||||
# otherwise try to load it.
|
||||
self.data = syaml.load(yaml_like)
|
||||
|
||||
# validate against the spec set schema
|
||||
validate(self.data, spec_set_schema.schema)
|
||||
|
||||
# chop off the initial spec-set label after valiation.
|
||||
self.data = self.data['spec-set']
|
||||
|
||||
# initialize these from data.
|
||||
self.cdash = self.data.get('cdash', None)
|
||||
if isinstance(self.cdash, str):
|
||||
self.cdash = [self.cdash]
|
||||
self.project = self.data.get('project', None)
|
||||
|
||||
# _spec_lists is a list of lists of specs, to be combined as a
|
||||
# cartesian product when we iterate over all specs in the set.
|
||||
# it's initialized lazily.
|
||||
self._spec_lists = None
|
||||
self._include = []
|
||||
self._exclude = []
|
||||
|
||||
@staticmethod
|
||||
def from_file(path):
|
||||
try:
|
||||
with open(path, 'r') as fin:
|
||||
specs_yaml = syaml.load(fin.read())
|
||||
|
||||
# For now, turn off ignoring invalid specs, as it prevents
|
||||
# iteration if the specified compilers can't be found.
|
||||
return CombinatorialSpecSet(specs_yaml, ignore_invalid=False)
|
||||
except Exception as e:
|
||||
emsg = e.message
|
||||
if not emsg:
|
||||
emsg = e.problem
|
||||
msg = ('Unable to create CombinatorialSpecSet from file ({0})'
|
||||
' due to {1}'.format(path, emsg))
|
||||
raise SpackError(msg)
|
||||
|
||||
def all_package_versions(self):
|
||||
"""Get package/version combinations for all spack packages."""
|
||||
for name in spack.repo.all_package_names():
|
||||
pkg = spack.repo.get(name)
|
||||
for v in pkg.versions:
|
||||
yield Spec('{0}@{1}'.format(name, v))
|
||||
|
||||
def _specs(self, data):
|
||||
"""Read a list of specs from YAML data"""
|
||||
return [Spec(s) for s in data]
|
||||
|
||||
def _compiler_specs(self, data):
|
||||
"""Read compiler specs from YAML data.
|
||||
Example YAML:
|
||||
gcc:
|
||||
versions: [4.4.8, 4.9.3]
|
||||
clang:
|
||||
versions: [3.6.1, 3.7.2, 3.8]
|
||||
|
||||
Optionally, data can be 'all', in which case all compilers for
|
||||
the current platform are returned.
|
||||
"""
|
||||
# get usable compilers for current platform.
|
||||
arch = ArchSpec(str(sarch.platform()), 'default_os', 'default_target')
|
||||
available_compilers = [
|
||||
c.spec for c in spack.compilers.compilers_for_arch(arch)]
|
||||
|
||||
# return compilers for this platform if asked for everything.
|
||||
if data == 'all':
|
||||
return [cspec.copy() for cspec in available_compilers]
|
||||
|
||||
# otherwise create specs from the YAML file.
|
||||
cspecs = set([
|
||||
Spec('%{0}@{1}'.format(compiler, version))
|
||||
for compiler in data for version in data[compiler]['versions']])
|
||||
|
||||
# filter out invalid specs if caller said to ignore them.
|
||||
if self.ignore_invalid:
|
||||
missing = [c for c in cspecs if not any(
|
||||
c.compiler.satisfies(comp) for comp in available_compilers)]
|
||||
tty.warn("The following compilers were unavailable:")
|
||||
colify(sorted(m.compiler for m in missing))
|
||||
cspecs -= set(missing)
|
||||
|
||||
return cspecs
|
||||
|
||||
def _package_specs(self, data):
|
||||
"""Read package/version specs from YAML data.
|
||||
Example YAML:
|
||||
gmake:
|
||||
versions: [4.0, 4.1, 4.2]
|
||||
qt:
|
||||
versions: [4.8.6, 5.2.1, 5.7.1]
|
||||
|
||||
Optionally, data can be 'all', in which case all packages and
|
||||
versions from the package repository are returned.
|
||||
"""
|
||||
if data == 'all':
|
||||
return set(self.all_package_versions())
|
||||
|
||||
return set([
|
||||
Spec('{0}@{1}'.format(name, version))
|
||||
for name in data for version in data[name]['versions']])
|
||||
|
||||
def _get_specs(self, matrix_dict):
|
||||
"""Parse specs out of an element in the build matrix."""
|
||||
readers = {
|
||||
'packages': self._package_specs,
|
||||
'compilers': self._compiler_specs,
|
||||
'specs': self._specs
|
||||
}
|
||||
|
||||
key = next(iter(matrix_dict), None)
|
||||
assert key in readers
|
||||
return readers[key](matrix_dict[key])
|
||||
|
||||
def __iter__(self):
|
||||
# read in data from YAML file lazily.
|
||||
if self._spec_lists is None:
|
||||
self._spec_lists = [self._get_specs(spec_list)
|
||||
for spec_list in self.data['matrix']]
|
||||
|
||||
if 'include' in self.data:
|
||||
self._include = [Spec(s) for s in self.data['include']]
|
||||
if 'exclude' in self.data:
|
||||
self._exclude = [Spec(s) for s in self.data['exclude']]
|
||||
|
||||
for spec_list in itertools.product(*self._spec_lists):
|
||||
# if there is an empty array in spec_lists, we'll get this.
|
||||
if not spec_list:
|
||||
yield spec_list
|
||||
continue
|
||||
|
||||
# merge all the constraints in spec_list with each other
|
||||
spec = spec_list[0].copy()
|
||||
for s in spec_list[1:]:
|
||||
spec.constrain(s)
|
||||
|
||||
# test each spec for include/exclude
|
||||
if (self._include and
|
||||
not any(spec.satisfies(s) for s in self._include)):
|
||||
continue
|
||||
|
||||
if any(spec.satisfies(s) for s in self._exclude):
|
||||
continue
|
||||
|
||||
# we now know we can include this spec in the set
|
||||
yield spec
|
299
lib/spack/spack/test/spec_set.py
Normal file
299
lib/spack/spack/test/spec_set.py
Normal file
@ -0,0 +1,299 @@
|
||||
# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import pytest
|
||||
|
||||
from spack.spec import Spec
|
||||
from jsonschema import ValidationError
|
||||
from spack.spec_set import CombinatorialSpecSet
|
||||
|
||||
|
||||
pytestmark = pytest.mark.usefixtures('config')
|
||||
|
||||
|
||||
basic_yaml_file = {
|
||||
'spec-set': {
|
||||
'cdash': 'http://example.com/cdash',
|
||||
'project': 'testproj',
|
||||
'include': ['gmake'],
|
||||
'matrix': [
|
||||
{'packages': {
|
||||
'gmake': {
|
||||
'versions': ['4.0']
|
||||
}
|
||||
}},
|
||||
{'compilers': {
|
||||
'gcc': {
|
||||
'versions': ['4.2.1', '6.3.0']
|
||||
}, 'clang': {
|
||||
'versions': ['8.0', '3.8']
|
||||
}
|
||||
}},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_spec_set_basic():
|
||||
"""The "include" isn't required, but if it is present, we should only
|
||||
see specs mentioned there. Also, if we include cdash and project
|
||||
properties, those should be captured and stored on the resulting
|
||||
CombinatorialSpecSet as attributes."""
|
||||
spec_set = CombinatorialSpecSet(basic_yaml_file, False)
|
||||
specs = list(spec for spec in spec_set)
|
||||
assert len(specs) == 4
|
||||
assert spec_set.cdash == ['http://example.com/cdash']
|
||||
assert spec_set.project == 'testproj'
|
||||
|
||||
|
||||
def test_spec_set_no_include():
|
||||
"""Make sure that without any exclude or include, we get the full cross-
|
||||
product of specs/versions."""
|
||||
yaml_file = {
|
||||
'spec-set': {
|
||||
'matrix': [
|
||||
{'packages': {
|
||||
'gmake': {
|
||||
'versions': ['4.0']
|
||||
}
|
||||
}},
|
||||
{'compilers': {
|
||||
'gcc': {
|
||||
'versions': ['4.2.1', '6.3.0']
|
||||
}, 'clang': {
|
||||
'versions': ['8.0', '3.8']
|
||||
}
|
||||
}},
|
||||
]
|
||||
}
|
||||
}
|
||||
spec_set = CombinatorialSpecSet(yaml_file, False)
|
||||
specs = list(spec for spec in spec_set)
|
||||
assert len(specs) == 4
|
||||
|
||||
|
||||
def test_spec_set_include_exclude_conflict():
|
||||
"""Exclude should override include"""
|
||||
yaml_file = {
|
||||
'spec-set': {
|
||||
'include': ['gmake'],
|
||||
'exclude': ['gmake'],
|
||||
'matrix': [
|
||||
{'packages': {
|
||||
'gmake': {
|
||||
'versions': ['4.0']
|
||||
}
|
||||
}},
|
||||
{'compilers': {
|
||||
'gcc': {
|
||||
'versions': ['4.2.1', '6.3.0']
|
||||
}, 'clang': {
|
||||
'versions': ['8.0', '3.8']
|
||||
}
|
||||
}},
|
||||
]
|
||||
}
|
||||
}
|
||||
spec_set = CombinatorialSpecSet(yaml_file, False)
|
||||
specs = list(spec for spec in spec_set)
|
||||
assert len(specs) == 0
|
||||
|
||||
|
||||
def test_spec_set_exclude():
|
||||
"""The exclude property isn't required, but if it appears, any specs
|
||||
mentioned there should not appear in the output specs"""
|
||||
yaml_file = {
|
||||
'spec-set': {
|
||||
'exclude': ['gmake'],
|
||||
'matrix': [
|
||||
{'packages': {
|
||||
'gmake': {
|
||||
'versions': ['4.0']
|
||||
},
|
||||
'appres': {
|
||||
'versions': ['1.0.4']
|
||||
},
|
||||
'allinea-reports': {
|
||||
'versions': ['6.0.4']
|
||||
}
|
||||
}},
|
||||
{'compilers': {
|
||||
'gcc': {
|
||||
'versions': ['4.2.1', '6.3.0']
|
||||
}, 'clang': {
|
||||
'versions': ['8.0', '3.8']
|
||||
}
|
||||
}},
|
||||
]
|
||||
}
|
||||
}
|
||||
spec_set = CombinatorialSpecSet(yaml_file, False)
|
||||
specs = list(spec for spec in spec_set)
|
||||
assert len(specs) == 8
|
||||
|
||||
|
||||
def test_spec_set_include_limited_packages():
|
||||
"""If we see the include key, it is a filter and only the specs mentioned
|
||||
there should actually be included."""
|
||||
yaml_file = {
|
||||
'spec-set': {
|
||||
'include': ['gmake'],
|
||||
'matrix': [
|
||||
{'packages': {
|
||||
'gmake': {
|
||||
'versions': ['4.0']
|
||||
},
|
||||
'appres': {
|
||||
'versions': ['1.0.4']
|
||||
},
|
||||
'allinea-reports': {
|
||||
'versions': ['6.0.4']
|
||||
}
|
||||
}},
|
||||
{'compilers': {
|
||||
'gcc': {
|
||||
'versions': ['4.2.1', '6.3.0']
|
||||
}, 'clang': {
|
||||
'versions': ['8.0', '3.8']
|
||||
}
|
||||
}},
|
||||
]
|
||||
}
|
||||
}
|
||||
spec_set = CombinatorialSpecSet(yaml_file, False)
|
||||
specs = list(spec for spec in spec_set)
|
||||
assert len(specs) == 4
|
||||
|
||||
|
||||
def test_spec_set_simple_spec_list():
|
||||
"""Make sure we can handle the slightly more concise syntax where we
|
||||
include the package name/version together and skip the extra keys in
|
||||
the dictionary."""
|
||||
yaml_file = {
|
||||
'spec-set': {
|
||||
'matrix': [
|
||||
{'specs': [
|
||||
'gmake@4.0',
|
||||
'appres@1.0.4',
|
||||
'allinea-reports@6.0.4'
|
||||
]},
|
||||
]
|
||||
}
|
||||
}
|
||||
spec_set = CombinatorialSpecSet(yaml_file, False)
|
||||
specs = list(spec for spec in spec_set)
|
||||
assert len(specs) == 3
|
||||
|
||||
|
||||
def test_spec_set_with_specs():
|
||||
"""Make sure we only see the specs mentioned in the include"""
|
||||
yaml_file = {
|
||||
'spec-set': {
|
||||
'include': ['gmake', 'appres'],
|
||||
'matrix': [
|
||||
{'specs': [
|
||||
'gmake@4.0',
|
||||
'appres@1.0.4',
|
||||
'allinea-reports@6.0.4'
|
||||
]},
|
||||
{'compilers': {
|
||||
'gcc': {
|
||||
'versions': ['4.2.1', '6.3.0']
|
||||
}, 'clang': {
|
||||
'versions': ['8.0', '3.8']
|
||||
}
|
||||
}},
|
||||
]
|
||||
}
|
||||
}
|
||||
spec_set = CombinatorialSpecSet(yaml_file, False)
|
||||
specs = list(spec for spec in spec_set)
|
||||
assert len(specs) == 8
|
||||
|
||||
|
||||
def test_spec_set_packages_no_matrix():
|
||||
"""The matrix property is required, make sure we error out if it is
|
||||
missing"""
|
||||
yaml_file = {
|
||||
'spec-set': {
|
||||
'include': ['gmake'],
|
||||
'packages': {
|
||||
'gmake': {
|
||||
'versions': ['4.0']
|
||||
},
|
||||
'appres': {
|
||||
'versions': ['1.0.4']
|
||||
},
|
||||
'allinea-reports': {
|
||||
'versions': ['6.0.4']
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
with pytest.raises(ValidationError):
|
||||
CombinatorialSpecSet(yaml_file)
|
||||
|
||||
|
||||
def test_spec_set_get_cdash_array():
|
||||
"""Make sure we can handle multiple cdash sites in a list"""
|
||||
yaml_file = {
|
||||
'spec-set': {
|
||||
'cdash': ['http://example.com/cdash', 'http://example.com/cdash2'],
|
||||
'project': 'testproj',
|
||||
'matrix': [
|
||||
{'packages': {
|
||||
'gmake': {'versions': ['4.0']},
|
||||
}},
|
||||
{'compilers': {
|
||||
'gcc': {'versions': ['4.2.1', '6.3.0']},
|
||||
'clang': {'versions': ['8.0', '3.8']},
|
||||
}},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
spec_set = CombinatorialSpecSet(yaml_file)
|
||||
assert spec_set.cdash == [
|
||||
'http://example.com/cdash', 'http://example.com/cdash2']
|
||||
assert spec_set.project == 'testproj'
|
||||
|
||||
|
||||
def test_compiler_specs():
|
||||
spec_set = CombinatorialSpecSet(basic_yaml_file, False)
|
||||
compilers = spec_set._compiler_specs({
|
||||
'gcc': {
|
||||
'versions': ['4.2.1', '6.3.0']
|
||||
}, 'clang': {
|
||||
'versions': ['8.0', '3.8']
|
||||
}})
|
||||
|
||||
assert len(list(compilers)) == 4
|
||||
assert Spec('%gcc@4.2.1') in compilers
|
||||
assert Spec('%gcc@6.3.0') in compilers
|
||||
assert Spec('%clang@8.0') in compilers
|
||||
assert Spec('%clang@3.8') in compilers
|
||||
|
||||
|
||||
def test_package_specs():
|
||||
spec_set = CombinatorialSpecSet(basic_yaml_file, False)
|
||||
|
||||
packages = spec_set._package_specs({
|
||||
'gmake': {
|
||||
'versions': ['4.0', '5.0']
|
||||
},
|
||||
'appres': {
|
||||
'versions': ['1.0.4']
|
||||
},
|
||||
'allinea-reports': {
|
||||
'versions': ['6.0.1', '6.0.3', '6.0.4']
|
||||
}
|
||||
})
|
||||
|
||||
assert Spec('gmake@4.0') in packages
|
||||
assert Spec('gmake@5.0') in packages
|
||||
assert Spec('appres@1.0.4') in packages
|
||||
assert Spec('allinea-reports@6.0.1') in packages
|
||||
assert Spec('allinea-reports@6.0.3') in packages
|
||||
assert Spec('allinea-reports@6.0.4') in packages
|
Loading…
Reference in New Issue
Block a user