zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add CombinatorialSpecSet class for taking cross-products of Specs.

Browse Source 
- add CombinatorialSpecSet in spack.util.spec_set module.
  - class is iterable and encaspulated YAML parsing and validation.

- Adjust YAML format to be more generic
  - YAML spec-set format now has a `matrix` section, which can contain
    multiple lists of specs, generated different ways. Including:
    - specs: a raw list of specs.
    - packages: a list of package names and versions
    - compilers: a list of compiler names and versions

  - All of the elements of `matrix` are dimensions for the build matrix;
    we take the cartesian product of these lists of specs to generate a
    build matrix.  This means we can add things like [^mpich, ^openmpi]
    to get builds with different MPI versions.  It also means we can
    multiply the build matrix out with lots of different parameters.

- Add a schema format for spec-sets
This commit is contained in:
Todd Gamblin 2018-06-26 03:28:49 -07:00 committed by Peter Scheibel
parent ad8036e5a2
commit be4b95ee30
5 changed files with 623 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
lib/spack/docs/example_files/spec_set.yaml Normal file
View File

@ -0,0 +1,21 @@
spec-set:
include: [ ape, atompaw, transset]
exclude: [binutils,tk]
packages:
ape:
versions: [2.2.1]
atompaw:
versions: [3.1.0.3, 4.0.0.13]
binutils:
versions: [2.20.1, 2.25, 2.23.2, 2.24, 2.27, 2.26]
tk:
versions: [8.6.5, 8.6.3]
transset:
versions: [1.0.1]
compilers:
gcc:
versions: [4.9, 4.8, 4.7]
clang:
versions: [3.5, 3.6]
dashboard: ["https://spack.io/cdash/submit.php?project=spack"]

5
lib/spack/docs/index.rst
View File

@ -81,6 +81,11 @@ or refer to the full manual below.
build_systems
developer_guide
docker_for_developers
.. toctree::
:maxdepth: 2
:caption: API Docs
Spack API Docs <spack>
LLNL API Docs <llnl>

110
lib/spack/spack/schema/spec_set.py Normal file
View File

@ -0,0 +1,110 @@
# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Schema for Spack spec-set configuration file.
.. literalinclude:: ../spack/schema/spec_set.py
:lines: 32-
"""
schema = {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack test configuration file schema',
'definitions': {
# used for include/exclude
'list_of_specs': {
'type': 'array',
'items': {'type': 'string'}
},
# used for compilers and for packages
'objects_with_version_list': {
'type': 'object',
'additionalProperties': False,
'patternProperties': {
r'\w[\w-]*': {
'type': 'object',
'additionalProperties': False,
'required': ['versions'],
'properties': {
'versions': {
'type': 'array',
'items': {
'oneOf': [
{'type': 'string'},
{'type': 'number'},
],
},
},
},
},
},
},
'packages': {
'type': 'object',
'additionalProperties': False,
'properties': {
'packages': {
'$ref': '#/definitions/objects_with_version_list'
},
}
},
'compilers': {
'type': 'object',
'additionalProperties': False,
'properties': {
'compilers': {
'$ref': '#/definitions/objects_with_version_list'
},
}
},
'specs': {
'type': 'object',
'additionalProperties': False,
'properties': {
'specs': {'$ref': '#/definitions/list_of_specs'},
}
},
},
# this is the actual top level object
'type': 'object',
'additionalProperties': False,
'properties': {
'spec-set': {
'type': 'object',
'additionalProperties': False,
'required': ['matrix'],
'properties': {
# top-level settings are keys and need to be unique
'include': {'$ref': '#/definitions/list_of_specs'},
'exclude': {'$ref': '#/definitions/list_of_specs'},
'cdash': {
'oneOf': [
{'type': 'string'},
{'type': 'array',
'items': {'type': 'string'}
},
],
},
'project': {
'type': 'string',
},
# things under matrix (packages, compilers, etc.) are a
# list so that we can potentiall have multiple of them.
'matrix': {
'type': 'array',
'items': {
'type': 'object',
'oneOf': [
{'$ref': '#/definitions/specs'},
{'$ref': '#/definitions/packages'},
{'$ref': '#/definitions/compilers'},
],
},
},
},
},
},
}

188
lib/spack/spack/spec_set.py Normal file
View File

@ -0,0 +1,188 @@
# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import itertools
from jsonschema import validate
import llnl.util.tty as tty
from llnl.util.tty.colify import colify
import spack
import spack.compilers
import spack.architecture as sarch
import spack.schema.spec_set as spec_set_schema
import spack.util.spack_yaml as syaml
from spack.error import SpackError
from spack.spec import Spec, ArchSpec
class CombinatorialSpecSet:
"""Set of combinatorial Specs constructed from YAML file."""
def __init__(self, yaml_like, ignore_invalid=True):
"""Construct a combinatorial Spec set.
Args:
yaml_like: either raw YAML data as a dict, a file-like object
to read the YAML from, or a string containing YAML. In the
first case, we assume already-parsed YAML data. In the second
two cases, we just run yaml.load() on the data.
ignore_invalid (bool): whether to ignore invalid specs when
expanding the values of this spec set.
"""
self.ignore_invalid = ignore_invalid
if isinstance(yaml_like, dict):
# if it's raw data, just assign it to self.data
self.data = yaml_like
else:
# otherwise try to load it.
self.data = syaml.load(yaml_like)
# validate against the spec set schema
validate(self.data, spec_set_schema.schema)
# chop off the initial spec-set label after valiation.
self.data = self.data['spec-set']
# initialize these from data.
self.cdash = self.data.get('cdash', None)
if isinstance(self.cdash, str):
self.cdash = [self.cdash]
self.project = self.data.get('project', None)
# _spec_lists is a list of lists of specs, to be combined as a
# cartesian product when we iterate over all specs in the set.
# it's initialized lazily.
self._spec_lists = None
self._include = []
self._exclude = []
@staticmethod
def from_file(path):
try:
with open(path, 'r') as fin:
specs_yaml = syaml.load(fin.read())
# For now, turn off ignoring invalid specs, as it prevents
# iteration if the specified compilers can't be found.
return CombinatorialSpecSet(specs_yaml, ignore_invalid=False)
except Exception as e:
emsg = e.message
if not emsg:
emsg = e.problem
msg = ('Unable to create CombinatorialSpecSet from file ({0})'
' due to {1}'.format(path, emsg))
raise SpackError(msg)
def all_package_versions(self):
"""Get package/version combinations for all spack packages."""
for name in spack.repo.all_package_names():
pkg = spack.repo.get(name)
for v in pkg.versions:
yield Spec('{0}@{1}'.format(name, v))
def _specs(self, data):
"""Read a list of specs from YAML data"""
return [Spec(s) for s in data]
def _compiler_specs(self, data):
"""Read compiler specs from YAML data.
Example YAML:
gcc:
versions: [4.4.8, 4.9.3]
clang:
versions: [3.6.1, 3.7.2, 3.8]
Optionally, data can be 'all', in which case all compilers for
the current platform are returned.
"""
# get usable compilers for current platform.
arch = ArchSpec(str(sarch.platform()), 'default_os', 'default_target')
available_compilers = [
c.spec for c in spack.compilers.compilers_for_arch(arch)]
# return compilers for this platform if asked for everything.
if data == 'all':
return [cspec.copy() for cspec in available_compilers]
# otherwise create specs from the YAML file.
cspecs = set([
Spec('%{0}@{1}'.format(compiler, version))
for compiler in data for version in data[compiler]['versions']])
# filter out invalid specs if caller said to ignore them.
if self.ignore_invalid:
missing = [c for c in cspecs if not any(
c.compiler.satisfies(comp) for comp in available_compilers)]
tty.warn("The following compilers were unavailable:")
colify(sorted(m.compiler for m in missing))
cspecs -= set(missing)
return cspecs
def _package_specs(self, data):
"""Read package/version specs from YAML data.
Example YAML:
gmake:
versions: [4.0, 4.1, 4.2]
qt:
versions: [4.8.6, 5.2.1, 5.7.1]
Optionally, data can be 'all', in which case all packages and
versions from the package repository are returned.
"""
if data == 'all':
return set(self.all_package_versions())
return set([
Spec('{0}@{1}'.format(name, version))
for name in data for version in data[name]['versions']])
def _get_specs(self, matrix_dict):
"""Parse specs out of an element in the build matrix."""
readers = {
'packages': self._package_specs,
'compilers': self._compiler_specs,
'specs': self._specs
}
key = next(iter(matrix_dict), None)
assert key in readers
return readers[key](matrix_dict[key])
def __iter__(self):
# read in data from YAML file lazily.
if self._spec_lists is None:
self._spec_lists = [self._get_specs(spec_list)
for spec_list in self.data['matrix']]
if 'include' in self.data:
self._include = [Spec(s) for s in self.data['include']]
if 'exclude' in self.data:
self._exclude = [Spec(s) for s in self.data['exclude']]
for spec_list in itertools.product(*self._spec_lists):
# if there is an empty array in spec_lists, we'll get this.
if not spec_list:
yield spec_list
continue
# merge all the constraints in spec_list with each other
spec = spec_list[0].copy()
for s in spec_list[1:]:
spec.constrain(s)
# test each spec for include/exclude
if (self._include and
not any(spec.satisfies(s) for s in self._include)):
continue
if any(spec.satisfies(s) for s in self._exclude):
continue
# we now know we can include this spec in the set
yield spec

299
lib/spack/spack/test/spec_set.py Normal file
View File

@ -0,0 +1,299 @@
# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import pytest
from spack.spec import Spec
from jsonschema import ValidationError
from spack.spec_set import CombinatorialSpecSet
pytestmark = pytest.mark.usefixtures('config')
basic_yaml_file = {
'spec-set': {
'cdash': 'http://example.com/cdash',
'project': 'testproj',
'include': ['gmake'],
'matrix': [
{'packages': {
'gmake': {
'versions': ['4.0']
}
}},
{'compilers': {
'gcc': {
'versions': ['4.2.1', '6.3.0']
}, 'clang': {
'versions': ['8.0', '3.8']
}
}},
]
}
}
def test_spec_set_basic():
"""The "include" isn't required, but if it is present, we should only
see specs mentioned there. Also, if we include cdash and project
properties, those should be captured and stored on the resulting
CombinatorialSpecSet as attributes."""
spec_set = CombinatorialSpecSet(basic_yaml_file, False)
specs = list(spec for spec in spec_set)
assert len(specs) == 4
assert spec_set.cdash == ['http://example.com/cdash']
assert spec_set.project == 'testproj'
def test_spec_set_no_include():
"""Make sure that without any exclude or include, we get the full cross-
product of specs/versions."""
yaml_file = {
'spec-set': {
'matrix': [
{'packages': {
'gmake': {
'versions': ['4.0']
}
}},
{'compilers': {
'gcc': {
'versions': ['4.2.1', '6.3.0']
}, 'clang': {
'versions': ['8.0', '3.8']
}
}},
]
}
}
spec_set = CombinatorialSpecSet(yaml_file, False)
specs = list(spec for spec in spec_set)
assert len(specs) == 4
def test_spec_set_include_exclude_conflict():
"""Exclude should override include"""
yaml_file = {
'spec-set': {
'include': ['gmake'],
'exclude': ['gmake'],
'matrix': [
{'packages': {
'gmake': {
'versions': ['4.0']
}
}},
{'compilers': {
'gcc': {
'versions': ['4.2.1', '6.3.0']
}, 'clang': {
'versions': ['8.0', '3.8']
}
}},
]
}
}
spec_set = CombinatorialSpecSet(yaml_file, False)
specs = list(spec for spec in spec_set)
assert len(specs) == 0
def test_spec_set_exclude():
"""The exclude property isn't required, but if it appears, any specs
mentioned there should not appear in the output specs"""
yaml_file = {
'spec-set': {
'exclude': ['gmake'],
'matrix': [
{'packages': {
'gmake': {
'versions': ['4.0']
},
'appres': {
'versions': ['1.0.4']
},
'allinea-reports': {
'versions': ['6.0.4']
}
}},
{'compilers': {
'gcc': {
'versions': ['4.2.1', '6.3.0']
}, 'clang': {
'versions': ['8.0', '3.8']
}
}},
]
}
}
spec_set = CombinatorialSpecSet(yaml_file, False)
specs = list(spec for spec in spec_set)
assert len(specs) == 8
def test_spec_set_include_limited_packages():
"""If we see the include key, it is a filter and only the specs mentioned
there should actually be included."""
yaml_file = {
'spec-set': {
'include': ['gmake'],
'matrix': [
{'packages': {
'gmake': {
'versions': ['4.0']
},
'appres': {
'versions': ['1.0.4']
},
'allinea-reports': {
'versions': ['6.0.4']
}
}},
{'compilers': {
'gcc': {
'versions': ['4.2.1', '6.3.0']
}, 'clang': {
'versions': ['8.0', '3.8']
}
}},
]
}
}
spec_set = CombinatorialSpecSet(yaml_file, False)
specs = list(spec for spec in spec_set)
assert len(specs) == 4
def test_spec_set_simple_spec_list():
"""Make sure we can handle the slightly more concise syntax where we
include the package name/version together and skip the extra keys in
the dictionary."""
yaml_file = {
'spec-set': {
'matrix': [
{'specs': [
'gmake@4.0',
'appres@1.0.4',
'allinea-reports@6.0.4'
]},
]
}
}
spec_set = CombinatorialSpecSet(yaml_file, False)
specs = list(spec for spec in spec_set)
assert len(specs) == 3
def test_spec_set_with_specs():
"""Make sure we only see the specs mentioned in the include"""
yaml_file = {
'spec-set': {
'include': ['gmake', 'appres'],
'matrix': [
{'specs': [
'gmake@4.0',
'appres@1.0.4',
'allinea-reports@6.0.4'
]},
{'compilers': {
'gcc': {
'versions': ['4.2.1', '6.3.0']
}, 'clang': {
'versions': ['8.0', '3.8']
}
}},
]
}
}
spec_set = CombinatorialSpecSet(yaml_file, False)
specs = list(spec for spec in spec_set)
assert len(specs) == 8
def test_spec_set_packages_no_matrix():
"""The matrix property is required, make sure we error out if it is
missing"""
yaml_file = {
'spec-set': {
'include': ['gmake'],
'packages': {
'gmake': {
'versions': ['4.0']
},
'appres': {
'versions': ['1.0.4']
},
'allinea-reports': {
'versions': ['6.0.4']
}
},
}
}
with pytest.raises(ValidationError):
CombinatorialSpecSet(yaml_file)
def test_spec_set_get_cdash_array():
"""Make sure we can handle multiple cdash sites in a list"""
yaml_file = {
'spec-set': {
'cdash': ['http://example.com/cdash', 'http://example.com/cdash2'],
'project': 'testproj',
'matrix': [
{'packages': {
'gmake': {'versions': ['4.0']},
}},
{'compilers': {
'gcc': {'versions': ['4.2.1', '6.3.0']},
'clang': {'versions': ['8.0', '3.8']},
}},
]
}
}
spec_set = CombinatorialSpecSet(yaml_file)
assert spec_set.cdash == [
'http://example.com/cdash', 'http://example.com/cdash2']
assert spec_set.project == 'testproj'
def test_compiler_specs():
spec_set = CombinatorialSpecSet(basic_yaml_file, False)
compilers = spec_set._compiler_specs({
'gcc': {
'versions': ['4.2.1', '6.3.0']
}, 'clang': {
'versions': ['8.0', '3.8']
}})
assert len(list(compilers)) == 4
assert Spec('%gcc@4.2.1') in compilers
assert Spec('%gcc@6.3.0') in compilers
assert Spec('%clang@8.0') in compilers
assert Spec('%clang@3.8') in compilers
def test_package_specs():
spec_set = CombinatorialSpecSet(basic_yaml_file, False)
packages = spec_set._package_specs({
'gmake': {
'versions': ['4.0', '5.0']
},
'appres': {
'versions': ['1.0.4']
},
'allinea-reports': {
'versions': ['6.0.1', '6.0.3', '6.0.4']
}
})
assert Spec('gmake@4.0') in packages
assert Spec('gmake@5.0') in packages
assert Spec('appres@1.0.4') in packages
assert Spec('allinea-reports@6.0.1') in packages
assert Spec('allinea-reports@6.0.3') in packages
assert Spec('allinea-reports@6.0.4') in packages