zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changed compiler yaml format

Browse Source
This commit is contained in:
Gregory Becker 2016-06-10 16:26:02 -07:00
parent 196737c217
commit 992bcac794
5 changed files with 86 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
lib/spack/spack/compilers/__init__.py
View File

@ -73,16 +73,14 @@ def _to_dict(compiler):
d = {} d = {}
d['spec'] = str(compiler.spec) d['spec'] = str(compiler.spec)
d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars ) d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars )
d['operating_system'] = compiler.operating_system.to_dict() d['operating_system'] = str(compiler.operating_system)
d['modules'] = compiler.modules d['modules'] = compiler.modules
if not compiler.alias: if compiler.alias:
yaml_text = yaml.dump( d['alias'] = compiler.alias
d, default_flow_style=True, width=sys.maxint)
sha = hashlib.sha1(yaml_text)
compiler.alias = base64.b32encode(sha.digest()).lower()[:8]
return { return {
compiler.alias: d 'compiler': d
} }
@ -91,11 +89,11 @@ def get_compiler_config(scope=None):
""" """
def init_compiler_config(): def init_compiler_config():
"""Compiler search used when Spack has no compilers.""" """Compiler search used when Spack has no compilers."""
config = {}
compilers = find_compilers() compilers = find_compilers()
compilers_dict = []
for compiler in compilers: for compiler in compilers:
config.update(_to_dict(compiler)) compilers_dict.append(_to_dict(compiler))
spack.config.update_config('compilers', config, scope=scope) spack.config.update_config('compilers', compilers_dict, scope=scope)
config = spack.config.get_config('compilers', scope=scope) config = spack.config.get_config('compilers', scope=scope)
# Update the configuration if there are currently no compilers # Update the configuration if there are currently no compilers
@ -127,7 +125,7 @@ def add_compilers_to_config(compilers, scope=None):
""" """
compiler_config = get_compiler_config(scope) compiler_config = get_compiler_config(scope)
for compiler in compilers: for compiler in compilers:
compiler_config = _to_dict(compiler) compiler_config.append(_to_dict(compiler))
spack.config.update_config('compilers', compiler_config, scope) spack.config.update_config('compilers', compiler_config, scope)
@ -167,8 +165,8 @@ def all_compilers_config(scope=None):
def all_compilers(scope=None): def all_compilers(scope=None):
# Return compiler specs from the merged config. # Return compiler specs from the merged config.
return [spack.spec.CompilerSpec(s['spec']) return [spack.spec.CompilerSpec(s['compiler']['spec'])
for s in all_compilers_config(scope).values()] for s in all_compilers_config(scope)]
def default_compiler(): def default_compiler():
@ -230,11 +228,10 @@ def compilers_for_spec(compiler_spec, scope=None, **kwargs):
def get_compilers(cspec): def get_compilers(cspec):
compilers = [] compilers = []
for aka, cmp in config.items(): for items in config:
if cmp['spec'] != str(cspec): if items['compiler']['spec'] != str(cspec):
continue continue
items = cmp items = items['compiler']
alias = aka
if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)):
raise InvalidCompilerConfigurationError(cspec) raise InvalidCompilerConfigurationError(cspec)
@ -258,6 +255,9 @@ def get_compilers(cspec):
else: else:
operating_system = None operating_system = None
alias = items['alias'] if 'alias' in items else None
flags = {} flags = {}
for f in spack.spec.FlagMap.valid_compiler_flags(): for f in spack.spec.FlagMap.valid_compiler_flags():
if f in items: if f in items:

24
lib/spack/spack/config.py
View File

@ -146,11 +146,9 @@
'additionalProperties': False, 'additionalProperties': False,
'patternProperties': { 'patternProperties': {
'compilers:?': { # optional colon for overriding site config. 'compilers:?': { # optional colon for overriding site config.
'type': 'object', 'type': 'array',
'default': {}, 'items': {
'additionalProperties': False, 'compiler': {
'patternProperties': {
r'\w[\w-]*': { # alias
'type': 'object', 'type': 'object',
'additionalProperties': False, 'additionalProperties': False,
'required': ['paths', 'spec', 'modules', 'operating_system'], 'required': ['paths', 'spec', 'modules', 'operating_system'],
@ -180,15 +178,10 @@
{'type' : 'null' }]}, {'type' : 'null' }]},
'ldlibs': { 'anyOf': [ {'type' : 'string' }, 'ldlibs': { 'anyOf': [ {'type' : 'string' },
{'type' : 'null' }]}}}, {'type' : 'null' }]}}},
'spec': { 'type': 'string'},#r'\w[\w-]*@\w[\w-]*' 'spec': { 'type': 'string'},
'operating_system': { 'operating_system': { 'type': 'string'},
'type': 'object', 'alias': { 'anyOf': [ {'type' : 'string'},
'required': ['name', 'version'], {'type' : 'null' }]},
'additionalProperties': False,
'properties': {
'name': {'type': 'string'},
'version': {'type': 'string'}
}},
'modules': { 'anyOf': [ {'type' : 'string'}, 'modules': { 'anyOf': [ {'type' : 'string'},
{'type' : 'null' }, {'type' : 'null' },
{'type': 'array'}, {'type': 'array'},
@ -591,8 +584,7 @@ def they_are(t):
# Source list is prepended (for precedence) # Source list is prepended (for precedence)
if they_are(list): if they_are(list):
seen = set(source) dest[:] = source + [x for x in dest if x not in source]
dest[:] = source + [x for x in dest if x not in seen]
return dest return dest
# Source dict is merged into dest. # Source dict is merged into dest.

82
lib/spack/spack/test/config.py
View File

@ -32,8 +32,8 @@
from spack.test.mock_packages_test import * from spack.test.mock_packages_test import *
# Some sample compiler config data # Some sample compiler config data
a_comps = { a_comps = [
'gcc473': { {'compiler': {
'paths': { 'paths': {
"cc" : "/gcc473", "cc" : "/gcc473",
"cxx": "/g++473", "cxx": "/g++473",
@ -42,12 +42,9 @@
}, },
'modules': None, 'modules': None,
'spec': 'gcc@4.7.3', 'spec': 'gcc@4.7.3',
'operating_system': { 'operating_system': 'CNL10'
'name': 'CNL', }},
'version': '10' {'compiler': {
}
},
'gcc450': {
'paths': { 'paths': {
"cc" : "/gcc450", "cc" : "/gcc450",
"cxx": "/g++450", "cxx": "/g++450",
@ -56,12 +53,9 @@
}, },
'modules': None, 'modules': None,
'spec': 'gcc@4.5.0', 'spec': 'gcc@4.5.0',
'operating_system': { 'operating_system': 'CNL10'
'name': 'CNL', }},
'version': '10' {'compiler': {
}
},
'clang33': {
'paths': { 'paths': {
"cc" : "<overwritten>", "cc" : "<overwritten>",
"cxx": "<overwritten>", "cxx": "<overwritten>",
@ -69,15 +63,12 @@
"fc" : '<overwritten>' }, "fc" : '<overwritten>' },
'modules': None, 'modules': None,
'spec': 'clang@3.3', 'spec': 'clang@3.3',
'operating_system': { 'operating_system': 'CNL10'
'name': 'CNL', }}
'version': '10' ]
}
}
}
b_comps = { b_comps = [
'icc100': { {'compiler': {
'paths': { 'paths': {
"cc" : "/icc100", "cc" : "/icc100",
"cxx": "/icp100", "cxx": "/icp100",
@ -86,12 +77,9 @@
}, },
'modules': None, 'modules': None,
'spec': 'icc@10.0', 'spec': 'icc@10.0',
'operating_system': { 'operating_system': 'CNL10'
'name': 'CNL', }},
'version': '10' {'compiler': {
}
},
'icc111': {
'paths': { 'paths': {
"cc" : "/icc111", "cc" : "/icc111",
"cxx": "/icp111", "cxx": "/icp111",
@ -100,12 +88,9 @@
}, },
'modules': None, 'modules': None,
'spec': 'icc@11.1', 'spec': 'icc@11.1',
'operating_system': { 'operating_system': 'CNL10'
'name': 'CNL', }},
'version': '10' {'compiler': {
}
},
'clang33': {
'paths': { 'paths': {
"cc" : "<overwritten>", "cc" : "<overwritten>",
"cxx": "<overwritten>", "cxx": "<overwritten>",
@ -113,12 +98,9 @@
"fc" : '<overwritten>' }, "fc" : '<overwritten>' },
'modules': None, 'modules': None,
'spec': 'clang@3.3', 'spec': 'clang@3.3',
'operating_system': { 'operating_system': 'CNL10'
'name': 'CNL', }}
'version': '10' ]
}
}
}
# Some Sample repo data # Some Sample repo data
repos_low = [ "/some/path" ] repos_low = [ "/some/path" ]
@ -143,15 +125,21 @@ def check_config(self, comps, *compiler_names):
config = spack.config.get_config('compilers') config = spack.config.get_config('compilers')
compiler_list = ['cc', 'cxx', 'f77', 'fc'] compiler_list = ['cc', 'cxx', 'f77', 'fc']
param_list = ['modules', 'paths', 'spec', 'operating_system'] param_list = ['modules', 'paths', 'spec', 'operating_system']
for alias, compiler in config.items(): for compiler in config:
if compiler['spec'] in compiler_names: conf = compiler['compiler']
if conf['spec'] in compiler_names:
comp = None
for c in comps:
if c['compiler']['spec'] == conf['spec']:
comp = c['compiler']
break
if not comp:
self.fail('Bad config spec')
for p in param_list: for p in param_list:
expected = comps[alias][p] self.assertEqual(conf[p], comp[p])
actual = config[alias][p]
self.assertEqual(expected, actual)
for c in compiler_list: for c in compiler_list:
expected = comps[alias]['paths'][c] expected = comp['paths'][c]
actual = config[alias]['paths'][c] actual = conf['paths'][c]
self.assertEqual(expected, actual) self.assertEqual(expected, actual)
def test_write_list_in_memory(self): def test_write_list_in_memory(self):

72
lib/spack/spack/test/mock_packages_test.py
View File

@ -46,132 +46,108 @@
mock_compiler_config = """\ mock_compiler_config = """\
compilers: compilers:
clang3.3GENLINUX: - compiler:
spec: clang@3.3 spec: clang@3.3
operating_system: operating_system: {0}{1}
name: {0}
version: '{1}'
paths: paths:
cc: /path/to/clang cc: /path/to/clang
cxx: /path/to/clang++ cxx: /path/to/clang++
f77: None f77: None
fc: None fc: None
modules: 'None' modules: 'None'
gcc4.5GENLINUX: - compiler:
spec: gcc@4.5.0 spec: gcc@4.5.0
operating_system: operating_system: {0}{1}
name: {0}
version: '{1}'
paths: paths:
cc: /path/to/gcc cc: /path/to/gcc
cxx: /path/to/g++ cxx: /path/to/g++
f77: None f77: None
fc: None fc: None
modules: 'None' modules: 'None'
clang3.3CNL: - compiler:
spec: clang@3.3 spec: clang@3.3
operating_system: operating_system: CNL10
name: CNL
version: '10'
paths: paths:
cc: /path/to/clang cc: /path/to/clang
cxx: /path/to/clang++ cxx: /path/to/clang++
f77: None f77: None
fc: None fc: None
modules: 'None' modules: 'None'
clang3.3SUSE: - compiler:
spec: clang@3.3 spec: clang@3.3
operating_system: operating_system: SuSE11
name: SuSE
version: '11'
paths: paths:
cc: /path/to/clang cc: /path/to/clang
cxx: /path/to/clang++ cxx: /path/to/clang++
f77: None f77: None
fc: None fc: None
modules: 'None' modules: 'None'
clang3.3RHL: - compiler:
spec: clang@3.3 spec: clang@3.3
operating_system: operating_system: redhat6
name: redhat
version: '6'
paths: paths:
cc: /path/to/clang cc: /path/to/clang
cxx: /path/to/clang++ cxx: /path/to/clang++
f77: None f77: None
fc: None fc: None
modules: 'None' modules: 'None'
clang3.3OSX: - compiler:
spec: clang@3.3 spec: clang@3.3
operating_system: operating_system: yosemite
name: yosemite
version: '10.10'
paths: paths:
cc: /path/to/clang cc: /path/to/clang
cxx: /path/to/clang++ cxx: /path/to/clang++
f77: None f77: None
fc: None fc: None
modules: 'None' modules: 'None'
gcc4.5.0CNL: - compiler:
paths: paths:
cc: /path/to/gcc cc: /path/to/gcc
cxx: /path/to/g++ cxx: /path/to/g++
f77: /path/to/gfortran f77: /path/to/gfortran
fc: /path/to/gfortran fc: /path/to/gfortran
operating_system: operating_system: CNL10
name: CNL
version: '10'
spec: gcc@4.5.0 spec: gcc@4.5.0
modules: 'None' modules: 'None'
gcc4.5.0SUSE: - compiler:
paths: paths:
cc: /path/to/gcc cc: /path/to/gcc
cxx: /path/to/g++ cxx: /path/to/g++
f77: /path/to/gfortran f77: /path/to/gfortran
fc: /path/to/gfortran fc: /path/to/gfortran
operating_system: operating_system: SuSE11
name: SuSE
version: '11'
spec: gcc@4.5.0 spec: gcc@4.5.0
modules: 'None' modules: 'None'
gcc4.5.0RHL: - compiler:
paths: paths:
cc: /path/to/gcc cc: /path/to/gcc
cxx: /path/to/g++ cxx: /path/to/g++
f77: /path/to/gfortran f77: /path/to/gfortran
fc: /path/to/gfortran fc: /path/to/gfortran
operating_system: operating_system: redhat6
name: redhat
version: '6'
spec: gcc@4.5.0 spec: gcc@4.5.0
modules: 'None' modules: 'None'
gcc4.5.0OSX: - compiler:
paths: paths:
cc: /path/to/gcc cc: /path/to/gcc
cxx: /path/to/g++ cxx: /path/to/g++
f77: /path/to/gfortran f77: /path/to/gfortran
fc: /path/to/gfortran fc: /path/to/gfortran
operating_system: operating_system: yosemite
name: yosemite
version: '10.10'
spec: gcc@4.5.0 spec: gcc@4.5.0
modules: 'None' modules: 'None'
gcc4.5.0ELCAP: - compiler:
paths: paths:
cc: /path/to/gcc cc: /path/to/gcc
cxx: /path/to/g++ cxx: /path/to/g++
f77: /path/to/gfortran f77: /path/to/gfortran
fc: /path/to/gfortran fc: /path/to/gfortran
operating_system: operating_system: elcapitan
name: elcapitan
version: '10.11'
spec: gcc@4.5.0 spec: gcc@4.5.0
modules: 'None' modules: 'None'
clang3.3ELCAP: - compiler:
spec: clang@3.3 spec: clang@3.3
operating_system: operating_system: elcapitan
name: elcapitan
version: '10.11'
paths: paths:
cc: /path/to/clang cc: /path/to/clang
cxx: /path/to/clang++ cxx: /path/to/clang++

4
lib/spack/spack/yaml_version_check.py
View File

@ -44,9 +44,9 @@ def check_compiler_yaml_version():
data = syaml.load(f) data = syaml.load(f)
if data: if data:
compilers = data['compilers'].items() compilers = data['compilers']
if len(compilers) > 0: if len(compilers) > 0:
if 'operating_system' not in compilers[0][1]: if 'operating_system' not in compilers[0]['compiler']:
new_file = os.path.join(scope.path, '_old_compilers.yaml') new_file = os.path.join(scope.path, '_old_compilers.yaml')
tty.warn('%s in out of date compilers format. ' tty.warn('%s in out of date compilers format. '
'Moved to %s. Spack automatically generate ' 'Moved to %s. Spack automatically generate '