zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #984 from epfl-scitas/module_file_explicit_load

Browse Source 
module files : explicit load and mnemonic suffixes
This commit is contained in:
Todd Gamblin 2016-06-26 23:33:48 -07:00 committed by GitHub
commit 73213ac59d
3 changed files with 185 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/spack/spack/config.py
View File

@ -307,6 +307,8 @@
'autoload': {'$ref': '#/definitions/dependency_selection'},
'prerequisites': {'$ref': '#/definitions/dependency_selection'},
'conflict': {'$ref': '#/definitions/array_of_strings'},
'load': {'$ref': '#/definitions/array_of_strings'},
'suffixes': {'$ref': '#/definitions/dictionary_of_strings'},
'environment': {
'type': 'object',
'default': {},

21
lib/spack/spack/modules.py
View File

@ -285,11 +285,18 @@ def use_name(self):
naming_tokens = self.tokens
naming_scheme = self.naming_scheme
name = naming_scheme.format(**naming_tokens)
name += '-' + self.spec.dag_hash(
) # Always append the hash to make the module file unique
# Not everybody is working on linux...
parts = name.split('/')
name = join_path(*parts)
# Add optional suffixes based on constraints
configuration, _ = parse_config_options(self)
suffixes = [name]
for constraint, suffix in configuration.get('suffixes', {}).items():
if constraint in self.spec:
suffixes.append(suffix)
# Always append the hash to make the module file unique
suffixes.append(self.spec.dag_hash())
name = '-'.join(suffixes)
return name
@property
@ -381,6 +388,8 @@ def write(self):
for x in filter_blacklisted(
module_configuration.pop('autoload', []), self.name):
module_file_content += self.autoload(x)
for x in module_configuration.pop('load', []):
module_file_content += self.autoload(x)
for x in filter_blacklisted(
module_configuration.pop('prerequisites', []), self.name):
module_file_content += self.prerequisite(x)
@ -402,8 +411,12 @@ def module_specific_content(self, configuration):
return tuple()
def autoload(self, spec):
m = type(self)(spec)
return self.autoload_format.format(module_file=m.use_name)
if not isinstance(spec, str):
m = type(self)(spec)
module_file = m.use_name
else:
module_file = spec
return self.autoload_format.format(module_file=module_file)
def prerequisite(self, spec):
m = type(self)(spec)

172
lib/spack/spack/test/modules.py
View File

@ -27,6 +27,7 @@
import StringIO
import spack.modules
import unittest
from spack.test.mock_packages_test import MockPackagesTest
FILE_REGISTRY = collections.defaultdict(StringIO.StringIO)
@ -67,6 +68,24 @@ def mock_open(filename, mode):
}
}
configuration_prerequisites_direct = {
'enable': ['tcl'],
'tcl': {
'all': {
'prerequisites': 'direct'
}
}
}
configuration_prerequisites_all = {
'enable': ['tcl'],
'tcl': {
'all': {
'prerequisites': 'all'
}
}
}
configuration_alter_environment = {
'enable': ['tcl'],
'tcl': {
@ -74,8 +93,13 @@ def mock_open(filename, mode):
'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']}
},
'platform=test target=x86_64': {
'environment': {'set': {'FOO': 'foo'},
'unset': ['BAR']}
'environment': {
'set': {'FOO': 'foo'},
'unset': ['BAR']
}
},
'platform=test target=x86_32': {
'load': ['foo/bar']
}
}
}
@ -83,7 +107,8 @@ def mock_open(filename, mode):
configuration_blacklist = {
'enable': ['tcl'],
'tcl': {
'blacklist': ['callpath'],
'whitelist': ['zmpi'],
'blacklist': ['callpath', 'mpi'],
'all': {
'autoload': 'direct'
}
@ -100,8 +125,68 @@ def mock_open(filename, mode):
}
}
configuration_wrong_conflicts = {
'enable': ['tcl'],
'tcl': {
'naming_scheme': '{name}/{version}-{compiler.name}',
'all': {
'conflict': ['{name}/{compiler.name}']
}
}
}
configuration_suffix = {
'enable': ['tcl'],
'tcl': {
'mpileaks': {
'suffixes': {
'+debug': 'foo',
'~debug': 'bar'
}
}
}
}
class HelperFunctionsTests(unittest.TestCase):
def test_update_dictionary_extending_list(self):
target = {
'foo': {
'a': 1,
'b': 2,
'd': 4
},
'bar': [1, 2, 4],
'baz': 'foobar'
}
update = {
'foo': {
'c': 3,
},
'bar': [3],
'baz': 'foobaz',
'newkey': {
'd': 4
}
}
spack.modules.update_dictionary_extending_lists(target, update)
self.assertTrue(len(target) == 4)
self.assertTrue(len(target['foo']) == 4)
self.assertTrue(len(target['bar']) == 4)
self.assertEqual(target['baz'], 'foobaz')
def test_inspect_path(self):
env = spack.modules.inspect_path('/usr')
names = [item.name for item in env]
self.assertTrue('PATH' in names)
self.assertTrue('LIBRARY_PATH' in names)
self.assertTrue('LD_LIBRARY_PATH' in names)
self.assertTrue('CPATH' in names)
class TclTests(MockPackagesTest):
def setUp(self):
super(TclTests, self).setUp()
self.configuration_obj = spack.modules.CONFIGURATION
@ -116,7 +201,6 @@ def tearDown(self):
def get_modulefile_content(self, spec):
spec.concretize()
print spec, '&&&&&'
generator = spack.modules.TclModule(spec)
generator.write()
content = FILE_REGISTRY[generator.file_name].split('\n')
@ -127,6 +211,8 @@ def test_simple_case(self):
spec = spack.spec.Spec('mpich@3.0.4')
content = self.get_modulefile_content(spec)
self.assertTrue('module-whatis "mpich @3.0.4"' in content)
self.assertRaises(TypeError, spack.modules.dependencies,
spec, 'non-existing-tag')
def test_autoload(self):
spack.modules.CONFIGURATION = configuration_autoload_direct
@ -141,11 +227,21 @@ def test_autoload(self):
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5)
self.assertEqual(len([x for x in content if 'module load ' in x]), 5)
def test_prerequisites(self):
spack.modules.CONFIGURATION = configuration_prerequisites_direct
spec = spack.spec.Spec('mpileaks arch=x86-linux')
content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'prereq' in x]), 2)
spack.modules.CONFIGURATION = configuration_prerequisites_all
spec = spack.spec.Spec('mpileaks arch=x86-linux')
content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'prereq' in x]), 5)
def test_alter_environment(self):
spack.modules.CONFIGURATION = configuration_alter_environment
spec = spack.spec.Spec('mpileaks platform=test target=x86_64')
content = self.get_modulefile_content(spec)
print content
self.assertEqual(
len([x
for x in content
@ -156,7 +252,6 @@ def test_alter_environment(self):
spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32')
content = self.get_modulefile_content(spec)
print content
self.assertEqual(
len([x
for x in content
@ -164,6 +259,10 @@ def test_alter_environment(self):
self.assertEqual(
len([x for x in content if 'setenv FOO "foo"' in x]), 0)
self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0)
self.assertEqual(
len([x for x in content if 'is-loaded foo/bar' in x]), 1)
self.assertEqual(
len([x for x in content if 'module load foo/bar' in x]), 1)
def test_blacklist(self):
spack.modules.CONFIGURATION = configuration_blacklist
@ -171,6 +270,13 @@ def test_blacklist(self):
content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1)
self.assertEqual(len([x for x in content if 'module load ' in x]), 1)
spec = spack.spec.Spec('callpath arch=x86-linux')
# Returns a StringIO instead of a string as no module file was written
self.assertRaises(AttributeError, self.get_modulefile_content, spec)
spec = spack.spec.Spec('zmpi arch=x86-linux')
content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1)
self.assertEqual(len([x for x in content if 'module load ' in x]), 1)
def test_conflicts(self):
spack.modules.CONFIGURATION = configuration_conflicts
@ -182,3 +288,57 @@ def test_conflicts(self):
len([x for x in content if x == 'conflict mpileaks']), 1)
self.assertEqual(
len([x for x in content if x == 'conflict intel/14.0.1']), 1)
spack.modules.CONFIGURATION = configuration_wrong_conflicts
self.assertRaises(SystemExit, self.get_modulefile_content, spec)
def test_suffixes(self):
spack.modules.CONFIGURATION = configuration_suffix
spec = spack.spec.Spec('mpileaks+debug arch=x86-linux')
spec.concretize()
generator = spack.modules.TclModule(spec)
self.assertTrue('foo' in generator.use_name)
spec = spack.spec.Spec('mpileaks~debug arch=x86-linux')
spec.concretize()
generator = spack.modules.TclModule(spec)
self.assertTrue('bar' in generator.use_name)
configuration_dotkit = {
'enable': ['dotkit'],
'dotkit': {
'all': {
'prerequisites': 'direct'
}
}
}
class DotkitTests(MockPackagesTest):
def setUp(self):
super(DotkitTests, self).setUp()
self.configuration_obj = spack.modules.CONFIGURATION
spack.modules.open = mock_open
# Make sure that a non-mocked configuration will trigger an error
spack.modules.CONFIGURATION = None
def tearDown(self):
del spack.modules.open
spack.modules.CONFIGURATION = self.configuration_obj
super(DotkitTests, self).tearDown()
def get_modulefile_content(self, spec):
spec.concretize()
generator = spack.modules.Dotkit(spec)
generator.write()
content = FILE_REGISTRY[generator.file_name].split('\n')
return content
def test_dotkit(self):
spack.modules.CONFIGURATION = configuration_dotkit
spec = spack.spec.Spec('mpileaks arch=x86-linux')
content = self.get_modulefile_content(spec)
self.assertTrue('#c spack' in content)
self.assertTrue('#d mpileaks @2.3' in content)