Merge pull request #984 from epfl-scitas/module_file_explicit_load

module files : explicit load and mnemonic suffixes

lib/spack/spack/config.py

@@ -307,6 +307,8 @@
                     'autoload': {'$ref': '#/definitions/dependency_selection'},
                     'prerequisites': {'$ref': '#/definitions/dependency_selection'},
                     'conflict': {'$ref': '#/definitions/array_of_strings'},
+                    'load': {'$ref': '#/definitions/array_of_strings'},
+                    'suffixes': {'$ref': '#/definitions/dictionary_of_strings'},
                     'environment': {
                         'type': 'object',
                         'default': {},

lib/spack/spack/modules.py

@@ -285,11 +285,18 @@ def use_name(self):
         naming_tokens = self.tokens
         naming_scheme = self.naming_scheme
         name = naming_scheme.format(**naming_tokens)
-        name += '-' + self.spec.dag_hash(
-        )  # Always append the hash to make the module file unique
         # Not everybody is working on linux...
         parts = name.split('/')
         name = join_path(*parts)
+        # Add optional suffixes based on constraints
+        configuration, _ = parse_config_options(self)
+        suffixes = [name]
+        for constraint, suffix in configuration.get('suffixes', {}).items():
+            if constraint in self.spec:
+                suffixes.append(suffix)
+        # Always append the hash to make the module file unique
+        suffixes.append(self.spec.dag_hash())
+        name = '-'.join(suffixes)
         return name
 
     @property
@@ -381,6 +388,8 @@ def write(self):
         for x in filter_blacklisted(
                 module_configuration.pop('autoload', []), self.name):
             module_file_content += self.autoload(x)
+        for x in module_configuration.pop('load', []):
+            module_file_content += self.autoload(x)
         for x in filter_blacklisted(
                 module_configuration.pop('prerequisites', []), self.name):
             module_file_content += self.prerequisite(x)
@@ -402,8 +411,12 @@ def module_specific_content(self, configuration):
         return tuple()
 
     def autoload(self, spec):
+        if not isinstance(spec, str):
             m = type(self)(spec)
-        return self.autoload_format.format(module_file=m.use_name)
+            module_file = m.use_name
+        else:
+            module_file = spec
+        return self.autoload_format.format(module_file=module_file)
 
     def prerequisite(self, spec):
         m = type(self)(spec)

lib/spack/spack/test/modules.py

@@ -27,6 +27,7 @@
 
 import StringIO
 import spack.modules
+import unittest
 from spack.test.mock_packages_test import MockPackagesTest
 
 FILE_REGISTRY = collections.defaultdict(StringIO.StringIO)
@@ -67,6 +68,24 @@ def mock_open(filename, mode):
     }
 }
 
+configuration_prerequisites_direct = {
+    'enable': ['tcl'],
+    'tcl': {
+        'all': {
+            'prerequisites': 'direct'
+        }
+    }
+}
+
+configuration_prerequisites_all = {
+    'enable': ['tcl'],
+    'tcl': {
+        'all': {
+            'prerequisites': 'all'
+        }
+    }
+}
+
 configuration_alter_environment = {
     'enable': ['tcl'],
     'tcl': {
@@ -74,8 +93,13 @@ def mock_open(filename, mode):
             'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']}
         },
         'platform=test target=x86_64': {
-            'environment': {'set': {'FOO': 'foo'},
+            'environment': {
-                            'unset': ['BAR']}
+                'set': {'FOO': 'foo'},
+                'unset': ['BAR']
+            }
+        },
+        'platform=test target=x86_32': {
+            'load': ['foo/bar']
         }
     }
 }
@@ -83,7 +107,8 @@ def mock_open(filename, mode):
 configuration_blacklist = {
     'enable': ['tcl'],
     'tcl': {
-        'blacklist': ['callpath'],
+        'whitelist': ['zmpi'],
+        'blacklist': ['callpath', 'mpi'],
         'all': {
             'autoload': 'direct'
         }
@@ -100,8 +125,68 @@ def mock_open(filename, mode):
     }
 }
 
+configuration_wrong_conflicts = {
+    'enable': ['tcl'],
+    'tcl': {
+        'naming_scheme': '{name}/{version}-{compiler.name}',
+        'all': {
+            'conflict': ['{name}/{compiler.name}']
+        }
+    }
+}
+
+configuration_suffix = {
+    'enable': ['tcl'],
+    'tcl': {
+        'mpileaks': {
+            'suffixes': {
+                '+debug': 'foo',
+                '~debug': 'bar'
+            }
+        }
+    }
+}
+
+
+class HelperFunctionsTests(unittest.TestCase):
+
+    def test_update_dictionary_extending_list(self):
+        target = {
+            'foo': {
+                'a': 1,
+                'b': 2,
+                'd': 4
+            },
+            'bar': [1, 2, 4],
+            'baz': 'foobar'
+        }
+        update = {
+            'foo': {
+                'c': 3,
+            },
+            'bar': [3],
+            'baz': 'foobaz',
+            'newkey': {
+                'd': 4
+            }
+        }
+        spack.modules.update_dictionary_extending_lists(target, update)
+        self.assertTrue(len(target) == 4)
+        self.assertTrue(len(target['foo']) == 4)
+        self.assertTrue(len(target['bar']) == 4)
+        self.assertEqual(target['baz'], 'foobaz')
+
+    def test_inspect_path(self):
+        env = spack.modules.inspect_path('/usr')
+        names = [item.name for item in env]
+        self.assertTrue('PATH' in names)
+        self.assertTrue('LIBRARY_PATH' in names)
+        self.assertTrue('LD_LIBRARY_PATH' in names)
+        self.assertTrue('CPATH' in names)
+
 
 class TclTests(MockPackagesTest):
+
     def setUp(self):
         super(TclTests, self).setUp()
         self.configuration_obj = spack.modules.CONFIGURATION
@@ -116,7 +201,6 @@ def tearDown(self):
 
     def get_modulefile_content(self, spec):
         spec.concretize()
-        print spec, '&&&&&'
         generator = spack.modules.TclModule(spec)
         generator.write()
         content = FILE_REGISTRY[generator.file_name].split('\n')
@@ -127,6 +211,8 @@ def test_simple_case(self):
         spec = spack.spec.Spec('mpich@3.0.4')
         content = self.get_modulefile_content(spec)
         self.assertTrue('module-whatis "mpich @3.0.4"' in content)
+        self.assertRaises(TypeError, spack.modules.dependencies,
+                          spec, 'non-existing-tag')
 
     def test_autoload(self):
         spack.modules.CONFIGURATION = configuration_autoload_direct
@@ -141,11 +227,21 @@ def test_autoload(self):
         self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5)
         self.assertEqual(len([x for x in content if 'module load ' in x]), 5)
 
+    def test_prerequisites(self):
+        spack.modules.CONFIGURATION = configuration_prerequisites_direct
+        spec = spack.spec.Spec('mpileaks arch=x86-linux')
+        content = self.get_modulefile_content(spec)
+        self.assertEqual(len([x for x in content if 'prereq' in x]), 2)
+
+        spack.modules.CONFIGURATION = configuration_prerequisites_all
+        spec = spack.spec.Spec('mpileaks arch=x86-linux')
+        content = self.get_modulefile_content(spec)
+        self.assertEqual(len([x for x in content if 'prereq' in x]), 5)
+
     def test_alter_environment(self):
         spack.modules.CONFIGURATION = configuration_alter_environment
         spec = spack.spec.Spec('mpileaks platform=test target=x86_64')
         content = self.get_modulefile_content(spec)
-        print content
         self.assertEqual(
             len([x
                  for x in content
@@ -156,7 +252,6 @@ def test_alter_environment(self):
 
         spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32')
         content = self.get_modulefile_content(spec)
-        print content
         self.assertEqual(
             len([x
                  for x in content
@@ -164,6 +259,10 @@ def test_alter_environment(self):
         self.assertEqual(
             len([x for x in content if 'setenv FOO "foo"' in x]), 0)
         self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0)
+        self.assertEqual(
+            len([x for x in content if 'is-loaded foo/bar' in x]), 1)
+        self.assertEqual(
+            len([x for x in content if 'module load foo/bar' in x]), 1)
 
     def test_blacklist(self):
         spack.modules.CONFIGURATION = configuration_blacklist
@@ -171,6 +270,13 @@ def test_blacklist(self):
         content = self.get_modulefile_content(spec)
         self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1)
         self.assertEqual(len([x for x in content if 'module load ' in x]), 1)
+        spec = spack.spec.Spec('callpath arch=x86-linux')
+        # Returns a StringIO instead of a string as no module file was written
+        self.assertRaises(AttributeError, self.get_modulefile_content, spec)
+        spec = spack.spec.Spec('zmpi arch=x86-linux')
+        content = self.get_modulefile_content(spec)
+        self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1)
+        self.assertEqual(len([x for x in content if 'module load ' in x]), 1)
 
     def test_conflicts(self):
         spack.modules.CONFIGURATION = configuration_conflicts
@@ -182,3 +288,57 @@ def test_conflicts(self):
             len([x for x in content if x == 'conflict mpileaks']), 1)
         self.assertEqual(
             len([x for x in content if x == 'conflict intel/14.0.1']), 1)
+
+        spack.modules.CONFIGURATION = configuration_wrong_conflicts
+        self.assertRaises(SystemExit, self.get_modulefile_content, spec)
+
+    def test_suffixes(self):
+        spack.modules.CONFIGURATION = configuration_suffix
+        spec = spack.spec.Spec('mpileaks+debug arch=x86-linux')
+        spec.concretize()
+        generator = spack.modules.TclModule(spec)
+        self.assertTrue('foo' in generator.use_name)
+
+        spec = spack.spec.Spec('mpileaks~debug arch=x86-linux')
+        spec.concretize()
+        generator = spack.modules.TclModule(spec)
+        self.assertTrue('bar' in generator.use_name)
+
+
+configuration_dotkit = {
+    'enable': ['dotkit'],
+    'dotkit': {
+        'all': {
+            'prerequisites': 'direct'
+        }
+    }
+}
+
+
+class DotkitTests(MockPackagesTest):
+
+    def setUp(self):
+        super(DotkitTests, self).setUp()
+        self.configuration_obj = spack.modules.CONFIGURATION
+        spack.modules.open = mock_open
+        # Make sure that a non-mocked configuration will trigger an error
+        spack.modules.CONFIGURATION = None
+
+    def tearDown(self):
+        del spack.modules.open
+        spack.modules.CONFIGURATION = self.configuration_obj
+        super(DotkitTests, self).tearDown()
+
+    def get_modulefile_content(self, spec):
+        spec.concretize()
+        generator = spack.modules.Dotkit(spec)
+        generator.write()
+        content = FILE_REGISTRY[generator.file_name].split('\n')
+        return content
+
+    def test_dotkit(self):
+        spack.modules.CONFIGURATION = configuration_dotkit
+        spec = spack.spec.Spec('mpileaks arch=x86-linux')
+        content = self.get_modulefile_content(spec)
+        self.assertTrue('#c spack' in content)
+        self.assertTrue('#d mpileaks @2.3' in content)