modules: use projections format for naming schemes (#16629)

* update tcl naming_scheme to use projections

* add projections to lmod modules

lib/spack/docs/build_systems/intelpackage.rst

@@ -1055,6 +1055,6 @@ Footnotes
    2. Set the hash length in ``install-path-scheme``, also in ``config.yaml``
       (:ref:`q.v. <config-yaml>`).
    3. You will want to set the *same* hash length for
-      :ref:`tcl module files <modules-naming-scheme>`
+      :ref:`module files <modules-projections>`
-      if you have Spack produce them for you, under ``naming_scheme`` in
+      if you have Spack produce them for you, under ``projections`` in
-      ``modules.yaml``.  Other module dialects cannot be altered in this manner.
+      ``modules.yaml``.

lib/spack/docs/module_file_support.rst

@@ -459,14 +459,14 @@ is compiled with ``gcc@4.4.7``, with the only exception of any ``gcc``
 or any ``llvm`` installation.
 
 
-.. _modules-naming-scheme:
+.. _modules-projections:
 
-"""""""""""""""""""""""""""
+"""""""""""""""""""""""""""""""
-Customize the naming scheme
+Customize the naming of modules
-"""""""""""""""""""""""""""
+"""""""""""""""""""""""""""""""
 
 The names of environment modules generated by spack are not always easy to
-fully comprehend due to the long hash in the name. There are two module
+fully comprehend due to the long hash in the name. There are three module
 configuration options to help with that. The first is a global setting to
 adjust the hash length. It can be set anywhere from 0 to 32 and has a default
 length of 7. This is the representation of the hash in the module file name and
@@ -500,16 +500,42 @@ version of python a set of python extensions is associated with. Likewise, the
 ``openblas`` string is attached to any program that has openblas in the spec,
 most likely via the ``+blas`` variant specification.
 
-.. note::
+The most heavyweight solution to module naming is to change the entire
-   TCL module files
+naming convention for module files. This uses the projections format
-     A modification that is specific to ``tcl`` module files is the possibility
+covered in :ref:`adding_projections_to_views`.
-     to change the naming scheme of modules.
 
 .. code-block:: yaml
 
   modules:
     tcl:
-           naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}'
+      projections:
+        all: '{name}/{version}-{compiler.name}-{compiler.version}-module'
+        ^mpi: '{name}/{version}-{^mpi.name}-{^mpi.version}-{compiler.name}-{compiler.version}-module'
+
+will create module files that are nested in directories by package
+name, contain the version and compiler name and version, and have the
+word ``module`` before the hash for all specs that do not depend on
+mpi, and will have the same information plus the MPI implementation
+name and version for all packages that depend on mpi.
+
+When specifying module names by projection for Lmod modules, we
+recommend NOT including names of dependencies (e.g., MPI, compilers)
+that are already in the LMod hierarchy.
+
+
+
+.. note::
+   TCL modules
+     TCL modules also allow for explicit conflicts between modulefiles.
+
+     .. code-block:: yaml
+
+        modules:
+          enable:
+            - tcl
+          tcl:
+            projections:
+              all: '{name}/{version}-{compiler.name}-{compiler.version}'
             all:
               conflict:
                 - '{name}'

lib/spack/spack/filesystem_view.py

@@ -22,6 +22,7 @@
 import spack.spec
 import spack.store
 import spack.schema.projections
+import spack.projections
 import spack.config
 from spack.error import SpackError
 from spack.directory_layout import ExtensionAlreadyInstalledError
@@ -470,14 +471,9 @@ def get_projection_for_spec(self, spec):
         if spec.package.extendee_spec:
             locator_spec = spec.package.extendee_spec
 
-        all_fmt_str = None
+        proj = spack.projections.get_projection(self.projections, locator_spec)
-        for spec_like, fmt_str in self.projections.items():
+        if proj:
-            if locator_spec.satisfies(spec_like, strict=True):
+            return os.path.join(self._root, locator_spec.format(proj))
-                return os.path.join(self._root, locator_spec.format(fmt_str))
-            elif spec_like == 'all':
-                all_fmt_str = fmt_str
-        if all_fmt_str:
-            return os.path.join(self._root, locator_spec.format(all_fmt_str))
         return self._root
 
     def get_all_specs(self):

lib/spack/spack/modules/common.py

@@ -41,6 +41,7 @@
 import spack.error
 import spack.paths
 import spack.schema.environment
+import spack.projections as proj
 import spack.tengine as tengine
 import spack.util.environment
 import spack.util.file_permissions as fp
@@ -381,6 +382,9 @@ class BaseConfiguration(object):
     querying easier. It needs to be sub-classed for specific module types.
     """
 
+    default_projections = {
+        'all': '{name}-{version}-{compiler.name}-{compiler.version}'}
+
     def __init__(self, spec):
         # Module where type(self) is defined
         self.module = inspect.getmodule(self)
@@ -391,19 +395,18 @@ def __init__(self, spec):
         self.conf = merge_config_rules(self.module.configuration(), self.spec)
 
     @property
-    def naming_scheme(self):
+    def projections(self):
-        """Naming scheme suitable for non-hierarchical layouts"""
+        """Projection from specs to module names"""
-        scheme = self.module.configuration().get(
+        projections = self.module.configuration().get(
-            'naming_scheme',
+            'projections', self.default_projections)
-            '{name}-{version}-{compiler.name}-{compiler.version}'
-        )
 
         # Ensure the named tokens we are expanding are allowed, see
         # issue #2884 for reference
         msg = 'some tokens cannot be part of the module naming scheme'
-        _check_tokens_are_valid(scheme, message=msg)
+        for projection in projections.values():
+            _check_tokens_are_valid(projection, message=msg)
 
-        return scheme
+        return projections
 
     @property
     def template(self):
@@ -551,7 +554,11 @@ def use_name(self):
         to console to use it. This implementation fits the needs of most
         non-hierarchical layouts.
         """
-        name = self.spec.format(self.conf.naming_scheme)
+        projection = proj.get_projection(self.conf.projections, self.spec)
+        if not projection:
+            projection = self.conf.default_projections['all']
+
+        name = self.spec.format(projection)
         # Not everybody is working on linux...
         parts = name.split('/')
         name = os.path.join(*parts)

lib/spack/spack/modules/lmod.py

@@ -91,6 +91,7 @@ def guess_core_compilers(store=False):
 
 class LmodConfiguration(BaseConfiguration):
     """Configuration class for lmod module files."""
+    default_projections = {'all': os.path.join('{name}', '{version}')}
 
     @property
     def core_compilers(self):
@@ -243,18 +244,6 @@ def filename(self):
         )
         return fullname
 
-    @property
-    def use_name(self):
-        """Returns the 'use' name of the module i.e. the name you have to type
-        to console to use it.
-        """
-        # Package name and version
-        base = os.path.join("{name}", "{version}")
-        name_parts = [self.spec.format(base)]
-        # The remaining elements are filename suffixes
-        name_parts.extend(self.conf.suffixes)
-        return '-'.join(name_parts)
-
     def token_to_path(self, name, value):
         """Transforms a hierarchy token into the corresponding path part.
 

lib/spack/spack/modules/tcl.py

@@ -12,6 +12,7 @@
 import llnl.util.tty as tty
 
 import spack.config
+import spack.projections as proj
 import spack.tengine as tengine
 from .common import BaseConfiguration, BaseFileLayout
 from .common import BaseContext, BaseModuleFileWriter
@@ -72,12 +73,12 @@ def prerequisites(self):
     def conflicts(self):
         """List of conflicts for the tcl module file."""
         fmts = []
-        naming_scheme = self.conf.naming_scheme
+        projection = proj.get_projection(self.conf.projections, self.spec)
         f = string.Formatter()
         for item in self.conf.conflicts:
             if len([x for x in f.parse(item)]) > 1:
                 for naming_dir, conflict_dir in zip(
-                        naming_scheme.split('/'), item.split('/')
+                        projection.split('/'), item.split('/')
                 ):
                     if naming_dir != conflict_dir:
                         message = 'conflict scheme does not match naming '
@@ -87,7 +88,7 @@ def conflicts(self):
                         message += '** You may want to check your '
                         message += '`modules.yaml` configuration file **\n'
                         tty.error(message.format(spec=self.spec,
-                                                 nformat=naming_scheme,
+                                                 nformat=projection,
                                                  cformat=item))
                         raise SystemExit('Module generation aborted.')
                 item = self.spec.format(item)

lib/spack/spack/projections.py

@@ -0,0 +1,16 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+def get_projection(projections, spec):
+    """
+    Get the projection for a spec from a projections dict.
+    """
+    all_projection = None
+    for spec_like, projection in projections.items():
+        if spec.satisfies(spec_like, strict=True):
+            return projection
+        elif spec_like == 'all':
+            all_projection = projection
+    return all_projection

lib/spack/spack/schema/modules.py

@@ -9,7 +9,7 @@
    :lines: 13-
 """
 import spack.schema.environment
-
+import spack.schema.projections
 
 #: Matches a spec or a multi-valued variant but not another
 #: valid keyword.
@@ -17,7 +17,7 @@
 #: THIS NEEDS TO BE UPDATED FOR EVERY NEW KEYWORD THAT
 #: IS ADDED IMMEDIATELY BELOW THE MODULE TYPE ATTRIBUTE
 spec_regex = r'(?!hierarchy|core_specs|verbose|hash_length|whitelist|' \
-             r'blacklist|naming_scheme|core_compilers|all)(^\w[\w-]*)'
+             r'blacklist|projections|core_compilers|all)(^\w[\w-]*)'
 
 #: Matches an anonymous spec, i.e. a spec without a root name
 anonymous_spec_regex = r'^[\^@%+~]'
@@ -72,6 +72,8 @@
     }
 }
 
+projections_scheme = spack.schema.projections.properties['projections']
+
 module_type_configuration = {
     'type': 'object',
     'default': {},
@@ -92,9 +94,7 @@
                 'type': 'boolean',
                 'default': False
             },
-            'naming_scheme': {
+            'projections': projections_scheme,
-                'type': 'string'  # Can we be more specific here?
-            },
             'all': module_file_configuration,
         }
         },

lib/spack/spack/test/data/modules/lmod/projections.yaml

@@ -0,0 +1,6 @@
+enable:
+  - lmod
+lmod:
+  projections:
+    all: '{name}/v{version}'
+    mpileaks: '{name}-mpiprojection'

lib/spack/spack/test/data/modules/tcl/conflicts.yaml

@@ -1,7 +1,8 @@
 enable:
   - tcl
 tcl:
-  naming_scheme: '{name}/{version}-{compiler.name}'
+  projections:
+    all: '{name}/{version}-{compiler.name}'
   all:
     conflict:
       - '{name}'

lib/spack/spack/test/data/modules/tcl/invalid_naming_scheme.yaml

@@ -2,4 +2,5 @@ enable:
   - tcl
 tcl:
   # {variants} is not allowed in the naming scheme, see #2884
-  naming_scheme: '{name}/{version}-{compiler.name}-{variants}'
+  projections:
+    all: '{name}/{version}-{compiler.name}-{variants}'

lib/spack/spack/test/data/modules/tcl/projections.yaml

@@ -0,0 +1,6 @@
+enable:
+  - tcl
+tcl:
+  projections:
+    all: '{name}/{version}-{compiler.name}'
+    mpileaks: '{name}-mpiprojection'

lib/spack/spack/test/data/modules/tcl/wrong_conflicts.yaml

@@ -1,7 +1,8 @@
 enable:
   - tcl
 tcl:
-  naming_scheme: '{name}/{version}-{compiler.name}'
+  projections:
+    all: '{name}/{version}-{compiler.name}'
   all:
     conflict:
       - '{name}/{compiler.name}'

lib/spack/spack/test/modules/lmod.py

@@ -280,3 +280,39 @@ def test_only_generic_microarchitectures_in_root(
         assert str(spec.target.family) in writer.layout.arch_dirname
         if spec.target.family != spec.target:
             assert str(spec.target) not in writer.layout.arch_dirname
+
+    def test_projections_specific(self, factory, module_configuration):
+        """Tests reading the correct naming scheme."""
+
+        # This configuration has no error, so check the conflicts directives
+        # are there
+        module_configuration('projections')
+
+        # Test we read the expected configuration for the naming scheme
+        writer, _ = factory('mpileaks')
+        expected = {
+            'all': '{name}/v{version}',
+            'mpileaks': '{name}-mpiprojection'
+        }
+
+        assert writer.conf.projections == expected
+        projection = writer.spec.format(writer.conf.projections['mpileaks'])
+        assert projection in writer.layout.use_name
+
+    def test_projections_all(self, factory, module_configuration):
+        """Tests reading the correct naming scheme."""
+
+        # This configuration has no error, so check the conflicts directives
+        # are there
+        module_configuration('projections')
+
+        # Test we read the expected configuration for the naming scheme
+        writer, _ = factory('libelf')
+        expected = {
+            'all': '{name}/v{version}',
+            'mpileaks': '{name}-mpiprojection'
+        }
+
+        assert writer.conf.projections == expected
+        projection = writer.spec.format(writer.conf.projections['all'])
+        assert projection in writer.layout.use_name

lib/spack/spack/test/modules/tcl.py

@@ -142,18 +142,41 @@ def test_blacklist(self, modulefile_content, module_configuration):
         assert len([x for x in content if 'is-loaded' in x]) == 1
         assert len([x for x in content if 'module load ' in x]) == 1
 
-    def test_naming_scheme(self, factory, module_configuration):
+    def test_projections_specific(self, factory, module_configuration):
         """Tests reading the correct naming scheme."""
 
         # This configuration has no error, so check the conflicts directives
         # are there
-        module_configuration('conflicts')
+        module_configuration('projections')
 
         # Test we read the expected configuration for the naming scheme
         writer, _ = factory('mpileaks')
-        expected = '{name}/{version}-{compiler.name}'
+        expected = {
+            'all': '{name}/{version}-{compiler.name}',
+            'mpileaks': '{name}-mpiprojection'
+        }
 
-        assert writer.conf.naming_scheme == expected
+        assert writer.conf.projections == expected
+        projection = writer.spec.format(writer.conf.projections['mpileaks'])
+        assert projection in writer.layout.use_name
+
+    def test_projections_all(self, factory, module_configuration):
+        """Tests reading the correct naming scheme."""
+
+        # This configuration has no error, so check the conflicts directives
+        # are there
+        module_configuration('projections')
+
+        # Test we read the expected configuration for the naming scheme
+        writer, _ = factory('libelf')
+        expected = {
+            'all': '{name}/{version}-{compiler.name}',
+            'mpileaks': '{name}-mpiprojection'
+        }
+
+        assert writer.conf.projections == expected
+        projection = writer.spec.format(writer.conf.projections['all'])
+        assert projection in writer.layout.use_name
 
     def test_invalid_naming_scheme(self, factory, module_configuration):
         """Tests the evaluation of an invalid naming scheme."""