modules: use projections format for naming schemes (#16629)
* update tcl naming_scheme to use projections * add projections to lmod modules
This commit is contained in:
parent
955a3db206
commit
32a9adcf60
@ -1055,6 +1055,6 @@ Footnotes
|
|||||||
2. Set the hash length in ``install-path-scheme``, also in ``config.yaml``
|
2. Set the hash length in ``install-path-scheme``, also in ``config.yaml``
|
||||||
(:ref:`q.v. <config-yaml>`).
|
(:ref:`q.v. <config-yaml>`).
|
||||||
3. You will want to set the *same* hash length for
|
3. You will want to set the *same* hash length for
|
||||||
:ref:`tcl module files <modules-naming-scheme>`
|
:ref:`module files <modules-projections>`
|
||||||
if you have Spack produce them for you, under ``naming_scheme`` in
|
if you have Spack produce them for you, under ``projections`` in
|
||||||
``modules.yaml``. Other module dialects cannot be altered in this manner.
|
``modules.yaml``.
|
||||||
|
@ -459,14 +459,14 @@ is compiled with ``gcc@4.4.7``, with the only exception of any ``gcc``
|
|||||||
or any ``llvm`` installation.
|
or any ``llvm`` installation.
|
||||||
|
|
||||||
|
|
||||||
.. _modules-naming-scheme:
|
.. _modules-projections:
|
||||||
|
|
||||||
"""""""""""""""""""""""""""
|
"""""""""""""""""""""""""""""""
|
||||||
Customize the naming scheme
|
Customize the naming of modules
|
||||||
"""""""""""""""""""""""""""
|
"""""""""""""""""""""""""""""""
|
||||||
|
|
||||||
The names of environment modules generated by spack are not always easy to
|
The names of environment modules generated by spack are not always easy to
|
||||||
fully comprehend due to the long hash in the name. There are two module
|
fully comprehend due to the long hash in the name. There are three module
|
||||||
configuration options to help with that. The first is a global setting to
|
configuration options to help with that. The first is a global setting to
|
||||||
adjust the hash length. It can be set anywhere from 0 to 32 and has a default
|
adjust the hash length. It can be set anywhere from 0 to 32 and has a default
|
||||||
length of 7. This is the representation of the hash in the module file name and
|
length of 7. This is the representation of the hash in the module file name and
|
||||||
@ -500,20 +500,46 @@ version of python a set of python extensions is associated with. Likewise, the
|
|||||||
``openblas`` string is attached to any program that has openblas in the spec,
|
``openblas`` string is attached to any program that has openblas in the spec,
|
||||||
most likely via the ``+blas`` variant specification.
|
most likely via the ``+blas`` variant specification.
|
||||||
|
|
||||||
|
The most heavyweight solution to module naming is to change the entire
|
||||||
|
naming convention for module files. This uses the projections format
|
||||||
|
covered in :ref:`adding_projections_to_views`.
|
||||||
|
|
||||||
|
.. code-block:: yaml
|
||||||
|
|
||||||
|
modules:
|
||||||
|
tcl:
|
||||||
|
projections:
|
||||||
|
all: '{name}/{version}-{compiler.name}-{compiler.version}-module'
|
||||||
|
^mpi: '{name}/{version}-{^mpi.name}-{^mpi.version}-{compiler.name}-{compiler.version}-module'
|
||||||
|
|
||||||
|
will create module files that are nested in directories by package
|
||||||
|
name, contain the version and compiler name and version, and have the
|
||||||
|
word ``module`` before the hash for all specs that do not depend on
|
||||||
|
mpi, and will have the same information plus the MPI implementation
|
||||||
|
name and version for all packages that depend on mpi.
|
||||||
|
|
||||||
|
When specifying module names by projection for Lmod modules, we
|
||||||
|
recommend NOT including names of dependencies (e.g., MPI, compilers)
|
||||||
|
that are already in the LMod hierarchy.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
.. note::
|
.. note::
|
||||||
TCL module files
|
TCL modules
|
||||||
A modification that is specific to ``tcl`` module files is the possibility
|
TCL modules also allow for explicit conflicts between modulefiles.
|
||||||
to change the naming scheme of modules.
|
|
||||||
|
|
||||||
.. code-block:: yaml
|
.. code-block:: yaml
|
||||||
|
|
||||||
modules:
|
modules:
|
||||||
tcl:
|
enable:
|
||||||
naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}'
|
- tcl
|
||||||
all:
|
tcl:
|
||||||
conflict:
|
projections:
|
||||||
- '{name}'
|
all: '{name}/{version}-{compiler.name}-{compiler.version}'
|
||||||
- 'intel/14.0.1'
|
all:
|
||||||
|
conflict:
|
||||||
|
- '{name}'
|
||||||
|
- 'intel/14.0.1'
|
||||||
|
|
||||||
will create module files that will conflict with ``intel/14.0.1`` and with the
|
will create module files that will conflict with ``intel/14.0.1`` and with the
|
||||||
base directory of the same module, effectively preventing the possibility to
|
base directory of the same module, effectively preventing the possibility to
|
||||||
|
@ -22,6 +22,7 @@
|
|||||||
import spack.spec
|
import spack.spec
|
||||||
import spack.store
|
import spack.store
|
||||||
import spack.schema.projections
|
import spack.schema.projections
|
||||||
|
import spack.projections
|
||||||
import spack.config
|
import spack.config
|
||||||
from spack.error import SpackError
|
from spack.error import SpackError
|
||||||
from spack.directory_layout import ExtensionAlreadyInstalledError
|
from spack.directory_layout import ExtensionAlreadyInstalledError
|
||||||
@ -470,14 +471,9 @@ def get_projection_for_spec(self, spec):
|
|||||||
if spec.package.extendee_spec:
|
if spec.package.extendee_spec:
|
||||||
locator_spec = spec.package.extendee_spec
|
locator_spec = spec.package.extendee_spec
|
||||||
|
|
||||||
all_fmt_str = None
|
proj = spack.projections.get_projection(self.projections, locator_spec)
|
||||||
for spec_like, fmt_str in self.projections.items():
|
if proj:
|
||||||
if locator_spec.satisfies(spec_like, strict=True):
|
return os.path.join(self._root, locator_spec.format(proj))
|
||||||
return os.path.join(self._root, locator_spec.format(fmt_str))
|
|
||||||
elif spec_like == 'all':
|
|
||||||
all_fmt_str = fmt_str
|
|
||||||
if all_fmt_str:
|
|
||||||
return os.path.join(self._root, locator_spec.format(all_fmt_str))
|
|
||||||
return self._root
|
return self._root
|
||||||
|
|
||||||
def get_all_specs(self):
|
def get_all_specs(self):
|
||||||
|
@ -41,6 +41,7 @@
|
|||||||
import spack.error
|
import spack.error
|
||||||
import spack.paths
|
import spack.paths
|
||||||
import spack.schema.environment
|
import spack.schema.environment
|
||||||
|
import spack.projections as proj
|
||||||
import spack.tengine as tengine
|
import spack.tengine as tengine
|
||||||
import spack.util.environment
|
import spack.util.environment
|
||||||
import spack.util.file_permissions as fp
|
import spack.util.file_permissions as fp
|
||||||
@ -381,6 +382,9 @@ class BaseConfiguration(object):
|
|||||||
querying easier. It needs to be sub-classed for specific module types.
|
querying easier. It needs to be sub-classed for specific module types.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
default_projections = {
|
||||||
|
'all': '{name}-{version}-{compiler.name}-{compiler.version}'}
|
||||||
|
|
||||||
def __init__(self, spec):
|
def __init__(self, spec):
|
||||||
# Module where type(self) is defined
|
# Module where type(self) is defined
|
||||||
self.module = inspect.getmodule(self)
|
self.module = inspect.getmodule(self)
|
||||||
@ -391,19 +395,18 @@ def __init__(self, spec):
|
|||||||
self.conf = merge_config_rules(self.module.configuration(), self.spec)
|
self.conf = merge_config_rules(self.module.configuration(), self.spec)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def naming_scheme(self):
|
def projections(self):
|
||||||
"""Naming scheme suitable for non-hierarchical layouts"""
|
"""Projection from specs to module names"""
|
||||||
scheme = self.module.configuration().get(
|
projections = self.module.configuration().get(
|
||||||
'naming_scheme',
|
'projections', self.default_projections)
|
||||||
'{name}-{version}-{compiler.name}-{compiler.version}'
|
|
||||||
)
|
|
||||||
|
|
||||||
# Ensure the named tokens we are expanding are allowed, see
|
# Ensure the named tokens we are expanding are allowed, see
|
||||||
# issue #2884 for reference
|
# issue #2884 for reference
|
||||||
msg = 'some tokens cannot be part of the module naming scheme'
|
msg = 'some tokens cannot be part of the module naming scheme'
|
||||||
_check_tokens_are_valid(scheme, message=msg)
|
for projection in projections.values():
|
||||||
|
_check_tokens_are_valid(projection, message=msg)
|
||||||
|
|
||||||
return scheme
|
return projections
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def template(self):
|
def template(self):
|
||||||
@ -551,7 +554,11 @@ def use_name(self):
|
|||||||
to console to use it. This implementation fits the needs of most
|
to console to use it. This implementation fits the needs of most
|
||||||
non-hierarchical layouts.
|
non-hierarchical layouts.
|
||||||
"""
|
"""
|
||||||
name = self.spec.format(self.conf.naming_scheme)
|
projection = proj.get_projection(self.conf.projections, self.spec)
|
||||||
|
if not projection:
|
||||||
|
projection = self.conf.default_projections['all']
|
||||||
|
|
||||||
|
name = self.spec.format(projection)
|
||||||
# Not everybody is working on linux...
|
# Not everybody is working on linux...
|
||||||
parts = name.split('/')
|
parts = name.split('/')
|
||||||
name = os.path.join(*parts)
|
name = os.path.join(*parts)
|
||||||
|
@ -91,6 +91,7 @@ def guess_core_compilers(store=False):
|
|||||||
|
|
||||||
class LmodConfiguration(BaseConfiguration):
|
class LmodConfiguration(BaseConfiguration):
|
||||||
"""Configuration class for lmod module files."""
|
"""Configuration class for lmod module files."""
|
||||||
|
default_projections = {'all': os.path.join('{name}', '{version}')}
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def core_compilers(self):
|
def core_compilers(self):
|
||||||
@ -243,18 +244,6 @@ def filename(self):
|
|||||||
)
|
)
|
||||||
return fullname
|
return fullname
|
||||||
|
|
||||||
@property
|
|
||||||
def use_name(self):
|
|
||||||
"""Returns the 'use' name of the module i.e. the name you have to type
|
|
||||||
to console to use it.
|
|
||||||
"""
|
|
||||||
# Package name and version
|
|
||||||
base = os.path.join("{name}", "{version}")
|
|
||||||
name_parts = [self.spec.format(base)]
|
|
||||||
# The remaining elements are filename suffixes
|
|
||||||
name_parts.extend(self.conf.suffixes)
|
|
||||||
return '-'.join(name_parts)
|
|
||||||
|
|
||||||
def token_to_path(self, name, value):
|
def token_to_path(self, name, value):
|
||||||
"""Transforms a hierarchy token into the corresponding path part.
|
"""Transforms a hierarchy token into the corresponding path part.
|
||||||
|
|
||||||
|
@ -12,6 +12,7 @@
|
|||||||
import llnl.util.tty as tty
|
import llnl.util.tty as tty
|
||||||
|
|
||||||
import spack.config
|
import spack.config
|
||||||
|
import spack.projections as proj
|
||||||
import spack.tengine as tengine
|
import spack.tengine as tengine
|
||||||
from .common import BaseConfiguration, BaseFileLayout
|
from .common import BaseConfiguration, BaseFileLayout
|
||||||
from .common import BaseContext, BaseModuleFileWriter
|
from .common import BaseContext, BaseModuleFileWriter
|
||||||
@ -72,12 +73,12 @@ def prerequisites(self):
|
|||||||
def conflicts(self):
|
def conflicts(self):
|
||||||
"""List of conflicts for the tcl module file."""
|
"""List of conflicts for the tcl module file."""
|
||||||
fmts = []
|
fmts = []
|
||||||
naming_scheme = self.conf.naming_scheme
|
projection = proj.get_projection(self.conf.projections, self.spec)
|
||||||
f = string.Formatter()
|
f = string.Formatter()
|
||||||
for item in self.conf.conflicts:
|
for item in self.conf.conflicts:
|
||||||
if len([x for x in f.parse(item)]) > 1:
|
if len([x for x in f.parse(item)]) > 1:
|
||||||
for naming_dir, conflict_dir in zip(
|
for naming_dir, conflict_dir in zip(
|
||||||
naming_scheme.split('/'), item.split('/')
|
projection.split('/'), item.split('/')
|
||||||
):
|
):
|
||||||
if naming_dir != conflict_dir:
|
if naming_dir != conflict_dir:
|
||||||
message = 'conflict scheme does not match naming '
|
message = 'conflict scheme does not match naming '
|
||||||
@ -87,7 +88,7 @@ def conflicts(self):
|
|||||||
message += '** You may want to check your '
|
message += '** You may want to check your '
|
||||||
message += '`modules.yaml` configuration file **\n'
|
message += '`modules.yaml` configuration file **\n'
|
||||||
tty.error(message.format(spec=self.spec,
|
tty.error(message.format(spec=self.spec,
|
||||||
nformat=naming_scheme,
|
nformat=projection,
|
||||||
cformat=item))
|
cformat=item))
|
||||||
raise SystemExit('Module generation aborted.')
|
raise SystemExit('Module generation aborted.')
|
||||||
item = self.spec.format(item)
|
item = self.spec.format(item)
|
||||||
|
16
lib/spack/spack/projections.py
Normal file
16
lib/spack/spack/projections.py
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
|
||||||
|
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||||
|
|
||||||
|
def get_projection(projections, spec):
|
||||||
|
"""
|
||||||
|
Get the projection for a spec from a projections dict.
|
||||||
|
"""
|
||||||
|
all_projection = None
|
||||||
|
for spec_like, projection in projections.items():
|
||||||
|
if spec.satisfies(spec_like, strict=True):
|
||||||
|
return projection
|
||||||
|
elif spec_like == 'all':
|
||||||
|
all_projection = projection
|
||||||
|
return all_projection
|
@ -9,7 +9,7 @@
|
|||||||
:lines: 13-
|
:lines: 13-
|
||||||
"""
|
"""
|
||||||
import spack.schema.environment
|
import spack.schema.environment
|
||||||
|
import spack.schema.projections
|
||||||
|
|
||||||
#: Matches a spec or a multi-valued variant but not another
|
#: Matches a spec or a multi-valued variant but not another
|
||||||
#: valid keyword.
|
#: valid keyword.
|
||||||
@ -17,7 +17,7 @@
|
|||||||
#: THIS NEEDS TO BE UPDATED FOR EVERY NEW KEYWORD THAT
|
#: THIS NEEDS TO BE UPDATED FOR EVERY NEW KEYWORD THAT
|
||||||
#: IS ADDED IMMEDIATELY BELOW THE MODULE TYPE ATTRIBUTE
|
#: IS ADDED IMMEDIATELY BELOW THE MODULE TYPE ATTRIBUTE
|
||||||
spec_regex = r'(?!hierarchy|core_specs|verbose|hash_length|whitelist|' \
|
spec_regex = r'(?!hierarchy|core_specs|verbose|hash_length|whitelist|' \
|
||||||
r'blacklist|naming_scheme|core_compilers|all)(^\w[\w-]*)'
|
r'blacklist|projections|core_compilers|all)(^\w[\w-]*)'
|
||||||
|
|
||||||
#: Matches an anonymous spec, i.e. a spec without a root name
|
#: Matches an anonymous spec, i.e. a spec without a root name
|
||||||
anonymous_spec_regex = r'^[\^@%+~]'
|
anonymous_spec_regex = r'^[\^@%+~]'
|
||||||
@ -72,6 +72,8 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
projections_scheme = spack.schema.projections.properties['projections']
|
||||||
|
|
||||||
module_type_configuration = {
|
module_type_configuration = {
|
||||||
'type': 'object',
|
'type': 'object',
|
||||||
'default': {},
|
'default': {},
|
||||||
@ -92,9 +94,7 @@
|
|||||||
'type': 'boolean',
|
'type': 'boolean',
|
||||||
'default': False
|
'default': False
|
||||||
},
|
},
|
||||||
'naming_scheme': {
|
'projections': projections_scheme,
|
||||||
'type': 'string' # Can we be more specific here?
|
|
||||||
},
|
|
||||||
'all': module_file_configuration,
|
'all': module_file_configuration,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
6
lib/spack/spack/test/data/modules/lmod/projections.yaml
Normal file
6
lib/spack/spack/test/data/modules/lmod/projections.yaml
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
enable:
|
||||||
|
- lmod
|
||||||
|
lmod:
|
||||||
|
projections:
|
||||||
|
all: '{name}/v{version}'
|
||||||
|
mpileaks: '{name}-mpiprojection'
|
@ -1,7 +1,8 @@
|
|||||||
enable:
|
enable:
|
||||||
- tcl
|
- tcl
|
||||||
tcl:
|
tcl:
|
||||||
naming_scheme: '{name}/{version}-{compiler.name}'
|
projections:
|
||||||
|
all: '{name}/{version}-{compiler.name}'
|
||||||
all:
|
all:
|
||||||
conflict:
|
conflict:
|
||||||
- '{name}'
|
- '{name}'
|
||||||
|
@ -2,4 +2,5 @@ enable:
|
|||||||
- tcl
|
- tcl
|
||||||
tcl:
|
tcl:
|
||||||
# {variants} is not allowed in the naming scheme, see #2884
|
# {variants} is not allowed in the naming scheme, see #2884
|
||||||
naming_scheme: '{name}/{version}-{compiler.name}-{variants}'
|
projections:
|
||||||
|
all: '{name}/{version}-{compiler.name}-{variants}'
|
||||||
|
6
lib/spack/spack/test/data/modules/tcl/projections.yaml
Normal file
6
lib/spack/spack/test/data/modules/tcl/projections.yaml
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
enable:
|
||||||
|
- tcl
|
||||||
|
tcl:
|
||||||
|
projections:
|
||||||
|
all: '{name}/{version}-{compiler.name}'
|
||||||
|
mpileaks: '{name}-mpiprojection'
|
@ -1,7 +1,8 @@
|
|||||||
enable:
|
enable:
|
||||||
- tcl
|
- tcl
|
||||||
tcl:
|
tcl:
|
||||||
naming_scheme: '{name}/{version}-{compiler.name}'
|
projections:
|
||||||
|
all: '{name}/{version}-{compiler.name}'
|
||||||
all:
|
all:
|
||||||
conflict:
|
conflict:
|
||||||
- '{name}/{compiler.name}'
|
- '{name}/{compiler.name}'
|
||||||
|
@ -280,3 +280,39 @@ def test_only_generic_microarchitectures_in_root(
|
|||||||
assert str(spec.target.family) in writer.layout.arch_dirname
|
assert str(spec.target.family) in writer.layout.arch_dirname
|
||||||
if spec.target.family != spec.target:
|
if spec.target.family != spec.target:
|
||||||
assert str(spec.target) not in writer.layout.arch_dirname
|
assert str(spec.target) not in writer.layout.arch_dirname
|
||||||
|
|
||||||
|
def test_projections_specific(self, factory, module_configuration):
|
||||||
|
"""Tests reading the correct naming scheme."""
|
||||||
|
|
||||||
|
# This configuration has no error, so check the conflicts directives
|
||||||
|
# are there
|
||||||
|
module_configuration('projections')
|
||||||
|
|
||||||
|
# Test we read the expected configuration for the naming scheme
|
||||||
|
writer, _ = factory('mpileaks')
|
||||||
|
expected = {
|
||||||
|
'all': '{name}/v{version}',
|
||||||
|
'mpileaks': '{name}-mpiprojection'
|
||||||
|
}
|
||||||
|
|
||||||
|
assert writer.conf.projections == expected
|
||||||
|
projection = writer.spec.format(writer.conf.projections['mpileaks'])
|
||||||
|
assert projection in writer.layout.use_name
|
||||||
|
|
||||||
|
def test_projections_all(self, factory, module_configuration):
|
||||||
|
"""Tests reading the correct naming scheme."""
|
||||||
|
|
||||||
|
# This configuration has no error, so check the conflicts directives
|
||||||
|
# are there
|
||||||
|
module_configuration('projections')
|
||||||
|
|
||||||
|
# Test we read the expected configuration for the naming scheme
|
||||||
|
writer, _ = factory('libelf')
|
||||||
|
expected = {
|
||||||
|
'all': '{name}/v{version}',
|
||||||
|
'mpileaks': '{name}-mpiprojection'
|
||||||
|
}
|
||||||
|
|
||||||
|
assert writer.conf.projections == expected
|
||||||
|
projection = writer.spec.format(writer.conf.projections['all'])
|
||||||
|
assert projection in writer.layout.use_name
|
||||||
|
@ -142,18 +142,41 @@ def test_blacklist(self, modulefile_content, module_configuration):
|
|||||||
assert len([x for x in content if 'is-loaded' in x]) == 1
|
assert len([x for x in content if 'is-loaded' in x]) == 1
|
||||||
assert len([x for x in content if 'module load ' in x]) == 1
|
assert len([x for x in content if 'module load ' in x]) == 1
|
||||||
|
|
||||||
def test_naming_scheme(self, factory, module_configuration):
|
def test_projections_specific(self, factory, module_configuration):
|
||||||
"""Tests reading the correct naming scheme."""
|
"""Tests reading the correct naming scheme."""
|
||||||
|
|
||||||
# This configuration has no error, so check the conflicts directives
|
# This configuration has no error, so check the conflicts directives
|
||||||
# are there
|
# are there
|
||||||
module_configuration('conflicts')
|
module_configuration('projections')
|
||||||
|
|
||||||
# Test we read the expected configuration for the naming scheme
|
# Test we read the expected configuration for the naming scheme
|
||||||
writer, _ = factory('mpileaks')
|
writer, _ = factory('mpileaks')
|
||||||
expected = '{name}/{version}-{compiler.name}'
|
expected = {
|
||||||
|
'all': '{name}/{version}-{compiler.name}',
|
||||||
|
'mpileaks': '{name}-mpiprojection'
|
||||||
|
}
|
||||||
|
|
||||||
assert writer.conf.naming_scheme == expected
|
assert writer.conf.projections == expected
|
||||||
|
projection = writer.spec.format(writer.conf.projections['mpileaks'])
|
||||||
|
assert projection in writer.layout.use_name
|
||||||
|
|
||||||
|
def test_projections_all(self, factory, module_configuration):
|
||||||
|
"""Tests reading the correct naming scheme."""
|
||||||
|
|
||||||
|
# This configuration has no error, so check the conflicts directives
|
||||||
|
# are there
|
||||||
|
module_configuration('projections')
|
||||||
|
|
||||||
|
# Test we read the expected configuration for the naming scheme
|
||||||
|
writer, _ = factory('libelf')
|
||||||
|
expected = {
|
||||||
|
'all': '{name}/{version}-{compiler.name}',
|
||||||
|
'mpileaks': '{name}-mpiprojection'
|
||||||
|
}
|
||||||
|
|
||||||
|
assert writer.conf.projections == expected
|
||||||
|
projection = writer.spec.format(writer.conf.projections['all'])
|
||||||
|
assert projection in writer.layout.use_name
|
||||||
|
|
||||||
def test_invalid_naming_scheme(self, factory, module_configuration):
|
def test_invalid_naming_scheme(self, factory, module_configuration):
|
||||||
"""Tests the evaluation of an invalid naming scheme."""
|
"""Tests the evaluation of an invalid naming scheme."""
|
||||||
|
Loading…
Reference in New Issue
Block a user