zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add module_roots to config.yaml

Browse Source
This commit is contained in:
Todd Gamblin 2016-10-27 23:32:42 -07:00
parent 0da639298c
commit 22b14e0f23
5 changed files with 69 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
etc/spack/defaults/config.yaml
View File

@ -14,17 +14,18 @@
# ~/.spack/config.yaml # ~/.spack/config.yaml
# ------------------------------------------------------------------------- # -------------------------------------------------------------------------
config: config:
# Precedence of configuration scopes, high to low.
# The user can override this paradoxically.
# scopes: [user, spack, default]
# This is the path to the root of the Spack install tree. # This is the path to the root of the Spack install tree.
# You can use $spack here to refer to the root of the spack instance. # You can use $spack here to refer to the root of the spack instance.
install_tree: $spack/opt/spack install_tree: $spack/opt/spack
# Locations where different types of modules should be installed.
module_roots:
tcl: $spack/share/spack/modules
lmod: $spack/share/spack/lmod
dotkit: $spack/share/spack/dotkit
# Temporary locations Spack can try to use for builds. # Temporary locations Spack can try to use for builds.
# #
# Spack will use the first one it finds that exists and is writable. # Spack will use the first one it finds that exists and is writable.

3
lib/spack/spack/cmd/module.py
View File

@ -210,7 +210,8 @@ def refresh(mtype, specs, args):
cls = module_types[mtype] cls = module_types[mtype]
# Detect name clashes # Detect name clashes
writers = [cls(spec) for spec in specs] writers = [cls(spec) for spec in specs
if spack.repo.exists(spec.name)] # skip unknown packages.
file2writer = collections.defaultdict(list) file2writer = collections.defaultdict(list)
for item in writers: for item in writers:
file2writer[item.file_name].append(item) file2writer[item.file_name].append(item)

36
lib/spack/spack/modules.py
View File

@ -48,10 +48,12 @@
import textwrap import textwrap
import llnl.util.tty as tty import llnl.util.tty as tty
from llnl.util.filesystem import join_path, mkdirp
import spack import spack
import spack.compilers # Needed by LmodModules import spack.compilers # Needed by LmodModules
import spack.config import spack.config
from llnl.util.filesystem import join_path, mkdirp from spack.util.path import canonicalize_path
from spack.build_environment import parent_class_modules from spack.build_environment import parent_class_modules
from spack.build_environment import set_module_variables_for_package from spack.build_environment import set_module_variables_for_package
from spack.environment import * from spack.environment import *
@ -62,7 +64,11 @@
metaclass.""" metaclass."""
module_types = {} module_types = {}
CONFIGURATION = spack.config.get_config('modules') """Module install roots are in config.yaml."""
_roots = spack.config.get_config('config').get('module_roots', {})
"""Specifics about modules are in modules.yaml"""
_module_config = spack.config.get_config('modules')
def print_help(): def print_help():
@ -92,7 +98,7 @@ def inspect_path(prefix):
""" """
env = EnvironmentModifications() env = EnvironmentModifications()
# Inspect the prefix to check for the existence of common directories # Inspect the prefix to check for the existence of common directories
prefix_inspections = CONFIGURATION.get('prefix_inspections', {}) prefix_inspections = _module_config.get('prefix_inspections', {})
for relative_path, variables in prefix_inspections.items(): for relative_path, variables in prefix_inspections.items():
expected = join_path(prefix, relative_path) expected = join_path(prefix, relative_path)
if os.path.isdir(expected): if os.path.isdir(expected):
@ -168,7 +174,7 @@ def parse_config_options(module_generator):
module file module file
""" """
# Get the configuration for this kind of generator # Get the configuration for this kind of generator
module_configuration = copy.deepcopy(CONFIGURATION.get( module_configuration = copy.deepcopy(_module_config.get(
module_generator.name, {})) module_generator.name, {}))
##### #####
@ -248,7 +254,7 @@ class __metaclass__(type):
def __init__(cls, name, bases, dict): def __init__(cls, name, bases, dict):
type.__init__(cls, name, bases, dict) type.__init__(cls, name, bases, dict)
if cls.name != 'env_module' and cls.name in CONFIGURATION[ if cls.name != 'env_module' and cls.name in _module_config[
'enable']: 'enable']:
module_types[cls.name] = cls module_types[cls.name] = cls
@ -271,7 +277,7 @@ def __init__(self, spec=None):
@property @property
def naming_scheme(self): def naming_scheme(self):
try: try:
naming_scheme = CONFIGURATION[self.name]['naming_scheme'] naming_scheme = _module_config[self.name]['naming_scheme']
except KeyError: except KeyError:
naming_scheme = self.default_naming_format naming_scheme = self.default_naming_format
return naming_scheme return naming_scheme
@ -314,7 +320,7 @@ def category(self):
@property @property
def blacklisted(self): def blacklisted(self):
configuration = CONFIGURATION.get(self.name, {}) configuration = _module_config.get(self.name, {})
whitelist_matches = [x whitelist_matches = [x
for x in configuration.get('whitelist', []) for x in configuration.get('whitelist', [])
if self.spec.satisfies(x)] if self.spec.satisfies(x)]
@ -473,7 +479,9 @@ def remove(self):
class Dotkit(EnvModule): class Dotkit(EnvModule):
name = 'dotkit' name = 'dotkit'
path = join_path(spack.share_path, 'dotkit') path = canonicalize_path(
_roots.get(name, join_path(spack.share_path, name)))
environment_modifications_formats = { environment_modifications_formats = {
PrependPath: 'dk_alter {name} {value}\n', PrependPath: 'dk_alter {name} {value}\n',
RemovePath: 'dk_unalter {name} {value}\n', RemovePath: 'dk_unalter {name} {value}\n',
@ -516,7 +524,8 @@ def prerequisite(self, spec):
class TclModule(EnvModule): class TclModule(EnvModule):
name = 'tcl' name = 'tcl'
path = join_path(spack.share_path, "modules") path = canonicalize_path(
_roots.get(name, join_path(spack.share_path, 'modules')))
autoload_format = ('if ![ is-loaded {module_file} ] {{\n' autoload_format = ('if ![ is-loaded {module_file} ] {{\n'
' puts stderr "Autoloading {module_file}"\n' ' puts stderr "Autoloading {module_file}"\n'
@ -635,7 +644,8 @@ def module_specific_content(self, configuration):
class LmodModule(EnvModule): class LmodModule(EnvModule):
name = 'lmod' name = 'lmod'
path = join_path(spack.share_path, "lmod") path = canonicalize_path(
_roots.get(name, join_path(spack.share_path, name)))
environment_modifications_formats = { environment_modifications_formats = {
PrependPath: 'prepend_path("{name}", "{value}")\n', PrependPath: 'prepend_path("{name}", "{value}")\n',
@ -656,6 +666,12 @@ class LmodModule(EnvModule):
path_part_without_hash = join_path('{token.name}', '{token.version}') path_part_without_hash = join_path('{token.name}', '{token.version}')
# TODO : Check that extra tokens specified in configuration file
# TODO : are actually virtual dependencies
configuration = _module_config.get('lmod', {})
hierarchy_tokens = configuration.get('hierarchical_scheme', [])
hierarchy_tokens = hierarchy_tokens + ['mpi', 'compiler']
def __init__(self, spec=None): def __init__(self, spec=None):
super(LmodModule, self).__init__(spec) super(LmodModule, self).__init__(spec)

10
lib/spack/spack/schema/config.py
View File

@ -42,6 +42,16 @@
{'type': 'array', {'type': 'array',
'items': {'type': 'string'}}], 'items': {'type': 'string'}}],
}, },
'module_roots': {
'type': 'object',
'default': {},
'additionalProperties': False,
'properties': {
'tcl': {'type': 'string'},
'lmod': {'type': 'string'},
'dotkit': {'type': 'string'},
},
},
'source_cache': {'type': 'string'}, 'source_cache': {'type': 'string'},
'misc_cache': {'type': 'string'}, 'misc_cache': {'type': 'string'},
'verify_ssl': { 'verify_ssl': {

48
lib/spack/spack/test/modules.py
View File

@ -102,18 +102,18 @@ class ModuleFileGeneratorTests(MockPackagesTest):
def setUp(self): def setUp(self):
super(ModuleFileGeneratorTests, self).setUp() super(ModuleFileGeneratorTests, self).setUp()
self.configuration_instance = spack.modules.CONFIGURATION self.configuration_instance = spack.modules._module_config
self.module_types_instance = spack.modules.module_types self.module_types_instance = spack.modules.module_types
spack.modules.open = mock_open spack.modules.open = mock_open
spack.modules.mkdirp = lambda x: None spack.modules.mkdirp = lambda x: None
# Make sure that a non-mocked configuration will trigger an error # Make sure that a non-mocked configuration will trigger an error
spack.modules.CONFIGURATION = None spack.modules._module_config = None
spack.modules.module_types = {self.factory.name: self.factory} spack.modules.module_types = {self.factory.name: self.factory}
def tearDown(self): def tearDown(self):
del spack.modules.open del spack.modules.open
spack.modules.module_types = self.module_types_instance spack.modules.module_types = self.module_types_instance
spack.modules.CONFIGURATION = self.configuration_instance spack.modules._module_config = self.configuration_instance
spack.modules.mkdirp = llnl.util.filesystem.mkdirp spack.modules.mkdirp = llnl.util.filesystem.mkdirp
super(ModuleFileGeneratorTests, self).tearDown() super(ModuleFileGeneratorTests, self).tearDown()
@ -231,7 +231,7 @@ class TclTests(ModuleFileGeneratorTests):
} }
def test_simple_case(self): def test_simple_case(self):
spack.modules.CONFIGURATION = self.configuration_autoload_direct spack.modules._module_config = self.configuration_autoload_direct
spec = spack.spec.Spec(mpich_spec_string) spec = spack.spec.Spec(mpich_spec_string)
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertTrue('module-whatis "mpich @3.0.4"' in content) self.assertTrue('module-whatis "mpich @3.0.4"' in content)
@ -239,13 +239,13 @@ def test_simple_case(self):
spec, 'non-existing-tag') spec, 'non-existing-tag')
def test_autoload(self): def test_autoload(self):
spack.modules.CONFIGURATION = self.configuration_autoload_direct spack.modules._module_config = self.configuration_autoload_direct
spec = spack.spec.Spec(mpileaks_spec_string) spec = spack.spec.Spec(mpileaks_spec_string)
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2)
self.assertEqual(len([x for x in content if 'module load ' in x]), 2) self.assertEqual(len([x for x in content if 'module load ' in x]), 2)
spack.modules.CONFIGURATION = self.configuration_autoload_all spack.modules._module_config = self.configuration_autoload_all
spec = spack.spec.Spec(mpileaks_spec_string) spec = spack.spec.Spec(mpileaks_spec_string)
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5)
@ -256,7 +256,7 @@ def test_autoload(self):
# - 1 ('build','link') dependency # - 1 ('build','link') dependency
# - 1 ('build',) dependency # - 1 ('build',) dependency
# Just make sure the 'build' dependency is not there # Just make sure the 'build' dependency is not there
spack.modules.CONFIGURATION = self.configuration_autoload_direct spack.modules._module_config = self.configuration_autoload_direct
spec = spack.spec.Spec('dtbuild1') spec = spack.spec.Spec('dtbuild1')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2)
@ -267,25 +267,25 @@ def test_autoload(self):
# - 1 ('build','link') dependency # - 1 ('build','link') dependency
# - 1 ('build',) dependency # - 1 ('build',) dependency
# Just make sure the 'build' dependency is not there # Just make sure the 'build' dependency is not there
spack.modules.CONFIGURATION = self.configuration_autoload_all spack.modules._module_config = self.configuration_autoload_all
spec = spack.spec.Spec('dtbuild1') spec = spack.spec.Spec('dtbuild1')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2)
self.assertEqual(len([x for x in content if 'module load ' in x]), 2) self.assertEqual(len([x for x in content if 'module load ' in x]), 2)
def test_prerequisites(self): def test_prerequisites(self):
spack.modules.CONFIGURATION = self.configuration_prerequisites_direct spack.modules._module_config = self.configuration_prerequisites_direct
spec = spack.spec.Spec('mpileaks arch=x86-linux') spec = spack.spec.Spec('mpileaks arch=x86-linux')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'prereq' in x]), 2) self.assertEqual(len([x for x in content if 'prereq' in x]), 2)
spack.modules.CONFIGURATION = self.configuration_prerequisites_all spack.modules._module_config = self.configuration_prerequisites_all
spec = spack.spec.Spec('mpileaks arch=x86-linux') spec = spack.spec.Spec('mpileaks arch=x86-linux')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'prereq' in x]), 5) self.assertEqual(len([x for x in content if 'prereq' in x]), 5)
def test_alter_environment(self): def test_alter_environment(self):
spack.modules.CONFIGURATION = self.configuration_alter_environment spack.modules._module_config = self.configuration_alter_environment
spec = spack.spec.Spec('mpileaks platform=test target=x86_64') spec = spack.spec.Spec('mpileaks platform=test target=x86_64')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual( self.assertEqual(
@ -315,7 +315,7 @@ def test_alter_environment(self):
len([x for x in content if 'setenv LIBDWARF_ROOT' in x]), 1) len([x for x in content if 'setenv LIBDWARF_ROOT' in x]), 1)
def test_blacklist(self): def test_blacklist(self):
spack.modules.CONFIGURATION = self.configuration_blacklist spack.modules._module_config = self.configuration_blacklist
spec = spack.spec.Spec('mpileaks ^zmpi') spec = spack.spec.Spec('mpileaks ^zmpi')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1)
@ -329,7 +329,7 @@ def test_blacklist(self):
self.assertEqual(len([x for x in content if 'module load ' in x]), 1) self.assertEqual(len([x for x in content if 'module load ' in x]), 1)
def test_conflicts(self): def test_conflicts(self):
spack.modules.CONFIGURATION = self.configuration_conflicts spack.modules._module_config = self.configuration_conflicts
spec = spack.spec.Spec('mpileaks') spec = spack.spec.Spec('mpileaks')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual( self.assertEqual(
@ -339,11 +339,11 @@ def test_conflicts(self):
self.assertEqual( self.assertEqual(
len([x for x in content if x == 'conflict intel/14.0.1']), 1) len([x for x in content if x == 'conflict intel/14.0.1']), 1)
spack.modules.CONFIGURATION = self.configuration_wrong_conflicts spack.modules._module_config = self.configuration_wrong_conflicts
self.assertRaises(SystemExit, self.get_modulefile_content, spec) self.assertRaises(SystemExit, self.get_modulefile_content, spec)
def test_suffixes(self): def test_suffixes(self):
spack.modules.CONFIGURATION = self.configuration_suffix spack.modules._module_config = self.configuration_suffix
spec = spack.spec.Spec('mpileaks+debug arch=x86-linux') spec = spack.spec.Spec('mpileaks+debug arch=x86-linux')
spec.concretize() spec.concretize()
generator = spack.modules.TclModule(spec) generator = spack.modules.TclModule(spec)
@ -412,7 +412,7 @@ class LmodTests(ModuleFileGeneratorTests):
} }
def test_simple_case(self): def test_simple_case(self):
spack.modules.CONFIGURATION = self.configuration_autoload_direct spack.modules._module_config = self.configuration_autoload_direct
spec = spack.spec.Spec(mpich_spec_string) spec = spack.spec.Spec(mpich_spec_string)
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertTrue('-- -*- lua -*-' in content) self.assertTrue('-- -*- lua -*-' in content)
@ -420,14 +420,14 @@ def test_simple_case(self):
self.assertTrue('whatis([[Version : 3.0.4]])' in content) self.assertTrue('whatis([[Version : 3.0.4]])' in content)
def test_autoload(self): def test_autoload(self):
spack.modules.CONFIGURATION = self.configuration_autoload_direct spack.modules._module_config = self.configuration_autoload_direct
spec = spack.spec.Spec(mpileaks_spec_string) spec = spack.spec.Spec(mpileaks_spec_string)
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual( self.assertEqual(
len([x for x in content if 'if not isloaded(' in x]), 2) len([x for x in content if 'if not isloaded(' in x]), 2)
self.assertEqual(len([x for x in content if 'load(' in x]), 2) self.assertEqual(len([x for x in content if 'load(' in x]), 2)
spack.modules.CONFIGURATION = self.configuration_autoload_all spack.modules._module_config = self.configuration_autoload_all
spec = spack.spec.Spec(mpileaks_spec_string) spec = spack.spec.Spec(mpileaks_spec_string)
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual( self.assertEqual(
@ -435,7 +435,7 @@ def test_autoload(self):
self.assertEqual(len([x for x in content if 'load(' in x]), 5) self.assertEqual(len([x for x in content if 'load(' in x]), 5)
def test_alter_environment(self): def test_alter_environment(self):
spack.modules.CONFIGURATION = self.configuration_alter_environment spack.modules._module_config = self.configuration_alter_environment
spec = spack.spec.Spec('mpileaks platform=test target=x86_64') spec = spack.spec.Spec('mpileaks platform=test target=x86_64')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual( self.assertEqual(
@ -460,7 +460,7 @@ def test_alter_environment(self):
len([x for x in content if 'unsetenv("BAR")' in x]), 0) len([x for x in content if 'unsetenv("BAR")' in x]), 0)
def test_blacklist(self): def test_blacklist(self):
spack.modules.CONFIGURATION = self.configuration_blacklist spack.modules._module_config = self.configuration_blacklist
spec = spack.spec.Spec(mpileaks_spec_string) spec = spack.spec.Spec(mpileaks_spec_string)
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual( self.assertEqual(
@ -499,14 +499,14 @@ class DotkitTests(MockPackagesTest):
def setUp(self): def setUp(self):
super(DotkitTests, self).setUp() super(DotkitTests, self).setUp()
self.configuration_obj = spack.modules.CONFIGURATION self.configuration_obj = spack.modules._module_config
spack.modules.open = mock_open spack.modules.open = mock_open
# Make sure that a non-mocked configuration will trigger an error # Make sure that a non-mocked configuration will trigger an error
spack.modules.CONFIGURATION = None spack.modules._module_config = None
def tearDown(self): def tearDown(self):
del spack.modules.open del spack.modules.open
spack.modules.CONFIGURATION = self.configuration_obj spack.modules._module_config = self.configuration_obj
super(DotkitTests, self).tearDown() super(DotkitTests, self).tearDown()
def get_modulefile_content(self, spec): def get_modulefile_content(self, spec):
@ -517,7 +517,7 @@ def get_modulefile_content(self, spec):
return content return content
def test_dotkit(self): def test_dotkit(self):
spack.modules.CONFIGURATION = self.configuration_dotkit spack.modules._module_config = self.configuration_dotkit
spec = spack.spec.Spec('mpileaks arch=x86-linux') spec = spack.spec.Spec('mpileaks arch=x86-linux')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertTrue('#c spack' in content) self.assertTrue('#c spack' in content)