Merge pull request #19 in SCALE/spack from features/modules to develop

# By Todd Gamblin (4) and David Beckingsale (2)
# Via Todd Gamblin
* commit 'b601fd08caf21b5fc11e6998a5ebd20a04ac57ad':
  Bugfixes for csh environment modules.
  Bugfixes, more consolidation of modules code.
  Add csh/tcsh support for modules
  Consolidate most module code into spack.modules and spack.cmd.module
  Fixed up module support
  Added inital module support
This commit is contained in:
George Todd Gamblin
2014-08-18 23:11:02 -07:00
18 changed files with 824 additions and 305 deletions

View File

@@ -58,7 +58,6 @@
stage_path = join_path(var_path, "stage")
install_path = join_path(prefix, "opt")
share_path = join_path(prefix, "share", "spack")
dotkit_path = join_path(share_path, "dotkit")
#
# Set up the packages database.

View File

@@ -1,99 +0,0 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import sys
import os
import shutil
import argparse
import llnl.util.tty as tty
from llnl.util.lang import partition_list
from llnl.util.filesystem import mkdirp
import spack.cmd
import spack.hooks.dotkit
from spack.spec import Spec
description ="Find dotkits for packages if they exist."
def setup_parser(subparser):
subparser.add_argument(
'--refresh', action='store_true', help='Regenerate all dotkits')
subparser.add_argument(
'spec', nargs=argparse.REMAINDER, help='spec to find a dotkit for.')
def dotkit_find(parser, args):
if not args.spec:
parser.parse_args(['dotkit', '-h'])
spec = spack.cmd.parse_specs(args.spec)
if len(spec) > 1:
tty.die("You can only pass one spec.")
spec = spec[0]
if not spack.db.exists(spec.name):
tty.die("No such package: %s" % spec.name)
specs = [s for s in spack.db.installed_package_specs() if s.satisfies(spec)]
if len(specs) == 0:
tty.die("No installed packages match spec %s" % spec)
if len(specs) > 1:
tty.error("Multiple matches for spec %s. Choose one:" % spec)
for s in specs:
sys.stderr.write(s.tree(color=True))
sys.exit(1)
match = specs[0]
if not os.path.isfile(spack.hooks.dotkit.dotkit_file(match.package)):
tty.die("No dotkit is installed for package %s." % spec)
print match.format('$_$@$+$%@$=$#')
def dotkit_refresh(parser, args):
query_specs = spack.cmd.parse_specs(args.spec)
specs = spack.db.installed_package_specs()
if query_specs:
specs = [s for s in specs
if any(s.satisfies(q) for q in query_specs)]
else:
shutil.rmtree(spack.dotkit_path, ignore_errors=False)
mkdirp(spack.dotkit_path)
for spec in specs:
spack.hooks.dotkit.post_install(spec.package)
def dotkit(parser, args):
if args.refresh:
dotkit_refresh(parser, args)
else:
dotkit_find(parser, args)

View File

@@ -0,0 +1,38 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by David Beckingsale, david@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import argparse
import spack.modules
description ="Add package to environment using modules."
def setup_parser(subparser):
"""Parser is only constructed so that this prints a nice help
message with -h. """
subparser.add_argument(
'spec', nargs=argparse.REMAINDER, help='Spec of package to load with modules.')
def load(parser, args):
spack.modules.print_help()

View File

@@ -0,0 +1,108 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import sys
import os
import shutil
import argparse
import llnl.util.tty as tty
from llnl.util.lang import partition_list
from llnl.util.filesystem import mkdirp
import spack.cmd
from spack.modules import module_types
from spack.util.string import *
from spack.spec import Spec
description ="Manipulate modules and dotkits."
def setup_parser(subparser):
sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='module_command')
refresh_parser = sp.add_parser('refresh', help='Regenerate all module files.')
find_parser = sp.add_parser('find', help='Find module files for packages.')
find_parser.add_argument(
'module_type', help="Type of module to find file for. [" + '|'.join(module_types) + "]")
find_parser.add_argument('spec', nargs='+', help='spec to find a module file for.')
def module_find(mtype, spec_array):
"""Look at all installed packages and see if the spec provided
matches any. If it does, check whether there is a module file
of type <mtype> there, and print out the name that the user
should type to use that package's module.
"""
specs = spack.cmd.parse_specs(spec_array)
if len(specs) > 1:
tty.die("You can only pass one spec.")
spec = specs[0]
if mtype not in module_types:
tty.die("Invalid module type: '%s'. Options are " + comma_and(module_types))
specs = [s for s in spack.db.installed_package_specs() if s.satisfies(spec)]
if len(specs) == 0:
tty.die("No installed packages match spec %s" % spec)
if len(specs) > 1:
tty.error("Multiple matches for spec %s. Choose one:" % spec)
for s in specs:
sys.stderr.write(s.tree(color=True))
sys.exit(1)
mt = module_types[mtype]
mod = mt(specs[0].package)
if not os.path.isfile(mod.file_name):
tty.error( mod.file_name)
tty.die("No %s module is installed for package %s." % (mtype, spec))
print mod.use_name
def module_refresh():
"""Regenerate all module files for installed packages known to
spack (some packages may no longer exist)."""
specs = [s for s in spack.db.installed_known_package_specs()]
for name, cls in module_types.items():
tty.msg("Regenerating %s module files." % name)
if os.path.isdir(cls.path):
shutil.rmtree(cls.path, ignore_errors=False)
mkdirp(cls.path)
for spec in specs:
tty.debug(" Writing file for %s." % spec)
cls(spec.package).write()
def module(parser, args):
if args.module_command == 'refresh':
module_refresh()
elif args.module_command == 'find':
module_find(args.module_type, args.spec)

View File

@@ -0,0 +1,38 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by David Beckingsale, david@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import argparse
import spack.modules
description ="Remove package from environment using module."
def setup_parser(subparser):
"""Parser is only constructed so that this prints a nice help
message with -h. """
subparser.add_argument(
'spec', nargs=argparse.REMAINDER, help='Spec of package to unload with modules.')
def unload(parser, args):
spack.modules.print_help()

View File

@@ -23,14 +23,16 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import argparse
import spack.cmd.use
import spack.modules
description ="Remove package from environment using dotkit."
def setup_parser(subparser):
"""Parser is only constructed so that this prints a nice help
message with -h. """
subparser.add_argument(
'spec', nargs=argparse.REMAINDER, help='Spec of package to remove.')
'spec', nargs=argparse.REMAINDER, help='Spec of package to unuse with dotkit.')
def unuse(parser, args):
spack.cmd.use.print_help()
spack.modules.print_help()

View File

@@ -23,28 +23,16 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import argparse
import llnl.util.tty as tty
import spack
import spack.modules
description ="Add package to environment using dotkit."
def setup_parser(subparser):
"""Parser is only constructed so that this prints a nice help
message with -h. """
subparser.add_argument(
'spec', nargs=argparse.REMAINDER, help='Spec of package to add.')
def print_help():
tty.msg("Spack dotkit support is not initialized.",
"",
"To use dotkit with Spack, you must first run the command",
"below, which you can copy and paste:",
"",
"For bash:",
" . %s/setup-env.bash" % spack.share_path,
"",
"ksh/csh/tcsh shells are currently unsupported",
"")
'spec', nargs=argparse.REMAINDER, help='Spec of package to use with dotkit.')
def use(parser, args):
print_help()
spack.modules.print_help()

View File

@@ -22,62 +22,14 @@
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import os
import re
import textwrap
import shutil
from contextlib import closing
from llnl.util.filesystem import join_path, mkdirp
import spack
def dotkit_file(pkg):
dk_file_name = pkg.spec.format('$_$@$%@$+$=$#') + ".dk"
return join_path(spack.dotkit_path, dk_file_name)
import spack.modules
def post_install(pkg):
if not os.path.exists(spack.dotkit_path):
mkdirp(spack.dotkit_path)
alterations = []
for var, path in [
('PATH', pkg.prefix.bin),
('MANPATH', pkg.prefix.man),
('MANPATH', pkg.prefix.share_man),
('LD_LIBRARY_PATH', pkg.prefix.lib),
('LD_LIBRARY_PATH', pkg.prefix.lib64)]:
if os.path.isdir(path):
alterations.append("dk_alter %s %s\n" % (var, path))
if not alterations:
return
alterations.append("dk_alter CMAKE_PREFIX_PATH %s\n" % pkg.prefix)
dk_file = dotkit_file(pkg)
with closing(open(dk_file, 'w')) as dk:
# Put everything in the spack category.
dk.write('#c spack\n')
dk.write('#d %s\n' % pkg.spec.format("$_ $@"))
# Recycle the description
if pkg.__doc__:
doc = re.sub(r'\s+', ' ', pkg.__doc__)
for line in textwrap.wrap(doc, 72):
dk.write("#h %s\n" % line)
# Write alterations
for alter in alterations:
dk.write(alter)
dk = spack.modules.Dotkit(pkg)
dk.write()
def post_uninstall(pkg):
dk_file = dotkit_file(pkg)
if os.path.exists(dk_file):
shutil.rmtree(dk_file, ignore_errors=True)
dk = spack.modules.Dotkit(pkg)
dk.remove()

View File

@@ -0,0 +1,35 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by David Beckingsale, david@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import spack.modules
def post_install(pkg):
dk = spack.modules.TclModule(pkg)
dk.write()
def post_uninstall(pkg):
dk = spack.modules.TclModule(pkg)
dk.remove()

247
lib/spack/spack/modules.py Normal file
View File

@@ -0,0 +1,247 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""This module contains code for creating environment modules, which
can include dotkits, tcl modules, lmod, and others.
The various types of modules are installed by post-install hooks and
removed after an uninstall by post-uninstall hooks. This class
consolidates the logic for creating an abstract description of the
information that module systems need. Currently that includes a
number directories to be appended to paths in the user's environment:
* /bin directories to be appended to PATH
* /lib* directories for LD_LIBRARY_PATH
* /man* and /share/man* directories for LD_LIBRARY_PATH
* the package prefix for CMAKE_PREFIX_PATH
This module also includes logic for coming up with unique names for
the module files so that they can be found by the various
shell-support files in $SPACK/share/spack/setup-env.*.
Each hook in hooks/ implements the logic for writing its specific type
of module file.
"""
__all__ = ['EnvModule', 'Dotkit', 'TclModule']
import os
import re
import textwrap
import shutil
from contextlib import closing
import llnl.util.tty as tty
from llnl.util.filesystem import join_path, mkdirp
import spack
"""Registry of all types of modules. Entries created by EnvModule's
metaclass."""
module_types = {}
def print_help():
"""For use by commands to tell user how to activate shell support."""
tty.msg("Spack module/dotkit support is not initialized.",
"",
"To use dotkit or modules with Spack, you must first run",
"one of the commands below. You can copy/paste them.",
"",
"For bash and zsh:",
" . %s/setup-env.sh" % spack.share_path,
"",
"For csh and tcsh:",
" setenv SPACK_ROOT %s" % spack.prefix,
" source %s/setup-env.csh" % spack.share_path,
"")
class EnvModule(object):
name = 'env_module'
class __metaclass__(type):
def __init__(cls, name, bases, dict):
type.__init__(cls, name, bases, dict)
if cls.name != 'env_module':
module_types[cls.name] = cls
def __init__(self, pkg=None):
# category in the modules system
# TODO: come up with smarter category names.
self.category = "spack"
# Descriptions for the module system's UI
self.short_description = ""
self.long_description = ""
# dict pathname -> list of directories to be prepended to in
# the module file.
self._paths = None
self.pkg = pkg
@property
def paths(self):
if self._paths is None:
self._paths = {}
def add_path(path_name, directory):
path = self._paths.setdefault(path_name, [])
path.append(directory)
# Add paths if they exist.
for var, directory in [
('PATH', self.pkg.prefix.bin),
('MANPATH', self.pkg.prefix.man),
('MANPATH', self.pkg.prefix.share_man),
('LD_LIBRARY_PATH', self.pkg.prefix.lib),
('LD_LIBRARY_PATH', self.pkg.prefix.lib64)]:
if os.path.isdir(directory):
add_path(var, directory)
# short description is just the package + version
# TODO: maybe packages can optionally provide it.
self.short_description = self.pkg.spec.format("$_ $@")
# long description is the docstring with reduced whitespace.
if self.pkg.__doc__:
self.long_description = re.sub(r'\s+', ' ', self.pkg.__doc__)
return self._paths
def write(self):
"""Write out a module file for this object."""
module_dir = os.path.dirname(self.file_name)
if not os.path.exists(module_dir):
mkdirp(module_dir)
# If there are no paths, no need for a dotkit.
if not self.paths:
return
with closing(open(self.file_name, 'w')) as f:
self._write(f)
def _write(self, stream):
"""To be implemented by subclasses."""
raise NotImplementedError()
@property
def file_name(self):
"""Subclasses should implement this to return the name of the file
where this module lives."""
raise NotImplementedError()
@property
def use_name(self):
"""Subclasses should implement this to return the name the
module command uses to refer to the package."""
raise NotImplementedError()
def remove(self):
mod_file = self.file_name
if os.path.exists(mod_file):
shutil.rmtree(mod_file, ignore_errors=True)
class Dotkit(EnvModule):
name = 'dotkit'
path = join_path(spack.share_path, "dotkit")
@property
def file_name(self):
return join_path(Dotkit.path, self.pkg.spec.architecture,
self.pkg.spec.format('$_$@$%@$+$#.dk'))
@property
def use_name(self):
return self.pkg.spec.format('$_$@$%@$+$#')
def _write(self, dk_file):
# Category
if self.category:
dk_file.write('#c %s\n' % self.category)
# Short description
if self.short_description:
dk_file.write('#d %s\n' % self.short_description)
# Long description
if self.long_description:
for line in textwrap.wrap(self.long_description, 72):
dk_file.write("#h %s\n" % line)
# Path alterations
for var, dirs in self.paths.items():
for directory in dirs:
dk_file.write("dk_alter %s %s\n" % (var, directory))
# Let CMake find this package.
dk_file.write("dk_alter CMAKE_PREFIX_PATH %s\n" % self.pkg.prefix)
class TclModule(EnvModule):
name = 'tcl'
path = join_path(spack.share_path, "modules")
@property
def file_name(self):
return join_path(TclModule.path, self.pkg.spec.architecture, self.use_name)
@property
def use_name(self):
return self.pkg.spec.format('$_$@$%@$+$#')
def _write(self, m_file):
# TODO: cateogry?
m_file.write('#%Module1.0\n')
# Short description
if self.short_description:
m_file.write('module-whatis \"%s\"\n\n' % self.short_description)
# Long description
if self.long_description:
m_file.write('proc ModulesHelp { } {\n')
doc = re.sub(r'"', '\"', self.long_description)
m_file.write("puts stderr \"%s\"\n" % doc)
m_file.write('}\n\n')
# Path alterations
for var, dirs in self.paths.items():
for directory in dirs:
m_file.write("prepend-path %s \"%s\"\n" % (var, directory))
m_file.write("prepend-path CMAKE_PREFIX_PATH \"%s\"\n" % self.pkg.prefix)

View File

@@ -124,6 +124,16 @@ def installed_package_specs(self):
return installed
def installed_known_package_specs(self):
"""Read installed package names straight from the install
directory layout, but return only specs for which the
package is known to this version of spack.
"""
for spec in spack.install_layout.all_specs():
if self.exists(spec.name):
yield spec
@memoized
def all_package_names(self):
"""Generator function for all packages. This looks for