zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'efischer/160401-RecursiveModules' of https://github.com/citibeth/spack into citibeth-efischer/160401-RecursiveModules

Browse Source
This commit is contained in:
Todd Gamblin 2016-06-30 02:12:00 -07:00
commit 30d083c6b7
3 changed files with 157 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
lib/spack/docs/basic_usage.rst
View File

@ -1131,6 +1131,79 @@ of module files:
"""Set up the compile and runtime environments for a package."""
pass
Recursive Modules
``````````````````
In some cases, it is desirable to load not just a module, but also all
the modules it depends on. This is not required for most modules
because Spack builds binaries with RPATH support. However, not all
packages use RPATH to find their dependencies: this can be true in
particular for Python extensions, which are currently *not* built with
RPATH.
Modules may be loaded recursively with the command:
.. code-block:: sh
$ module load `spack module tcl --dependencies <spec>...
More than one spec may be placed on the command line here.
Module Comamnds for Shell Scripts
``````````````````````````````````
Although Spack is flexbile, the ``module`` command is much faster.
This could become an issue when emitting a series of ``spack load``
commands inside a shell script. By adding the ``--shell`` flag,
``spack module find`` may also be used to generate code that can be
cut-and-pasted into a shell script. For example:
.. code-block:: sh
$ spack module find tcl --dependencies --shell py-numpy git
# bzip2@1.0.6%gcc@4.9.3=linux-x86_64
module load bzip2-1.0.6-gcc-4.9.3-ktnrhkrmbbtlvnagfatrarzjojmkvzsx
# ncurses@6.0%gcc@4.9.3=linux-x86_64
module load ncurses-6.0-gcc-4.9.3-kaazyneh3bjkfnalunchyqtygoe2mncv
# zlib@1.2.8%gcc@4.9.3=linux-x86_64
module load zlib-1.2.8-gcc-4.9.3-v3ufwaahjnviyvgjcelo36nywx2ufj7z
# sqlite@3.8.5%gcc@4.9.3=linux-x86_64
module load sqlite-3.8.5-gcc-4.9.3-a3eediswgd5f3rmto7g3szoew5nhehbr
# readline@6.3%gcc@4.9.3=linux-x86_64
module load readline-6.3-gcc-4.9.3-se6r3lsycrwxyhreg4lqirp6xixxejh3
# python@3.5.1%gcc@4.9.3=linux-x86_64
module load python-3.5.1-gcc-4.9.3-5q5rsrtjld4u6jiicuvtnx52m7tfhegi
# py-setuptools@20.5%gcc@4.9.3=linux-x86_64
module load py-setuptools-20.5-gcc-4.9.3-4qr2suj6p6glepnedmwhl4f62x64wxw2
# py-nose@1.3.7%gcc@4.9.3=linux-x86_64
module load py-nose-1.3.7-gcc-4.9.3-pwhtjw2dvdvfzjwuuztkzr7b4l6zepli
# openblas@0.2.17%gcc@4.9.3+shared=linux-x86_64
module load openblas-0.2.17-gcc-4.9.3-pw6rmlom7apfsnjtzfttyayzc7nx5e7y
# py-numpy@1.11.0%gcc@4.9.3+blas+lapack=linux-x86_64
module load py-numpy-1.11.0-gcc-4.9.3-mulodttw5pcyjufva4htsktwty4qd52r
# curl@7.47.1%gcc@4.9.3=linux-x86_64
module load curl-7.47.1-gcc-4.9.3-ohz3fwsepm3b462p5lnaquv7op7naqbi
# autoconf@2.69%gcc@4.9.3=linux-x86_64
module load autoconf-2.69-gcc-4.9.3-bkibjqhgqm5e3o423ogfv2y3o6h2uoq4
# cmake@3.5.0%gcc@4.9.3~doc+ncurses+openssl~qt=linux-x86_64
module load cmake-3.5.0-gcc-4.9.3-x7xnsklmgwla3ubfgzppamtbqk5rwn7t
# expat@2.1.0%gcc@4.9.3=linux-x86_64
module load expat-2.1.0-gcc-4.9.3-6pkz2ucnk2e62imwakejjvbv6egncppd
# git@2.8.0-rc2%gcc@4.9.3+curl+expat=linux-x86_64
module load git-2.8.0-rc2-gcc-4.9.3-3bib4hqtnv5xjjoq5ugt3inblt4xrgkd
The script may be further edited by removing unnecessary modules.
This script may be directly executed in bash via
.. code-block :: sh
source <( spack module find tcl --dependencies --shell py-numpy git )
Regenerating Module files
~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code-block:: python
def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
@ -1395,23 +1468,23 @@ files in the ``cmake`` package while retaining its dependencies.
.. code-block:: sh
$ spack view -v symlink myview cmake@3.5.2
==> Linking package: "ncurses"
==> Linking package: "zlib"
==> Linking package: "openssl"
==> Linking package: "cmake"
$ ls myview/
bin doc etc include lib share
$ ls myview/bin/
captoinfo clear cpack ctest infotocap openssl tabs toe tset
ccmake cmake c_rehash infocmp ncurses6-config reset tic tput
$ spack view -v -d false rm myview cmake@3.5.2
==> Removing package: "cmake"
$ ls myview/bin/
captoinfo c_rehash infotocap openssl tabs toe tset
clear infocmp ncurses6-config reset tic tput
@ -1421,7 +1494,7 @@ Limitations of Filesystem Views
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
This section describes some limitations that should be considered in
using filesystems views.
using filesystems views.
Filesystem views are merely organizational. The binary executable
programs, shared libraries and other build products found in a view

2
lib/spack/spack/cmd/load.py
View File

@ -31,7 +31,7 @@ def setup_parser(subparser):
"""Parser is only constructed so that this prints a nice help
message with -h. """
subparser.add_argument(
'spec', nargs=argparse.REMAINDER, help='Spec of package to load with modules.')
'spec', nargs=argparse.REMAINDER, help="Spec of package to load with modules. (If -, read specs from STDIN)")
def load(parser, args):

103
lib/spack/spack/cmd/module.py
View File

@ -22,6 +22,7 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from __future__ import print_function
import os
import shutil
import sys
@ -41,46 +42,98 @@ def setup_parser(subparser):
sp.add_parser('refresh', help='Regenerate all module files.')
find_parser = sp.add_parser('find', help='Find module files for packages.')
find_parser.add_argument('module_type',
help="Type of module to find file for. [" +
'|'.join(module_types) + "]")
find_parser.add_argument('spec',
nargs='+',
help='spec to find a module file for.')
find_parser.add_argument(
'module_type',
help="Type of module to find file for. [" +
'|'.join(module_types) + "]")
find_parser.add_argument(
'-r', '--dependencies', action='store_true',
dest='recurse_dependencies',
help='Recursively traverse dependencies for modules to load.')
find_parser.add_argument(
'-s', '--shell', action='store_true', dest='shell',
help='Generate shell script (instead of input for module command)')
find_parser.add_argument(
'-p', '--prefix', dest='prefix',
help='Prepend to module names when issuing module load commands')
find_parser.add_argument(
'spec', nargs='+',
help='spec to find a module file for.')
def module_find(mtype, spec_array):
def module_find(mtype, flags, spec_array):
"""Look at all installed packages and see if the spec provided
matches any. If it does, check whether there is a module file
of type <mtype> there, and print out the name that the user
should type to use that package's module.
prefix:
Prepend this to module names when issuing "module load" commands.
Some systems seem to need it.
"""
if mtype not in module_types:
tty.die("Invalid module type: '%s'. Options are %s" %
(mtype, comma_or(module_types)))
specs = spack.cmd.parse_specs(spec_array)
if len(specs) > 1:
tty.die("You can only pass one spec.")
spec = specs[0]
# --------------------------------------
def _find_modules(spec, modules_list):
"""Finds all modules and sub-modules for a spec"""
if str(spec.version) == 'system':
# No Spack module for system-installed packages
return
specs = spack.installed_db.query(spec)
if len(specs) == 0:
tty.die("No installed packages match spec %s" % spec)
if flags.recurse_dependencies:
for dep in spec.dependencies.values():
_find_modules(dep, modules_list)
if len(specs) > 1:
tty.error("Multiple matches for spec %s. Choose one:" % spec)
for s in specs:
sys.stderr.write(s.tree(color=True))
sys.exit(1)
mod = module_types[mtype](spec)
if not os.path.isfile(mod.file_name):
tty.die("No %s module is installed for %s" % (mtype, spec))
modules_list.append((spec, mod))
mt = module_types[mtype]
mod = mt(specs[0])
if not os.path.isfile(mod.file_name):
tty.die("No %s module is installed for %s" % (mtype, spec))
print(mod.use_name)
# --------------------------------------
raw_specs = spack.cmd.parse_specs(spec_array)
modules = set() # Modules we will load
seen = set()
for raw_spec in raw_specs:
# ----------- Make sure the spec only resolves to ONE thing
specs = spack.installed_db.query(raw_spec)
if len(specs) == 0:
tty.die("No installed packages match spec %s" % raw_spec)
if len(specs) > 1:
tty.error("Multiple matches for spec %s. Choose one:" % spec)
for s in specs:
sys.stderr.write(s.tree(color=True))
sys.exit(1)
spec = specs[0]
# ----------- Chase down modules for it and all its dependencies
modules_dups = list()
_find_modules(spec, modules_dups)
# Remove duplicates while keeping order
modules_unique = list()
for spec,mod in modules_dups:
if mod.use_name not in seen:
modules_unique.append((spec,mod))
seen.add(mod.use_name)
# Output...
if flags.shell:
module_cmd = {'tcl': 'module load', 'dotkit': 'dotkit use'}[mtype]
for spec,mod in modules_unique:
if flags.shell:
print('# %s' % spec.format())
print('%s %s%s' % (module_cmd, flags.prefix, mod.use_name))
else:
print(mod.use_name)
def module_refresh():
"""Regenerate all module files for installed packages known to
@ -101,4 +154,4 @@ def module(parser, args):
module_refresh()
elif args.module_command == 'find':
module_find(args.module_type, args.spec)
module_find(args.module_type, args, args.spec)