Merge branch 'efischer/160401-RecursiveModules' of https://github.com/citibeth/spack into citibeth-efischer/160401-RecursiveModules

lib/spack/docs/basic_usage.rst

@@ -1131,6 +1131,79 @@ of module files:
       """Set up the compile and runtime environments for a package."""
       pass
 
+
+Recursive Modules
+``````````````````
+
+In some cases, it is desirable to load not just a module, but also all
+the modules it depends on.  This is not required for most modules
+because Spack builds binaries with RPATH support.  However, not all
+packages use RPATH to find their dependencies: this can be true in
+particular for Python extensions, which are currently *not* built with
+RPATH.
+
+Modules may be loaded recursively with the command:
+
+.. code-block:: sh
+
+    $ module load `spack module tcl --dependencies <spec>...
+
+More than one spec may be placed on the command line here.
+
+Module Comamnds for Shell Scripts
+``````````````````````````````````
+
+Although Spack is flexbile, the ``module`` command is much faster.
+This could become an issue when emitting a series of ``spack load``
+commands inside a shell script.  By adding the ``--shell`` flag,
+``spack module find`` may also be used to generate code that can be
+cut-and-pasted into a shell script.  For example:
+
+.. code-block:: sh
+
+    $ spack module find tcl --dependencies --shell py-numpy git
+    # bzip2@1.0.6%gcc@4.9.3=linux-x86_64
+    module load bzip2-1.0.6-gcc-4.9.3-ktnrhkrmbbtlvnagfatrarzjojmkvzsx
+    # ncurses@6.0%gcc@4.9.3=linux-x86_64
+    module load ncurses-6.0-gcc-4.9.3-kaazyneh3bjkfnalunchyqtygoe2mncv
+    # zlib@1.2.8%gcc@4.9.3=linux-x86_64
+    module load zlib-1.2.8-gcc-4.9.3-v3ufwaahjnviyvgjcelo36nywx2ufj7z
+    # sqlite@3.8.5%gcc@4.9.3=linux-x86_64
+    module load sqlite-3.8.5-gcc-4.9.3-a3eediswgd5f3rmto7g3szoew5nhehbr
+    # readline@6.3%gcc@4.9.3=linux-x86_64
+    module load readline-6.3-gcc-4.9.3-se6r3lsycrwxyhreg4lqirp6xixxejh3
+    # python@3.5.1%gcc@4.9.3=linux-x86_64
+    module load python-3.5.1-gcc-4.9.3-5q5rsrtjld4u6jiicuvtnx52m7tfhegi
+    # py-setuptools@20.5%gcc@4.9.3=linux-x86_64
+    module load py-setuptools-20.5-gcc-4.9.3-4qr2suj6p6glepnedmwhl4f62x64wxw2
+    # py-nose@1.3.7%gcc@4.9.3=linux-x86_64
+    module load py-nose-1.3.7-gcc-4.9.3-pwhtjw2dvdvfzjwuuztkzr7b4l6zepli
+    # openblas@0.2.17%gcc@4.9.3+shared=linux-x86_64
+    module load openblas-0.2.17-gcc-4.9.3-pw6rmlom7apfsnjtzfttyayzc7nx5e7y
+    # py-numpy@1.11.0%gcc@4.9.3+blas+lapack=linux-x86_64
+    module load py-numpy-1.11.0-gcc-4.9.3-mulodttw5pcyjufva4htsktwty4qd52r
+    # curl@7.47.1%gcc@4.9.3=linux-x86_64
+    module load curl-7.47.1-gcc-4.9.3-ohz3fwsepm3b462p5lnaquv7op7naqbi
+    # autoconf@2.69%gcc@4.9.3=linux-x86_64
+    module load autoconf-2.69-gcc-4.9.3-bkibjqhgqm5e3o423ogfv2y3o6h2uoq4
+    # cmake@3.5.0%gcc@4.9.3~doc+ncurses+openssl~qt=linux-x86_64
+    module load cmake-3.5.0-gcc-4.9.3-x7xnsklmgwla3ubfgzppamtbqk5rwn7t
+    # expat@2.1.0%gcc@4.9.3=linux-x86_64
+    module load expat-2.1.0-gcc-4.9.3-6pkz2ucnk2e62imwakejjvbv6egncppd
+    # git@2.8.0-rc2%gcc@4.9.3+curl+expat=linux-x86_64
+    module load git-2.8.0-rc2-gcc-4.9.3-3bib4hqtnv5xjjoq5ugt3inblt4xrgkd
+
+The script may be further edited by removing unnecessary modules.
+This script may be directly executed in bash via
+
+.. code-block :: sh
+
+    source <( spack module find tcl --dependencies --shell py-numpy git )
+
+
+Regenerating Module files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 .. code-block:: python
 
   def setup_dependent_environment(self, spack_env, run_env, dependent_spec):

lib/spack/spack/cmd/load.py

@@ -31,7 +31,7 @@ def setup_parser(subparser):
     """Parser is only constructed so that this prints a nice help
        message with -h. """
     subparser.add_argument(
-        'spec', nargs=argparse.REMAINDER, help='Spec of package to load with modules.')
+        'spec', nargs=argparse.REMAINDER, help="Spec of package to load with modules.  (If -, read specs from STDIN)")
 
 
 def load(parser, args):

lib/spack/spack/cmd/module.py

@@ -22,6 +22,7 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+from __future__ import print_function
 import os
 import shutil
 import sys
@@ -41,47 +42,99 @@ def setup_parser(subparser):
     sp.add_parser('refresh', help='Regenerate all module files.')
 
     find_parser = sp.add_parser('find', help='Find module files for packages.')
-    find_parser.add_argument('module_type',
+
+    find_parser.add_argument(
+        'module_type',
         help="Type of module to find file for. [" +
         '|'.join(module_types) + "]")
-    find_parser.add_argument('spec',
+
-                             nargs='+',
+    find_parser.add_argument(
+        '-r', '--dependencies', action='store_true',
+        dest='recurse_dependencies',
+        help='Recursively traverse dependencies for modules to load.')
+
+    find_parser.add_argument(
+        '-s', '--shell', action='store_true', dest='shell',
+        help='Generate shell script (instead of input for module command)')
+
+    find_parser.add_argument(
+        '-p', '--prefix', dest='prefix',
+        help='Prepend to module names when issuing module load commands')
+
+    find_parser.add_argument(
+        'spec', nargs='+',
         help='spec to find a module file for.')
 
 
-def module_find(mtype, spec_array):
+def module_find(mtype, flags, spec_array):
     """Look at all installed packages and see if the spec provided
        matches any.  If it does, check whether there is a module file
        of type <mtype> there, and print out the name that the user
        should type to use that package's module.
+    prefix:
+        Prepend this to module names when issuing "module load" commands.
+        Some systems seem to need it.
     """
     if mtype not in module_types:
         tty.die("Invalid module type: '%s'.  Options are %s" %
                 (mtype, comma_or(module_types)))
 
-    specs = spack.cmd.parse_specs(spec_array)
+    # --------------------------------------
-    if len(specs) > 1:
+    def _find_modules(spec, modules_list):
-        tty.die("You can only pass one spec.")
+        """Finds all modules and sub-modules for a spec"""
-    spec = specs[0]
+        if str(spec.version) == 'system':
+            # No Spack module for system-installed packages
+            return
 
-    specs = spack.installed_db.query(spec)
+        if flags.recurse_dependencies:
+            for dep in spec.dependencies.values():
+                _find_modules(dep, modules_list)
+
+        mod = module_types[mtype](spec)
+        if not os.path.isfile(mod.file_name):
+            tty.die("No %s module is installed for %s" % (mtype, spec))
+        modules_list.append((spec, mod))
+
+
+    # --------------------------------------
+    raw_specs = spack.cmd.parse_specs(spec_array)
+    modules = set()    # Modules we will load
+    seen = set()
+    for raw_spec in raw_specs:
+
+        # ----------- Make sure the spec only resolves to ONE thing
+        specs = spack.installed_db.query(raw_spec)
         if len(specs) == 0:
-        tty.die("No installed packages match spec %s" % spec)
+            tty.die("No installed packages match spec %s" % raw_spec)
 
         if len(specs) > 1:
             tty.error("Multiple matches for spec %s.  Choose one:" % spec)
             for s in specs:
                 sys.stderr.write(s.tree(color=True))
             sys.exit(1)
+        spec = specs[0]
 
-    mt = module_types[mtype]
+        # ----------- Chase down modules for it and all its dependencies
-    mod = mt(specs[0])
+        modules_dups = list()
-    if not os.path.isfile(mod.file_name):
+        _find_modules(spec, modules_dups)
-        tty.die("No %s module is installed for %s" % (mtype, spec))
 
+        # Remove duplicates while keeping order
+        modules_unique = list()
+        for spec,mod in modules_dups:
+            if mod.use_name not in seen:
+                modules_unique.append((spec,mod))
+                seen.add(mod.use_name)
+
+        # Output...
+        if flags.shell:
+            module_cmd = {'tcl': 'module load', 'dotkit': 'dotkit use'}[mtype]
+        for spec,mod in modules_unique:
+            if flags.shell:
+                print('# %s' % spec.format())
+                print('%s %s%s' % (module_cmd, flags.prefix, mod.use_name))
+            else:
                 print(mod.use_name)
 
-
 def module_refresh():
     """Regenerate all module files for installed packages known to
        spack (some packages may no longer exist)."""
@@ -101,4 +154,4 @@ def module(parser, args):
         module_refresh()
 
     elif args.module_command == 'find':
-        module_find(args.module_type, args.spec)
+        module_find(args.module_type, args, args.spec)