Merge branch 'develop' of https://github.com/LLNL/spack into packages/blas_lapack_providers

Conflicts:
	var/spack/repos/builtin/packages/netlib-lapack/package.py

.travis.yml

@@ -16,7 +16,10 @@ before_install:
 
 script:
   - . share/spack/setup-env.sh
+  - spack compilers
+  - spack config get compilers
   - spack test
+  - spack install -v libdwarf
 
 notifications:
   email:

bin/sbang

@@ -0,0 +1,84 @@
+#!/bin/bash
+#
+# `sbang`: Run scripts with long shebang lines.
+#
+# Many operating systems limit the length of shebang lines, making it
+# hard to use interpreters that are deep in the directory hierarchy.
+# `sbang` can run such scripts, either as a shebang interpreter, or
+# directly on the command line.
+#
+# Usage
+# -----------------------------
+# Suppose you have a script, long-shebang.sh, like this:
+#
+#     1    #!/very/long/path/to/some/interpreter
+#     2
+#     3    echo "success!"
+#
+# Invoking this script will result in an error on some OS's.  On
+# Linux, you get this:
+#
+#     $ ./long-shebang.sh
+#     -bash: ./long: /very/long/path/to/some/interp: bad interpreter:
+#            No such file or directory
+#
+# On Mac OS X, the system simply assumes the interpreter is the shell
+# and tries to run with it, which is likely not what you want.
+#
+#
+# `sbang` on the command line
+# -----------------------------
+# You can use `sbang` in two ways.  The first is to use it directly,
+# from the command line, like this:
+#
+#     $ sbang ./long-shebang.sh
+#     success!
+#
+#
+# `sbang` as the interpreter
+# -----------------------------
+# You can also use `sbang` *as* the interpreter for your script. Put
+# `#!/bin/bash /path/to/sbang` on line 1, and move the original
+# shebang to line 2 of the script:
+#
+#     1    #!/bin/bash /path/to/sbang
+#     2    #!/long/path/to/real/interpreter with arguments
+#     3
+#     4    echo "success!"
+#
+#     $ ./long-shebang.sh
+#     success!
+#
+# On Linux, you could shorten line 1 to `#!/path/to/sbang`, but other
+# operating systems like Mac OS X require the interpreter to be a
+# binary, so it's best to use `sbang` as a `bash` argument.
+# Obviously, for this to work, `sbang` needs to have a short enough
+# path that *it* will run without hitting OS limits.
+#
+#
+# How it works
+# -----------------------------
+# `sbang` is a very simple bash script. It looks at the first two
+# lines of a script argument and runs the last line starting with
+# `#!`, with the script as an argument. It also forwards arguments.
+#
+
+# First argument is the script we want to actually run.
+script="$1"
+
+# Search the first two lines of script for interpreters.
+lines=0
+while read line && ((lines < 2)) ; do
+    if [[ "$line" = '#!'* ]]; then
+        interpreter="${line#\#!}"
+    fi
+    lines=$((lines+1))
+done < "$script"
+
+# Invoke any interpreter found, or raise an error if none was found.
+if [ -n "$interpreter" ]; then
+    exec $interpreter "$@"
+else
+    echo "error: sbang found no interpreter in $script"
+    exit 1
+fi

lib/spack/docs/basic_usage.rst

@@ -357,7 +357,7 @@ Spack, you can simply run ``spack compiler add`` with the path to
 where the compiler is installed.  For example::
 
     $ spack compiler add /usr/local/tools/ic-13.0.079
-    ==> Added 1 new compiler to /Users/gamblin2/.spackconfig
+    ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml
         intel@13.0.079
 
 Or you can run ``spack compiler add`` with no arguments to force
@@ -367,7 +367,7 @@ installed, but you know that new compilers have been added to your
 
     $ module load gcc-4.9.0
     $ spack compiler add
-    ==> Added 1 new compiler to /Users/gamblin2/.spackconfig
+    ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml
         gcc@4.9.0
 
 This loads the environment module for gcc-4.9.0 to get it into the
@@ -398,27 +398,34 @@ Manual compiler configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If auto-detection fails, you can manually configure a compiler by
-editing your ``~/.spackconfig`` file.  You can do this by running
+editing your ``~/.spack/compilers.yaml`` file.  You can do this by running
-``spack config edit``, which will open the file in your ``$EDITOR``.
+``spack config edit compilers``, which will open the file in your ``$EDITOR``.
 
 Each compiler configuration in the file looks like this::
 
     ...
-    [compiler "intel@15.0.0"]
+    chaos_5_x86_64_ib:
-        cc = /usr/local/bin/icc-15.0.024-beta
-        cxx = /usr/local/bin/icpc-15.0.024-beta
-        f77 = /usr/local/bin/ifort-15.0.024-beta
-        fc = /usr/local/bin/ifort-15.0.024-beta
       ...
+      intel@15.0.0:
+          cc: /usr/local/bin/icc-15.0.024-beta
+          cxx: /usr/local/bin/icpc-15.0.024-beta
+          f77: /usr/local/bin/ifort-15.0.024-beta
+          fc: /usr/local/bin/ifort-15.0.024-beta
+      ...
+
+The chaos_5_x86_64_ib string is an architecture string, and multiple
+compilers can be listed underneath an architecture.  The architecture
+string may be replaced with the string 'all' to signify compilers that
+work on all architectures.
 
 For compilers, like ``clang``, that do not support Fortran, put
 ``None`` for ``f77`` and ``fc``::
 
-    [compiler "clang@3.3svn"]
+    clang@3.3svn:
-        cc = /usr/bin/clang
+        cc: /usr/bin/clang
-        cxx = /usr/bin/clang++
+        cxx: /usr/bin/clang++
-        f77 = None
+        f77: None
-        fc = None
+        fc: None
 
 Once you save the file, the configured compilers will show up in the
 list displayed by ``spack compilers``.

lib/spack/docs/developer_guide.rst

@@ -73,19 +73,32 @@ with a high level view of Spack's directory structure::
   spack/                  <- installation root
      bin/
         spack             <- main spack executable
+
+     etc/
+        spack/            <- Spack config files.
+                             Can be overridden by files in ~/.spack.
+
      var/
         spack/            <- build & stage directories
+        repos/            <- contains package repositories
+           builtin/       <- pkg repository that comes with Spack
+              repo.yaml   <- descriptor for the builtin repository
+              packages/   <- directories under here contain packages
+
      opt/
         spack/            <- packages are installed here
+
      lib/
         spack/
            docs/          <- source for this documentation
            env/           <- compiler wrappers for build environment
 
+           external/      <- external libs included in Spack distro
+           llnl/          <- some general-use libraries
+
            spack/         <- spack module; contains Python code
               cmd/        <- each file in here is a spack subcommand
               compilers/  <- compiler description files
-              packages/   <- each file in here is a spack package
               test/       <- unit test modules
               util/       <- common code
 

lib/spack/docs/features.rst

@@ -103,7 +103,7 @@ creates a simple python file:
 It doesn't take much python coding to get from there to a working
 package:
 
-.. literalinclude:: ../../../var/spack/packages/libelf/package.py
+.. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py
    :lines: 25-
 
 Spack also provides wrapper functions around common commands like

lib/spack/docs/index.rst

@@ -18,7 +18,7 @@ configurations can coexist on the same system.
 Most importantly, Spack is *simple*.  It offers a simple *spec* syntax
 so that users can specify versions and configuration options
 concisely.  Spack is also simple for package authors: package files
-are writtin in pure Python, and specs allow package authors to
+are written in pure Python, and specs allow package authors to
 maintain a single file for many different builds of the same package.
 
 See the :doc:`features` for examples and highlights.

lib/spack/docs/mirrors.rst

@@ -38,7 +38,7 @@ contains tarballs for each package, named after each package.
 
 .. note::
 
-   Archives are **not** named exactly they were in the package's fetch
+   Archives are **not** named exactly the way they were in the package's fetch
    URL.  They have the form ``<name>-<version>.<extension>``, where
    ``<name>`` is Spack's name for the package, ``<version>`` is the
    version of the tarball, and ``<extension>`` is whatever format the
@@ -186,7 +186,7 @@ Each mirror has a name so that you can refer to it again later.
 ``spack mirror list``
 ----------------------------
 
-If you want to see all the mirrors Spack knows about you can run ``spack mirror list``::
+To see all the mirrors Spack knows about, run ``spack mirror list``::
 
    $ spack mirror list
    local_filesystem    file:///Users/gamblin2/spack-mirror-2014-06-24
@@ -196,7 +196,7 @@ If you want to see all the mirrors Spack knows about you can run ``spack mirror
 ``spack mirror remove``
 ----------------------------
 
-And, if you want to remove a mirror, just remove it by name::
+To remove a mirror by name::
 
    $ spack mirror remove local_filesystem
    $ spack mirror list
@@ -205,12 +205,11 @@ And, if you want to remove a mirror, just remove it by name::
 Mirror precedence
 ----------------------------
 
-Adding a mirror really just adds a section in ``~/.spackconfig``::
+Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``::
 
-   [mirror "local_filesystem"]
+   mirrors:
-       url = file:///Users/gamblin2/spack-mirror-2014-06-24
+     local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24
-   [mirror "remote_server"]
+     remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24
-       url = https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24
 
 If you want to change the order in which mirrors are searched for
 packages, you can edit this file and reorder the sections.  Spack will

lib/spack/docs/packaging_guide.rst

@@ -84,7 +84,7 @@ always choose to download just one tarball initially, and run
 
    If it fails entirely, you can get minimal boilerplate by using
    :ref:`spack-edit-f`, or you can manually create a directory and
-   ``package.py`` file for the package in ``var/spack/packages``.
+   ``package.py`` file for the package in ``var/spack/repos/builtin/packages``.
 
 .. note::
 
@@ -203,7 +203,7 @@ edit`` command:
 So, if you used ``spack create`` to create a package, then saved and
 closed the resulting file, you can get back to it with ``spack edit``.
 The ``cmake`` package actually lives in
-``$SPACK_ROOT/var/spack/packages/cmake/package.py``, but this provides
+``$SPACK_ROOT/var/spack/repos/builtin/packages/cmake/package.py``, but this provides
 a much simpler shortcut and saves you the trouble of typing the full
 path.
 
@@ -269,18 +269,18 @@ live in Spack's directory structure.  In general, `spack-create`_ and
 `spack-edit`_ handle creating package files for you, so you can skip
 most of the details here.
 
-``var/spack/packages``
+``var/spack/repos/builtin/packages``
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 A Spack installation directory is structured like a standard UNIX
 install prefix (``bin``, ``lib``, ``include``, ``var``, ``opt``,
 etc.).  Most of the code for Spack lives in ``$SPACK_ROOT/lib/spack``.
-Packages themselves live in ``$SPACK_ROOT/var/spack/packages``.
+Packages themselves live in ``$SPACK_ROOT/var/spack/repos/builtin/packages``.
 
 If you ``cd`` to that directory, you will see directories for each
 package:
 
-.. command-output::  cd $SPACK_ROOT/var/spack/packages;  ls -CF
+.. command-output::  cd $SPACK_ROOT/var/spack/repos/builtin/packages;  ls -CF
    :shell:
    :ellipsis: 10
 
@@ -288,7 +288,7 @@ Each directory contains a file called ``package.py``, which is where
 all the python code for the package goes.  For example, the ``libelf``
 package lives in::
 
-   $SPACK_ROOT/var/spack/packages/libelf/package.py
+   $SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py
 
 Alongside the ``package.py`` file, a package may contain extra
 directories or files (like patches) that it needs to build.
@@ -301,7 +301,7 @@ Packages are named after the directory containing ``package.py``.  So,
 ``libelf``'s ``package.py`` lives in a directory called ``libelf``.
 The ``package.py`` file defines a class called ``Libelf``, which
 extends Spack's ``Package`` class.  for example, here is
-``$SPACK_ROOT/var/spack/packages/libelf/package.py``:
+``$SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py``:
 
 .. code-block:: python
    :linenos:
@@ -328,7 +328,7 @@ these:
    $ spack install libelf@0.8.13
 
 Spack sees the package name in the spec and looks for
-``libelf/package.py`` in ``var/spack/packages``.  Likewise, if you say
+``libelf/package.py`` in ``var/spack/repos/builtin/packages``.  Likewise, if you say
 ``spack install py-numpy``, then Spack looks for
 ``py-numpy/package.py``.
 
@@ -401,6 +401,35 @@ construct the new one for ``8.2.1``.
 When you supply a custom URL for a version, Spack uses that URL
 *verbatim* and does not perform extrapolation.
 
+Skipping the expand step
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Spack normally expands archives automatically after downloading
+them. If you want to skip this step (e.g., for self-extracting
+executables and other custom archive types), you can add
+``expand=False`` to a ``version`` directive.
+
+.. code-block:: python
+
+   version('8.2.1', '4136d7b4c04df68b686570afa26988ac',
+           url='http://example.com/foo-8.2.1-special-version.tar.gz', 'expand=False')
+
+When ``expand`` is set to ``False``, Spack sets the current working
+directory to the directory containing the downloaded archive before it
+calls your ``install`` method.  Within ``install``, the path to the
+downloaded archive is available as ``self.stage.archive_file``.
+
+Here is an example snippet for packages distributed as self-extracting
+archives.  The example sets permissions on the downloaded file to make
+it executable, then runs it with some arguments.
+
+.. code-block:: python
+
+   def install(self, spec, prefix):
+       set_executable(self.stage.archive_file)
+       installer = Executable(self.stage.archive_file)
+       installer('--prefix=%s' % prefix, 'arg1', 'arg2', 'etc.')
+
 Checksums
 ~~~~~~~~~~~~~~~~~
 
@@ -703,7 +732,7 @@ supply is a filename, then the patch needs to live within the spack
 source tree.  For example, the patch above lives in a directory
 structure like this::
 
-   $SPACK_ROOT/var/spack/packages/
+   $SPACK_ROOT/var/spack/repos/builtin/packages/
        mvapich2/
            package.py
            ad_lustre_rwcontig_open_source.patch
@@ -1524,6 +1553,69 @@ This is useful when you want to know exactly what Spack will do when
 you ask for a particular spec.
 
 
+``Concretization Policies``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A user may have certain preferences for how packages should
+be concretized on their system.  For example, one user may prefer packages
+built with OpenMPI and the Intel compiler.  Another user may prefer
+packages be built with MVAPICH and GCC.
+
+Spack can be configured to prefer certain compilers, package
+versions, depends_on, and variants during concretization.
+The preferred configuration can be controlled via the
+``~/.spack/packages.yaml`` file for user configuations, or the
+``etc/spack/packages.yaml`` site configuration.
+
+
+Here's an example packages.yaml file that sets preferred packages:
+
+.. code-block:: sh
+
+    packages:
+      dyninst:
+        compiler: [gcc@4.9]
+        variants: +debug
+      gperftools:
+        version: [2.2, 2.4, 2.3]
+      all:
+        compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi]
+        providers:
+          mpi: [mvapich, mpich, openmpi]
+
+
+At a high level, this example is specifying how packages should be
+concretized.  The dyninst package should prefer using gcc 4.9 and
+be built with debug options.  The gperftools package should prefer version
+2.2 over 2.4.  Every package on the system should prefer mvapich for
+its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9).
+These options are used to fill in implicit defaults.  Any of them can be overwritten
+on the command line if explicitly requested.
+
+Each packages.yaml file begins with the string ``packages:`` and
+package names are specified on the next level. The special string ``all``
+applies settings to each package. Underneath each package name is
+one or more components: ``compiler``, ``variants``, ``version``,
+or ``providers``.  Each component has an ordered list of spec
+``constraints``, with earlier entries in the list being preferred over
+later entries.
+
+Sometimes a package installation may have constraints that forbid
+the first concretization rule, in which case Spack will use the first
+legal concretization rule.  Going back to the example, if a user
+requests gperftools 2.3 or later, then Spack will install version 2.4
+as the 2.4 version of gperftools is preferred over 2.3.
+
+An explicit concretization rule in the preferred section will always
+take preference over unlisted concretizations.  In the above example,
+xlc isn't listed in the compiler list.  Every listed compiler from
+gcc to pgi will thus be preferred over the xlc compiler.
+
+The syntax for the ``provider`` section differs slightly from other
+concretization rules.  A provider lists a value that packages may
+``depend_on`` (e.g, mpi) and a list of rules for fulfilling that
+dependency.
+
 .. _install-method:
 
 Implementing the ``install`` method
@@ -1533,7 +1625,7 @@ The last element of a package is its ``install()`` method.  This is
 where the real work of installation happens, and it's the main part of
 the package you'll need to customize for each piece of software.
 
-.. literalinclude::  ../../../var/spack/packages/libelf/package.py
+.. literalinclude::  ../../../var/spack/repos/builtin/packages/libelf/package.py
    :start-after: 0.8.12
    :linenos:
 
@@ -1711,15 +1803,15 @@ Compile-time library search paths
   * ``-L$dep_prefix/lib``
   * ``-L$dep_prefix/lib64``
 Runtime library search paths (RPATHs)
-  * ``-Wl,-rpath=$dep_prefix/lib``
+  * ``-Wl,-rpath,$dep_prefix/lib``
-  * ``-Wl,-rpath=$dep_prefix/lib64``
+  * ``-Wl,-rpath,$dep_prefix/lib64``
 Include search paths
   * ``-I$dep_prefix/include``
 
 An example of this would be the ``libdwarf`` build, which has one
 dependency: ``libelf``.  Every call to ``cc`` in the ``libdwarf``
 build will have ``-I$LIBELF_PREFIX/include``,
-``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath=$LIBELF_PREFIX/lib``
+``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath,$LIBELF_PREFIX/lib``
 inserted on the command line.  This is done transparently to the
 project's build system, which will just think it's using a system
 where ``libelf`` is readily available.  Because of this, you **do
@@ -1752,6 +1844,20 @@ dedicated process.
 
 .. _prefix-objects:
 
+
+Failing the build
+----------------------
+
+Sometimes you don't want a package to successfully install unless some
+condition is true.  You can explicitly cause the build to fail from
+``install()`` by raising an ``InstallError``, for example:
+
+.. code-block:: python
+
+   if spec.architecture.startswith('darwin'):
+       raise InstallError('This package does not build on Mac OS X!')
+
+
 Prefix objects
 ----------------------
 
@@ -2068,6 +2174,62 @@ package, this allows us to avoid race conditions in the library's
 build system.
 
 
+.. _sanity-checks:
+
+Sanity checking an intallation
+--------------------------------
+
+By default, Spack assumes that a build has failed if nothing is
+written to the install prefix, and that it has succeeded if anything
+(a file, a directory, etc.)  is written to the install prefix after
+``install()`` completes.
+
+Consider a simple autotools build like this:
+
+.. code-block:: python
+
+   def install(self, spec, prefix):
+       configure("--prefix=" + prefix)
+       make()
+       make("install")
+
+If you are using using standard autotools or CMake, ``configure`` and
+``make`` will not write anything to the install prefix.  Only ``make
+install`` writes the files, and only once the build is already
+complete.  Not all builds are like this.  Many builds of scientific
+software modify the install prefix *before* ``make install``. Builds
+like this can falsely report that they were successfully installed if
+an error occurs before the install is complete but after files have
+been written to the ``prefix``.
+
+
+``sanity_check_is_file`` and ``sanity_check_is_dir``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can optionally specify *sanity checks* to deal with this problem.
+Add properties like this to your package:
+
+.. code-block:: python
+
+   class MyPackage(Package):
+       ...
+
+       sanity_check_is_file = ['include/libelf.h']
+       sanity_check_is_dir  = [lib]
+
+       def install(self, spec, prefix):
+           configure("--prefix=" + prefix)
+           make()
+           make("install")
+
+Now, after ``install()`` runs, Spack will check whether
+``$prefix/include/libelf.h`` exists and is a file, and whether
+``$prefix/lib`` exists and is a directory.  If the checks fail, then
+the build will fail and the install prefix will be removed.  If they
+succeed, Spack considers the build succeeful and keeps the prefix in
+place.
+
+
 .. _file-manipulation:
 
 File manipulation functions
@@ -2108,6 +2270,15 @@ Filtering functions
 
   Examples:
 
+  #. Filtering a Makefile to force it to use Spack's compiler wrappers:
+
+     .. code-block:: python
+
+        filter_file(r'^CC\s*=.*',  spack_cc,  'Makefile')
+        filter_file(r'^CXX\s*=.*', spack_cxx, 'Makefile')
+        filter_file(r'^F77\s*=.*', spack_f77, 'Makefile')
+        filter_file(r'^FC\s*=.*',  spack_fc,  'Makefile')
+
   #. Replacing ``#!/usr/bin/perl`` with ``#!/usr/bin/env perl`` in ``bib2xhtml``:
 
      .. code-block:: python

lib/spack/docs/site_configuration.rst

@@ -54,87 +54,73 @@ more elements to the list to indicate where your own site's temporary
 directory is.
 
 
-.. _concretization-policies:
+External Packages
+~~~~~~~~~~~~~~~~~~~~~
+Spack can be configured to use externally-installed
+packages rather than building its own packages. This may be desirable
+if machines ship with system packages, such as a customized MPI
+that should be used instead of Spack building its own MPI.
 
-Concretization policies
+External packages are configured through the ``packages.yaml`` file found
-----------------------------
+in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/``
+directory. Here's an example of an external configuration:
 
-When a user asks for a package like ``mpileaks`` to be installed,
+.. code-block:: yaml
-Spack has to make decisions like what version should be installed,
-what compiler to use, and how its dependencies should be configured.
-This process is called *concretization*, and it's covered in detail in
-:ref:`its own section <abstract-and-concrete>`.
 
-The default concretization policies are in the
+   packages:
-:py:mod:`spack.concretize` module, specifically in the
+      openmpi:
-:py:class:`spack.concretize.DefaultConcretizer` class.  These are the
+         paths:
-important methods used in the concretization process:
+            openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3
+            openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug
+            openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel
 
-* :py:meth:`concretize_version(self, spec) <spack.concretize.DefaultConcretizer.concretize_version>`
+This example lists three installations of OpenMPI, one built with gcc,
-* :py:meth:`concretize_architecture(self, spec) <spack.concretize.DefaultConcretizer.concretize_architecture>`
+one built with gcc and debug information, and another built with Intel.
-* :py:meth:`concretize_compiler(self, spec) <spack.concretize.DefaultConcretizer.concretize_compiler>`
+If Spack is asked to build a package that uses one of these MPIs as a
-* :py:meth:`choose_provider(self, spec, providers) <spack.concretize.DefaultConcretizer.choose_provider>`
+dependency, it will use the the pre-installed OpenMPI in
+the given directory.  
 
-The first three take a :py:class:`Spec <spack.spec.Spec>` object and
+Each ``packages.yaml`` begins with a ``packages:`` token, followed
-modify it by adding constraints for the version.  For example, if the
+by a list of package names.  To specify externals, add a ``paths``
-input spec had a version range like `1.0:5.0.3`, then the
+token under the package name, which lists externals in a
-``concretize_version`` method should set the spec's version to a
+``spec : /path`` format.  Each spec should be as
-*single* version in that range.  Likewise, ``concretize_architecture``
+well-defined as reasonably possible.  If a
-selects an architecture when the input spec does not have one, and
+package lacks a spec component, such as missing a compiler or
-``concretize_compiler`` needs to set both a concrete compiler and a
+package version, then Spack will guess the missing component based
-concrete compiler version.
+on its most-favored packages, and it may guess incorrectly.
 
-``choose_provider()`` affects how concrete implementations are chosen
+Each package version and compilers listed in an external should
-based on a virtual dependency spec.  The input spec is some virtual
+have entries in Spack's packages and compiler configuration, even
-dependency and the ``providers`` index is a :py:class:`ProviderIndex
+though the package and compiler may not every be built.
-<spack.packages.ProviderIndex>` object.  The ``ProviderIndex`` maps
-the virtual spec to specs for possible implementations, and
-``choose_provider()`` should simply choose one of these.  The
-``concretize_*`` methods will be called on the chosen implementation
-later, so there is no need to fully concretize the spec when returning
-it.
 
-The ``DefaultConcretizer`` is intended to provide sensible defaults
+The packages configuration can tell Spack to use an external location
-for each policy, but there are certain choices that it can't know
+for certain package versions, but it does not restrict Spack to using
-about.  For example, one site might prefer ``OpenMPI`` over ``MPICH``,
+external packages.  In the above example, if an OpenMPI 1.8.4 became
-or another might prefer an old version of some packages.  These types
+available Spack may choose to start building and linking with that version
-of special cases can be integrated with custom concretizers.
+rather than continue using the pre-installed OpenMPI versions.
 
-Writing a custom concretizer
+To prevent this, the ``packages.yaml`` configuration also allows packages
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+to be flagged as non-buildable.  The previous example could be modified to
+be:
 
-To write your own concretizer, you need only subclass
+.. code-block:: yaml
-``DefaultConcretizer`` and override the methods you want to change.
-For example, you might write a class like this to change *only* the
-``concretize_version()`` behavior:
 
-.. code-block:: python
+  packages:
+    openmpi:
+      paths:
+        openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3
+        openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug
+        openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel                
+      buildable: False
 
-   from spack.concretize import DefaultConcretizer
+The addition of the ``buildable`` flag tells Spack that it should never build
+its own version of OpenMPI, and it will instead always rely on a pre-built
+OpenMPI.  Similar to ``paths``, ``buildable`` is specified as a property under
+a package name.
 
-   class MyConcretizer(DefaultConcretizer):
+The ``buildable`` does not need to be paired with external packages.  
-       def concretize_version(self, spec):
+It could also be used alone to forbid packages that may be 
-           # implement custom logic here.
+buggy or otherwise undesirable.
-
-Once you have written your custom concretizer, you can make Spack use
-it by editing ``globals.py``.  Find this part of the file:
-
-.. code-block:: python
-
-   #
-   # This controls how things are concretized in spack.
-   # Replace it with a subclass if you want different
-   # policies.
-   #
-   concretizer = DefaultConcretizer()
-
-Set concretizer to *your own* class instead of the default:
-
-.. code-block:: python
-
-   concretizer = MyConcretizer()
-
-The next time you run Spack, your changes should take effect.
 
 
 Profiling

lib/spack/env/cc

@@ -90,15 +90,15 @@ case "$command" in
         command="$SPACK_CC"
         language="C"
         ;;
-    c++|CC|g++|clang++|icpc|pgCC|xlc++)
+    c++|CC|g++|clang++|icpc|pgc++|xlc++)
         command="$SPACK_CXX"
         language="C++"
         ;;
-    f90|fc|f95|gfortran|ifort|pgf90|xlf90|nagfor)
+    f90|fc|f95|gfortran|ifort|pgfortran|xlf90|nagfor)
         command="$SPACK_FC"
         language="Fortran 90"
         ;;
-    f77|gfortran|ifort|pgf77|xlf|nagfor)
+    f77|gfortran|ifort|pgfortran|xlf|nagfor)
         command="$SPACK_F77"
         language="Fortran 77"
         ;;
@@ -113,14 +113,22 @@ case "$command" in
         ;;
 esac
 
-# Finish setting up the mode.
+# If any of the arguments below is present then the mode is vcheck. In vcheck mode nothing is added in terms of extra search paths or libraries
 if [ -z "$mode" ]; then
-    mode=ccld
     for arg in "$@"; do
         if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then
             mode=vcheck
             break
-        elif [ "$arg" = -E ]; then
+    fi
+    done
+fi
+
+# Finish setting up the mode.
+
+if [ -z "$mode" ]; then
+    mode=ccld
+    for arg in "$@"; do
+        if [ "$arg" = -E ]; then
             mode=cpp
             break
         elif [ "$arg" = -c ]; then
@@ -145,6 +153,10 @@ fi
 # Save original command for debug logging
 input_command="$@"
 
+if [ "$mode" == vcheck ] ; then
+    exec ${command} "$@"
+fi
+
 #
 # Now do real parsing of the command line args, trying hard to keep
 # non-rpath linker arguments in the proper order w.r.t. other command
@@ -175,32 +187,44 @@ while [ -n "$1" ]; do
             ;;
         -Wl,*)
             arg="${1#-Wl,}"
-            if [ -z "$arg" ]; then shift; arg="$1"; fi
+            # TODO: Handle multiple -Wl, continuations of -Wl,-rpath
-            if [[ "$arg" = -rpath=* ]]; then
+            if [[ $arg == -rpath=* ]]; then
-                rpaths+=("${arg#-rpath=}")
+                arg="${arg#-rpath=}"
-            elif [[ "$arg" = -rpath ]]; then
+                for rpath in ${arg//,/ }; do
+                    rpaths+=("$rpath")
+                done
+            elif [[ $arg == -rpath,* ]]; then
+                arg="${arg#-rpath,}"
+                for rpath in ${arg//,/ }; do
+                    rpaths+=("$rpath")
+                done
+            elif [[ $arg == -rpath ]]; then
                 shift; arg="$1"
-                if [[ "$arg" != -Wl,* ]]; then
+                if [[ $arg != '-Wl,'* ]]; then
                     die "-Wl,-rpath was not followed by -Wl,*"
                 fi
-                rpaths+=("${arg#-Wl,}")
+                arg="${arg#-Wl,}"
+                for rpath in ${arg//,/ }; do
+                    rpaths+=("$rpath")
+                done
             else
                 other_args+=("-Wl,$arg")
             fi
             ;;
-        -Xlinker,*)
+        -Xlinker)
-            arg="${1#-Xlinker,}"
+            shift; arg="$1";
-            if [ -z "$arg" ]; then shift; arg="$1"; fi
+            if [[ $arg = -rpath=* ]]; then
-            if [[ "$arg" = -rpath=* ]]; then
                 rpaths+=("${arg#-rpath=}")
-            elif [[ "$arg" = -rpath ]]; then
+            elif [[ $arg = -rpath ]]; then
                 shift; arg="$1"
-                if [[ "$arg" != -Xlinker,* ]]; then
+                if [[ $arg != -Xlinker ]]; then
-                    die "-Xlinker,-rpath was not followed by -Xlinker,*"
+                    die "-Xlinker -rpath was not followed by -Xlinker <arg>"
                 fi
-                rpaths+=("${arg#-Xlinker,}")
+                shift; arg="$1"
+                rpaths+=("$arg")
             else
-                other_args+=("-Xlinker,$arg")
+                other_args+=("-Xlinker")
+                other_args+=("$arg")
             fi
             ;;
         *)

lib/spack/env/pgi/case-insensitive/pgCC

@@ -1 +0,0 @@
-../../cc

lib/spack/llnl/util/filesystem.py

@@ -25,7 +25,9 @@
 __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree',
            'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp',
            'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file',
-           'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree']
+           'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink',
+           'set_executable', 'copy_mode', 'unset_executable_mode',
+           'remove_dead_links', 'remove_linked_tree']
 
 import os
 import sys
@@ -158,6 +160,14 @@ def copy_mode(src, dest):
     os.chmod(dest, dest_mode)
 
 
+def unset_executable_mode(path):
+    mode = os.stat(path).st_mode
+    mode &= ~stat.S_IXUSR
+    mode &= ~stat.S_IXGRP
+    mode &= ~stat.S_IXOTH
+    os.chmod(path, mode)
+
+
 def install(src, dest):
     """Manually install a file to a particular location."""
     tty.debug("Installing %s to %s" % (src, dest))
@@ -345,6 +355,12 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
     if order == 'post':
         yield (source_path, dest_path)
 
+
+def set_executable(path):
+    st = os.stat(path)
+    os.chmod(path, st.st_mode | stat.S_IEXEC)
+
+
 def remove_dead_links(root):
     """
     Removes any dead link that is present in root
@@ -362,8 +378,9 @@ def remove_dead_links(root):
 
 def remove_linked_tree(path):
     """
-    Removes a directory and its contents.  If the directory is a symlink, follows the link and removes the real
+    Removes a directory and its contents.  If the directory is a
-    directory before removing the link.
+    symlink, follows the link and removes the real directory before
+    removing the link.
 
     Args:
         path: directory to be removed

lib/spack/llnl/util/lang.py

@@ -235,11 +235,11 @@ def setter(name, value):
     if not has_method(cls, '_cmp_key'):
         raise TypeError("'%s' doesn't define _cmp_key()." % cls.__name__)
 
-    setter('__eq__', lambda s,o: o is not None and s._cmp_key() == o._cmp_key())
+    setter('__eq__', lambda s,o: (s is o) or (o is not None and s._cmp_key() == o._cmp_key()))
     setter('__lt__', lambda s,o: o is not None and s._cmp_key() <  o._cmp_key())
     setter('__le__', lambda s,o: o is not None and s._cmp_key() <= o._cmp_key())
 
-    setter('__ne__', lambda s,o: o is None or s._cmp_key() != o._cmp_key())
+    setter('__ne__', lambda s,o: (s is not o) and (o is None or s._cmp_key() != o._cmp_key()))
     setter('__gt__', lambda s,o: o is None or s._cmp_key() >  o._cmp_key())
     setter('__ge__', lambda s,o: o is None or s._cmp_key() >= o._cmp_key())
 

lib/spack/spack/__init__.py

@@ -81,6 +81,20 @@
 from spack.directory_layout import YamlDirectoryLayout
 install_layout = YamlDirectoryLayout(install_path)
 
+#
+# This controls how packages are sorted when trying to choose
+# the most preferred package.  More preferred packages are sorted
+# first.
+#
+from spack.preferred_packages import PreferredPackages
+pkgsort = PreferredPackages()
+
+#
+# This tests ABI compatibility between packages
+#
+from spack.abi import ABI
+abi = ABI()
+
 #
 # This controls how things are concretized in spack.
 # Replace it with a subclass if you want different
@@ -174,3 +188,10 @@
 import spack.util.executable
 from spack.util.executable import *
 __all__ += spack.util.executable.__all__
+
+from spack.package import \
+    install_dependency_symlinks, flatten_dependencies, DependencyConflictError, \
+    InstallError, ExternalPackageError
+__all__ += [
+    'install_dependency_symlinks', 'flatten_dependencies', 'DependencyConflictError',
+    'InstallError', 'ExternalPackageError']

lib/spack/spack/abi.py

@@ -0,0 +1,128 @@
+##############################################################################
+# Copyright (c) 2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import os
+import spack
+import spack.spec
+from spack.spec import CompilerSpec
+from spack.util.executable import Executable, ProcessError
+from llnl.util.lang import memoized
+
+class ABI(object):
+    """This class provides methods to test ABI compatibility between specs.
+       The current implementation is rather rough and could be improved."""
+
+    def architecture_compatible(self, parent, child):
+        """Returns true iff the parent and child specs have ABI compatible architectures."""
+        return not parent.architecture or not child.architecture or parent.architecture == child.architecture
+
+
+    @memoized
+    def _gcc_get_libstdcxx_version(self, version):
+        """Returns gcc ABI compatibility info by getting the library version of
+           a compiler's libstdc++.so or libgcc_s.so"""
+        spec = CompilerSpec("gcc", version)
+        compilers = spack.compilers.compilers_for_spec(spec)
+        if not compilers:
+            return None
+        compiler = compilers[0]
+        rungcc = None
+        libname = None
+        output = None
+        if compiler.cxx:
+            rungcc = Executable(compiler.cxx)
+            libname = "libstdc++.so"
+        elif compiler.cc:
+            rungcc = Executable(compiler.cc)
+            libname = "libgcc_s.so"
+        else:
+            return None
+        try:
+            output = rungcc("--print-file-name=%s" % libname, return_output=True)
+        except ProcessError, e:
+            return None
+        if not output:
+            return None
+        libpath = os.readlink(output.strip())
+        if not libpath:
+            return None
+        return os.path.basename(libpath)
+
+
+    @memoized
+    def _gcc_compiler_compare(self, pversion, cversion):
+        """Returns true iff the gcc version pversion and cversion
+          are ABI compatible."""
+        plib = self._gcc_get_libstdcxx_version(pversion)
+        clib = self._gcc_get_libstdcxx_version(cversion)
+        if not plib or not clib:
+            return False
+        return plib == clib
+
+
+    def _intel_compiler_compare(self, pversion, cversion):
+        """Returns true iff the intel version pversion and cversion
+           are ABI compatible"""
+
+        # Test major and minor versions.  Ignore build version.
+        if (len(pversion.version) < 2 or len(cversion.version) < 2):
+            return False
+        return pversion.version[:2] == cversion.version[:2]
+
+
+    def compiler_compatible(self, parent, child, **kwargs):
+        """Returns true iff the compilers for parent and child specs are ABI compatible"""
+        if not parent.compiler or not child.compiler:
+            return True
+
+        if parent.compiler.name != child.compiler.name:
+            # Different compiler families are assumed ABI incompatible
+            return False
+
+        if kwargs.get('loose', False):
+            return True
+
+        # TODO: Can we move the specialized ABI matching stuff
+        # TODO: into compiler classes?
+        for pversion in parent.compiler.versions:
+            for cversion in child.compiler.versions:
+                # For a few compilers use specialized comparisons.  Otherwise
+                # match on version match.
+                if pversion.satisfies(cversion):
+                    return True
+                elif (parent.compiler.name == "gcc" and
+                      self._gcc_compiler_compare(pversion, cversion)):
+                    return True
+                elif (parent.compiler.name == "intel" and
+                      self._intel_compiler_compare(pversion, cversion)):
+                    return True
+        return False
+
+
+    def compatible(self, parent, child, **kwargs):
+        """Returns true iff a parent and child spec are ABI compatible"""
+        loosematch = kwargs.get('loose', False)
+        return self.architecture_compatible(parent, child) and \
+               self.compiler_compatible(parent, child, loose=loosematch)

lib/spack/spack/build_environment.py

@@ -3,7 +3,7 @@
 build environment.  All of this is set up by package.py just before
 install() is called.
 
-There are two parts to the bulid environment:
+There are two parts to the build environment:
 
 1. Python build environment (i.e. install() method)
 
@@ -13,7 +13,7 @@
    the package's module scope.  Ths allows package writers to call
    them all directly in Package.install() without writing 'self.'
    everywhere.  No, this isn't Pythonic.  Yes, it makes the code more
-   readable and more like the shell script from whcih someone is
+   readable and more like the shell script from which someone is
    likely porting.
 
 2. Build execution environment
@@ -27,17 +27,18 @@
 Skimming this module is a nice way to get acquainted with the types of
 calls you can make from within the install() function.
 """
-import os
-import sys
-import shutil
 import multiprocessing
+import os
 import platform
-from llnl.util.filesystem import *
+import shutil
+import sys
 
 import spack
-import spack.compilers as compilers
+import llnl.util.tty as tty
-from spack.util.executable import Executable, which
+from llnl.util.filesystem import *
+from spack.environment import EnvironmentModifications, validate
 from spack.util.environment import *
+from spack.util.executable import Executable, which
 
 #
 # This can be set by the user to globally disable parallel builds.
@@ -83,85 +84,88 @@ def __call__(self, *args, **kwargs):
         return super(MakeExecutable, self).__call__(*args, **kwargs)
 
 
-def set_compiler_environment_variables(pkg):
+def set_compiler_environment_variables(pkg, env):
-    assert(pkg.spec.concrete)
+    assert pkg.spec.concrete
-    compiler = pkg.compiler
-
     # Set compiler variables used by CMake and autotools
-    assert all(key in pkg.compiler.link_paths
+    assert all(key in pkg.compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc'))
-               for key in ('cc', 'cxx', 'f77', 'fc'))
 
+    # Populate an object with the list of environment modifications
+    # and return it
+    # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc.
     link_dir = spack.build_env_path
-    os.environ['CC']  = join_path(link_dir, pkg.compiler.link_paths['cc'])
+    env.set('CC', join_path(link_dir, pkg.compiler.link_paths['cc']))
-    os.environ['CXX'] = join_path(link_dir, pkg.compiler.link_paths['cxx'])
+    env.set('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx']))
-    os.environ['F77'] = join_path(link_dir, pkg.compiler.link_paths['f77'])
+    env.set('F77', join_path(link_dir, pkg.compiler.link_paths['f77']))
-    os.environ['FC']  = join_path(link_dir, pkg.compiler.link_paths['fc'])
+    env.set('FC', join_path(link_dir, pkg.compiler.link_paths['fc']))
 
     # Set SPACK compiler variables so that our wrapper knows what to call
+    compiler = pkg.compiler
     if compiler.cc:
-        os.environ['SPACK_CC']  = compiler.cc
+        env.set('SPACK_CC', compiler.cc)
     if compiler.cxx:
-        os.environ['SPACK_CXX'] = compiler.cxx
+        env.set('SPACK_CXX', compiler.cxx)
     if compiler.f77:
-        os.environ['SPACK_F77'] = compiler.f77
+        env.set('SPACK_F77', compiler.f77)
     if compiler.fc:
-        os.environ['SPACK_FC']  = compiler.fc
+        env.set('SPACK_FC', compiler.fc)
 
-    os.environ['SPACK_COMPILER_SPEC']  = str(pkg.spec.compiler)
+    env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler))
+    return env
 
 
-def set_build_environment_variables(pkg):
+def set_build_environment_variables(pkg, env):
-    """This ensures a clean install environment when we build packages.
+    """
+    This ensures a clean install environment when we build packages
     """
     # Add spack build environment path with compiler wrappers first in
     # the path. We add both spack.env_path, which includes default
     # wrappers (cc, c++, f77, f90), AND a subdirectory containing
     # compiler-specific symlinks.  The latter ensures that builds that
     # are sensitive to the *name* of the compiler see the right name
-    # when we're building wtih the wrappers.
+    # when we're building with the wrappers.
     #
     # Conflicts on case-insensitive systems (like "CC" and "cc") are
     # handled by putting one in the <build_env_path>/case-insensitive
     # directory.  Add that to the path too.
     env_paths = []
-    def add_env_path(path):
+    for item in [spack.build_env_path, join_path(spack.build_env_path, pkg.compiler.name)]:
-        env_paths.append(path)
+        env_paths.append(item)
-        ci = join_path(path, 'case-insensitive')
+        ci = join_path(item, 'case-insensitive')
-        if os.path.isdir(ci): env_paths.append(ci)
+        if os.path.isdir(ci):
-    add_env_path(spack.build_env_path)
+            env_paths.append(ci)
-    add_env_path(join_path(spack.build_env_path, pkg.compiler.name))
 
-    path_put_first("PATH", env_paths)
+    for item in reversed(env_paths):
-    path_set(SPACK_ENV_PATH, env_paths)
+        env.prepend_path('PATH', item)
+    env.set_path(SPACK_ENV_PATH, env_paths)
 
-    # Prefixes of all of the package's dependencies go in
+    # Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES
-    # SPACK_DEPENDENCIES
     dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)]
-    path_set(SPACK_DEPENDENCIES, dep_prefixes)
+    env.set_path(SPACK_DEPENDENCIES, dep_prefixes)
+    env.set_path('CMAKE_PREFIX_PATH', dep_prefixes)  # Add dependencies to CMAKE_PREFIX_PATH
 
     # Install prefix
-    os.environ[SPACK_PREFIX] = pkg.prefix
+    env.set(SPACK_PREFIX, pkg.prefix)
 
     # Install root prefix
-    os.environ[SPACK_INSTALL] = spack.install_path
+    env.set(SPACK_INSTALL, spack.install_path)
 
     # Remove these vars from the environment during build because they
     # can affect how some packages find libraries.  We want to make
     # sure that builds never pull in unintended external dependencies.
-    pop_keys(os.environ, "LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH")
+    env.unset('LD_LIBRARY_PATH')
+    env.unset('LD_RUN_PATH')
+    env.unset('DYLD_LIBRARY_PATH')
 
     # Add bin directories from dependencies to the PATH for the build.
-    bin_dirs = ['%s/bin' % prefix for prefix in dep_prefixes]
+    bin_dirs = reversed(filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes]))
-    path_put_first('PATH', [bin for bin in bin_dirs if os.path.isdir(bin)])
+    for item in bin_dirs:
+        env.prepend_path('PATH', item)
 
     # Working directory for the spack command itself, for debug logs.
     if spack.debug:
-        os.environ[SPACK_DEBUG] = "TRUE"
+        env.set(SPACK_DEBUG, 'TRUE')
-    os.environ[SPACK_SHORT_SPEC] = pkg.spec.short_spec
+    env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec)
-    os.environ[SPACK_DEBUG_LOG_DIR] = spack.spack_working_dir
+    env.set(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir)
-
-    # Add dependencies to CMAKE_PREFIX_PATH
-    path_set("CMAKE_PREFIX_PATH", dep_prefixes)
 
     # Add any pkgconfig directories to PKG_CONFIG_PATH
     pkg_config_dirs = []
@@ -170,21 +174,23 @@ def add_env_path(path):
             pcdir = join_path(p, libdir, 'pkgconfig')
             if os.path.isdir(pcdir):
                 pkg_config_dirs.append(pcdir)
-    path_set("PKG_CONFIG_PATH", pkg_config_dirs)
+    env.set_path('PKG_CONFIG_PATH', pkg_config_dirs)
+
+    return env
 
 
-def set_module_variables_for_package(pkg, m):
+def set_module_variables_for_package(pkg, module):
     """Populate the module scope of install() with some useful functions.
        This makes things easier for package writers.
     """
-    m = pkg.module
-
     # number of jobs spack will to build with.
     jobs = multiprocessing.cpu_count()
     if not pkg.parallel:
         jobs = 1
     elif pkg.make_jobs:
         jobs = pkg.make_jobs
+
+    m = module
     m.make_jobs = jobs
 
     # TODO: make these build deps that can be installed if not found.
@@ -214,6 +220,13 @@ def set_module_variables_for_package(pkg, m):
     m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE')
     m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg)))
 
+    # Put spack compiler paths in module scope.
+    link_dir = spack.build_env_path
+    m.spack_cc  = join_path(link_dir, pkg.compiler.link_paths['cc'])
+    m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx'])
+    m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77'])
+    m.spack_fc  = join_path(link_dir, pkg.compiler.link_paths['fc'])
+
     # Emulate some shell commands for convenience
     m.pwd          = os.getcwd
     m.cd           = os.chdir
@@ -257,24 +270,63 @@ def parent_class_modules(cls):
     return result
 
 
-def setup_package(pkg):
+def setup_module_variables_for_dag(pkg):
-    """Execute all environment setup routines."""
+    """Set module-scope variables for all packages in the DAG."""
-    set_compiler_environment_variables(pkg)
+    for spec in pkg.spec.traverse(order='post'):
-    set_build_environment_variables(pkg)
-
         # If a user makes their own package repo, e.g.
         # spack.repos.mystuff.libelf.Libelf, and they inherit from
         # an existing class like spack.repos.original.libelf.Libelf,
         # then set the module variables for both classes so the
         # parent class can still use them if it gets called.
-    modules = parent_class_modules(pkg.__class__)
+        spkg = spec.package
+        modules = parent_class_modules(spkg.__class__)
         for mod in modules:
-        set_module_variables_for_package(pkg, mod)
+            set_module_variables_for_package(spkg, mod)
+        set_module_variables_for_package(spkg, spkg.module)
 
-    # Allow dependencies to set up environment as well.
+
-    for dep_spec in pkg.spec.traverse(root=False):
+def setup_package(pkg):
-        dep_spec.package.setup_dependent_environment(
+    """Execute all environment setup routines."""
-            pkg.module, dep_spec, pkg.spec)
+    spack_env = EnvironmentModifications()
+    run_env   = EnvironmentModifications()
+
+    # Before proceeding, ensure that specs and packages are consistent
+    #
+    # This is a confusing behavior due to how packages are
+    # constructed.  `setup_dependent_package` may set attributes on
+    # specs in the DAG for use by other packages' install
+    # method. However, spec.package will look up a package via
+    # spack.repo, which defensively copies specs into packages.  This
+    # code ensures that all packages in the DAG have pieces of the
+    # same spec object at build time.
+    #
+    # This is safe for the build process, b/c the build process is a
+    # throwaway environment, but it is kind of dirty.
+    #
+    # TODO: Think about how to avoid this fix and do something cleaner.
+    for s in pkg.spec.traverse(): s.package.spec = s
+
+    set_compiler_environment_variables(pkg, spack_env)
+    set_build_environment_variables(pkg, spack_env)
+    setup_module_variables_for_dag(pkg)
+
+    # Allow dependencies to modify the module
+    spec = pkg.spec
+    for dependency_spec in spec.traverse(root=False):
+        dpkg = dependency_spec.package
+        dpkg.setup_dependent_package(pkg.module, spec)
+
+    # Allow dependencies to set up environment as well
+    for dependency_spec in spec.traverse(root=False):
+        dpkg = dependency_spec.package
+        dpkg.setup_dependent_environment(spack_env, run_env, spec)
+
+    # Allow the package to apply some settings.
+    pkg.setup_environment(spack_env, run_env)
+
+    # Make sure nothing's strange about the Spack environment.
+    validate(spack_env, tty.warn)
+    spack_env.apply_modifications()
 
 
 def fork(pkg, function):
@@ -291,9 +343,9 @@ def child_fun():
            # do stuff
        build_env.fork(pkg, child_fun)
 
-    Forked processes are run with the build environemnt set up by
+    Forked processes are run with the build environment set up by
     spack.build_environment.  This allows package authors to have
-    full control over the environment, etc. without offecting
+    full control over the environment, etc. without affecting
     other builds that might be executed in the same spack call.
 
     If something goes wrong, the child process is expected to print
@@ -303,11 +355,11 @@ def child_fun():
     """
     try:
         pid = os.fork()
-    except OSError, e:
+    except OSError as e:
         raise InstallError("Unable to fork build process: %s" % e)
 
     if pid == 0:
-        # Give the child process the package's build environemnt.
+        # Give the child process the package's build environment.
         setup_package(pkg)
 
         try:
@@ -318,7 +370,7 @@ def child_fun():
             # which interferes with unit tests.
             os._exit(0)
 
-        except spack.error.SpackError, e:
+        except spack.error.SpackError as e:
             e.die()
 
         except:
@@ -333,8 +385,7 @@ def child_fun():
         # message.  Just make the parent exit with an error code.
         pid, returncode = os.waitpid(pid, 0)
         if returncode != 0:
-            raise InstallError("Installation process had nonzero exit code."
+            raise InstallError("Installation process had nonzero exit code.".format(str(returncode)))
-                .format(str(returncode)))
 
 
 class InstallError(spack.error.SpackError):

lib/spack/spack/cmd/checksum.py

@@ -22,16 +22,10 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import os
-import re
 import argparse
 import hashlib
-from pprint import pprint
-from subprocess import CalledProcessError
 
 import llnl.util.tty as tty
-from llnl.util.tty.colify import colify
-
 import spack
 import spack.cmd
 import spack.util.crypto
@@ -40,6 +34,7 @@
 
 description = "Checksum available versions of a package."
 
+
 def setup_parser(subparser):
     subparser.add_argument(
         'package', metavar='PACKAGE', help='Package to list versions for')
@@ -60,30 +55,23 @@ def get_checksums(versions, urls, **kwargs):
     hashes = []
     i = 0
     for url, version in zip(urls, versions):
-        stage = Stage(url)
         try:
+            with Stage(url, keep=keep_stage) as stage:
                 stage.fetch()
                 if i == 0 and first_stage_function:
                     first_stage_function(stage)
 
                 hashes.append((version,
                                spack.util.crypto.checksum(hashlib.md5, stage.archive_file)))
-        except FailedDownloadError, e:
-            tty.msg("Failed to fetch %s" % url)
-            continue
-	except Exception, e:
-            tty.msg('Something failed on %s, skipping.\n    (%s)' % (url, e))
-            continue
-
-        finally:
-            if not keep_stage:
-                stage.destroy()
                 i += 1
+        except FailedDownloadError as e:
+            tty.msg("Failed to fetch %s" % url)
+        except Exception as e:
+            tty.msg('Something failed on %s, skipping.\n    (%s)' % (url, e))
 
     return hashes
 
 
-
 def checksum(parser, args):
     # get the package we're going to generate checksums for
     pkg = spack.repo.get(args.package)
@@ -100,11 +88,11 @@ def checksum(parser, args):
     else:
         versions = pkg.fetch_remote_versions()
         if not versions:
-            tty.die("Could not fetch any versions for %s." % pkg.name)
+            tty.die("Could not fetch any versions for %s" % pkg.name)
 
     sorted_versions = sorted(versions, reverse=True)
 
-    tty.msg("Found %s versions of %s." % (len(versions), pkg.name),
+    tty.msg("Found %s versions of %s" % (len(versions), pkg.name),
             *spack.cmd.elide_list(
                 ["%-10s%s" % (v, versions[v]) for v in sorted_versions]))
     print
@@ -121,7 +109,7 @@ def checksum(parser, args):
         keep_stage=args.keep_stage)
 
     if not version_hashes:
-        tty.die("Could not fetch any versions for %s." % pkg.name)
+        tty.die("Could not fetch any versions for %s" % pkg.name)
 
     version_lines = ["    version('%s', '%s')" % (v, h) for v, h in version_hashes]
     tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines)

lib/spack/spack/cmd/clean.py

@@ -43,4 +43,4 @@ def clean(parser, args):
     specs = spack.cmd.parse_specs(args.packages, concretize=True)
     for spec in specs:
         package = spack.repo.get(spec)
-        package.stage.destroy()
+        package.do_clean()

lib/spack/spack/cmd/compiler.py

@@ -96,7 +96,7 @@ def compiler_remove(args):
     compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope)
 
     if not compilers:
-        tty.die("No compilers match spec %s." % cspec)
+        tty.die("No compilers match spec %s" % cspec)
     elif not args.all and len(compilers) > 1:
         tty.error("Multiple compilers match spec %s. Choose one:" % cspec)
         colify(reversed(sorted([c.spec for c in compilers])), indent=4)
@@ -105,7 +105,7 @@ def compiler_remove(args):
 
     for compiler in compilers:
         spack.compilers.remove_compiler_from_config(compiler.spec, scope=args.scope)
-        tty.msg("Removed compiler %s." % compiler.spec)
+        tty.msg("Removed compiler %s" % compiler.spec)
 
 
 def compiler_info(args):
@@ -114,7 +114,7 @@ def compiler_info(args):
     compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope)
 
     if not compilers:
-        tty.error("No compilers match spec %s." % cspec)
+        tty.error("No compilers match spec %s" % cspec)
     else:
         for c in compilers:
             print str(c.spec) + ":"

lib/spack/spack/cmd/create.py

@@ -156,7 +156,7 @@ def guess_name_and_version(url, args):
     # Try to deduce name and version of the new package from the URL
     version = spack.url.parse_version(url)
     if not version:
-        tty.die("Couldn't guess a version string from %s." % url)
+        tty.die("Couldn't guess a version string from %s" % url)
 
     # Try to guess a name.  If it doesn't work, allow the user to override.
     if args.alternate_name:
@@ -189,7 +189,7 @@ def find_repository(spec, args):
         try:
             repo = Repo(repo_path)
             if spec.namespace and spec.namespace != repo.namespace:
-                tty.die("Can't create package with namespace %s in repo with namespace %s."
+                tty.die("Can't create package with namespace %s in repo with namespace %s"
                         % (spec.namespace, repo.namespace))
         except RepoError as e:
             tty.die(str(e))
@@ -208,7 +208,7 @@ def find_repository(spec, args):
     return repo
 
 
-def fetch_tarballs(url, name, args):
+def fetch_tarballs(url, name, version):
     """Try to find versions of the supplied archive by scraping the web.
 
     Prompts the user to select how many to download if many are found.
@@ -222,7 +222,7 @@ def fetch_tarballs(url, name, args):
     archives_to_fetch = 1
     if not versions:
         # If the fetch failed for some reason, revert to what the user provided
-        versions = { "version" : url }
+        versions = { version : url }
     elif len(versions) > 1:
         tty.msg("Found %s versions of %s:" % (len(versions), name),
                 *spack.cmd.elide_list(
@@ -252,11 +252,11 @@ def create(parser, args):
     name = spec.name  # factors out namespace, if any
     repo = find_repository(spec, args)
 
-    tty.msg("This looks like a URL for %s version %s." % (name, version))
+    tty.msg("This looks like a URL for %s version %s" % (name, version))
     tty.msg("Creating template for package %s" % name)
 
     # Fetch tarballs (prompting user if necessary)
-    versions, urls = fetch_tarballs(url, name, args)
+    versions, urls = fetch_tarballs(url, name, version)
 
     # Try to guess what configure system is used.
     guesser = ConfigureGuesser()
@@ -266,7 +266,7 @@ def create(parser, args):
         keep_stage=args.keep_stage)
 
     if not ver_hash_tuples:
-        tty.die("Could not fetch any tarballs for %s." % name)
+        tty.die("Could not fetch any tarballs for %s" % name)
 
     # Prepend 'py-' to python package names, by convention.
     if guesser.build_system == 'python':
@@ -291,4 +291,4 @@ def create(parser, args):
 
     # If everything checks out, go ahead and edit.
     spack.editor(pkg_path)
-    tty.msg("Created package %s." % pkg_path)
+    tty.msg("Created package %s" % pkg_path)

lib/spack/spack/cmd/diy.py

@@ -45,6 +45,9 @@ def setup_parser(subparser):
     subparser.add_argument(
         '--skip-patch', action='store_true',
         help="Skip patching for the DIY build.")
+    subparser.add_argument(
+        '-q', '--quiet', action='store_true', dest='quiet',
+        help="Do not display verbose build output while installing.")
     subparser.add_argument(
         'spec', nargs=argparse.REMAINDER,
         help="specs to use for install.  Must contain package AND verison.")
@@ -72,8 +75,8 @@ def diy(self, args):
                 edit_package(spec.name, spack.repo.first_repo(), None, True)
                 return
 
-        if not spec.version.concrete:
+        if not spec.versions.concrete:
-            tty.die("spack diy spec must have a single, concrete version.")
+            tty.die("spack diy spec must have a single, concrete version.  Did you forget a package version number?")
 
         spec.concretize()
         package = spack.repo.get(spec)
@@ -92,4 +95,5 @@ def diy(self, args):
         package.do_install(
             keep_prefix=args.keep_prefix,
             ignore_deps=args.ignore_deps,
+            verbose=not args.quiet,
             keep_stage=True)   # don't remove source dir for DIY.

lib/spack/spack/cmd/md5.py

@@ -22,51 +22,51 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import os
 import argparse
 import hashlib
-
+import os
-from contextlib import contextmanager
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import *
-
 import spack.util.crypto
 from spack.stage import Stage, FailedDownloadError
 
 description = "Calculate md5 checksums for files/urls."
 
-@contextmanager
-def stager(url):
-    _cwd = os.getcwd()
-    _stager = Stage(url)
-    try:
-        _stager.fetch()
-        yield _stager
-    except FailedDownloadError:
-        tty.msg("Failed to fetch %s" % url)
-    finally:
-        _stager.destroy()
-        os.chdir(_cwd) # the Stage class changes the current working dir so it has to be restored
 
 def setup_parser(subparser):
     setup_parser.parser = subparser
     subparser.add_argument('files', nargs=argparse.REMAINDER,
                            help="Files to checksum.")
 
+
+def compute_md5_checksum(url):
+    if not os.path.isfile(url):
+        with Stage(url) as stage:
+            stage.fetch()
+            value = spack.util.crypto.checksum(hashlib.md5, stage.archive_file)
+    else:
+        value = spack.util.crypto.checksum(hashlib.md5, url)
+    return value
+
+
 def md5(parser, args):
     if not args.files:
         setup_parser.parser.print_help()
         return 1
 
-    for f in args.files:
+    results = []
-        if not os.path.isfile(f):
+    for url in args.files:
-            with stager(f) as stage:
+        try:
-                checksum = spack.util.crypto.checksum(hashlib.md5, stage.archive_file)
+            checksum = compute_md5_checksum(url)
-                print "%s  %s" % (checksum, f)
+            results.append((checksum, url))
-        else:
+        except FailedDownloadError as e:
-            if not can_access(f):
+            tty.warn("Failed to fetch %s" % url)
-                tty.die("Cannot read file: %s" % f)
+            tty.warn("%s" % e)
+        except IOError as e:
+            tty.warn("Error when reading %s" % url)
+            tty.warn("%s" % e)
 
-            checksum = spack.util.crypto.checksum(hashlib.md5, f)
+    # Dump the MD5s at last without interleaving them with downloads
-            print "%s  %s" % (checksum, f)
+    tty.msg("%d MD5 checksums:" % len(results))
+    for checksum, url in results:
+        print "%s  %s" % (checksum, url)

lib/spack/spack/cmd/mirror.py

@@ -126,7 +126,7 @@ def mirror_remove(args):
 
     old_value = mirrors.pop(name)
     spack.config.update_config('mirrors', mirrors, scope=args.scope)
-    tty.msg("Removed mirror %s with url %s." % (name, old_value))
+    tty.msg("Removed mirror %s with url %s" % (name, old_value))
 
 
 def mirror_list(args):
@@ -203,7 +203,7 @@ def mirror_create(args):
 
     verb = "updated" if existed else "created"
     tty.msg(
-        "Successfully %s mirror in %s." % (verb, directory),
+        "Successfully %s mirror in %s" % (verb, directory),
         "Archive stats:",
         "  %-4d already present"  % p,
         "  %-4d added"            % m,

lib/spack/spack/cmd/module.py

@@ -58,7 +58,7 @@ def module_find(mtype, spec_array):
        should type to use that package's module.
     """
     if mtype not in module_types:
-        tty.die("Invalid module type: '%s'.  Options are %s." % (mtype, comma_or(module_types)))
+        tty.die("Invalid module type: '%s'.  Options are %s" % (mtype, comma_or(module_types)))
 
     specs = spack.cmd.parse_specs(spec_array)
     if len(specs) > 1:
@@ -78,9 +78,9 @@ def module_find(mtype, spec_array):
     mt = module_types[mtype]
     mod = mt(specs[0])
     if not os.path.isfile(mod.file_name):
-        tty.die("No %s module is installed for %s." % (mtype, spec))
+        tty.die("No %s module is installed for %s" % (mtype, spec))
 
-    print mod.use_name
+    print(mod.use_name)
 
 
 def module_refresh():
@@ -94,7 +94,7 @@ def module_refresh():
             shutil.rmtree(cls.path, ignore_errors=False)
         mkdirp(cls.path)
         for spec in specs:
-            tty.debug("   Writing file for %s." % spec)
+            tty.debug("   Writing file for %s" % spec)
             cls(spec).write()
 
 

lib/spack/spack/cmd/patch.py

@@ -24,6 +24,7 @@
 ##############################################################################
 import argparse
 
+import llnl.util.tty as tty
 import spack.cmd
 import spack
 

lib/spack/spack/cmd/repo.py

@@ -89,11 +89,11 @@ def repo_add(args):
 
     # check if the path exists
     if not os.path.exists(canon_path):
-        tty.die("No such file or directory: '%s'." % path)
+        tty.die("No such file or directory: %s" % path)
 
     # Make sure the path is a directory.
     if not os.path.isdir(canon_path):
-        tty.die("Not a Spack repository: '%s'." % path)
+        tty.die("Not a Spack repository: %s" % path)
 
     # Make sure it's actually a spack repository by constructing it.
     repo = Repo(canon_path)
@@ -103,7 +103,7 @@ def repo_add(args):
     if not repos: repos = []
 
     if repo.root in repos or path in repos:
-        tty.die("Repository is already registered with Spack: '%s'" % path)
+        tty.die("Repository is already registered with Spack: %s" % path)
 
     repos.insert(0, canon_path)
     spack.config.update_config('repos', repos, args.scope)
@@ -122,7 +122,7 @@ def repo_remove(args):
         if canon_path == repo_canon_path:
             repos.remove(repo_path)
             spack.config.update_config('repos', repos, args.scope)
-            tty.msg("Removed repository '%s'." % repo_path)
+            tty.msg("Removed repository %s" % repo_path)
             return
 
     # If it is a namespace, remove corresponding repo
@@ -132,13 +132,13 @@ def repo_remove(args):
             if repo.namespace == path_or_namespace:
                 repos.remove(path)
                 spack.config.update_config('repos', repos, args.scope)
-                tty.msg("Removed repository '%s' with namespace %s."
+                tty.msg("Removed repository %s with namespace '%s'."
                         % (repo.root, repo.namespace))
                 return
         except RepoError as e:
             continue
 
-    tty.die("No repository with path or namespace: '%s'"
+    tty.die("No repository with path or namespace: %s"
             % path_or_namespace)
 
 

lib/spack/spack/cmd/uninstall.py

@@ -22,6 +22,7 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+from __future__ import print_function
 import sys
 import argparse
 
@@ -63,12 +64,12 @@ def uninstall(parser, args):
             matching_specs = spack.installed_db.query(spec)
             if not args.all and len(matching_specs) > 1:
                 tty.error("%s matches multiple packages:" % spec)
-                print
+                print()
                 display_specs(matching_specs, long=True)
-                print
+                print()
-                print "You can either:"
+                print("You can either:")
-                print "  a) Use a more specific spec, or"
+                print("  a) Use a more specific spec, or")
-                print "  b) use spack uninstall -a to uninstall ALL matching specs."
+                print("  b) use spack uninstall -a to uninstall ALL matching specs.")
                 sys.exit(1)
 
             if len(matching_specs) == 0:
@@ -79,7 +80,7 @@ def uninstall(parser, args):
                 try:
                     # should work if package is known to spack
                     pkgs.append(s.package)
-                except spack.repository.UnknownPackageError, e:
+                except spack.repository.UnknownPackageError as e:
                     # The package.py file has gone away -- but still
                     # want to uninstall.
                     spack.Package(s).do_uninstall(force=True)
@@ -94,11 +95,11 @@ def num_installed_deps(pkg):
         for pkg in pkgs:
             try:
                 pkg.do_uninstall(force=args.force)
-            except PackageStillNeededError, e:
+            except PackageStillNeededError as e:
                 tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True))
-                print
+                print('')
-                print "The following packages depend on it:"
+                print("The following packages depend on it:")
                 display_specs(e.dependents, long=True)
-                print
+                print('')
-                print "You can use spack uninstall -f to force this action."
+                print("You can use spack uninstall -f to force this action.")
                 sys.exit(1)

lib/spack/spack/compiler.py

@@ -256,12 +256,12 @@ def find(cls, *path):
 
 
     def __repr__(self):
-        """Return a string represntation of the compiler toolchain."""
+        """Return a string representation of the compiler toolchain."""
         return self.__str__()
 
 
     def __str__(self):
-        """Return a string represntation of the compiler toolchain."""
+        """Return a string representation of the compiler toolchain."""
         return "%s(%s)" % (
             self.name, '\n     '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc))))
 

lib/spack/spack/compilers/gcc.py

@@ -40,7 +40,8 @@ class Gcc(Compiler):
     fc_names = ['gfortran']
 
     # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
-    suffixes = [r'-mp-\d\.\d']
+    # Homebrew and Linuxes may build gcc with -X, -X.Y suffixes
+    suffixes = [r'-mp-\d\.\d', r'-\d\.\d', r'-\d']
 
     # Named wrapper links within spack.build_env_path
     link_paths = {'cc'  : 'gcc/gcc',

lib/spack/spack/compilers/pgi.py

@@ -29,28 +29,28 @@ class Pgi(Compiler):
     cc_names = ['pgcc']
 
     # Subclasses use possible names of C++ compiler
-    cxx_names = ['pgCC']
+    cxx_names = ['pgc++', 'pgCC']
 
     # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ['pgf77']
+    f77_names = ['pgfortran', 'pgf77']
 
     # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ['pgf95', 'pgf90']
+    fc_names = ['pgfortran', 'pgf95', 'pgf90']
 
     # Named wrapper links within spack.build_env_path
     link_paths = { 'cc'  : 'pgi/pgcc',
-                   'cxx' : 'pgi/case-insensitive/pgCC',
+                   'cxx' : 'pgi/pgc++',
-                   'f77' : 'pgi/pgf77',
+                   'f77' : 'pgi/pgfortran',
-                   'fc'  : 'pgi/pgf90' }
+                   'fc'  : 'pgi/pgfortran' }
 
     @classmethod
     def default_version(cls, comp):
         """The '-V' option works for all the PGI compilers.
            Output looks like this::
 
-               pgf95 10.2-0 64-bit target on x86-64 Linux -tp nehalem-64
+               pgcc 15.10-0 64-bit target on x86-64 Linux -tp sandybridge
-               Copyright 1989-2000, The Portland Group, Inc.  All Rights Reserved.
+               The Portland Group - PGI Compilers and Tools
-               Copyright 2000-2010, STMicroelectronics, Inc.  All Rights Reserved.
+               Copyright (c) 2015, NVIDIA CORPORATION.  All rights reserved.
         """
         return get_compiler_version(
             comp, '-V', r'pg[^ ]* ([^ ]+) \d\d\d?-bit target')

lib/spack/spack/concretize.py

@@ -33,12 +33,16 @@
 TODO: make this customizable and allow users to configure
       concretization  policies.
 """
+import spack
 import spack.spec
 import spack.compilers
 import spack.architecture
 import spack.error
 from spack.version import *
-
+from functools import partial
+from spec import DependencyMap
+from itertools import chain
+from spack.config import *
 
 class DefaultConcretizer(object):
     """This class doesn't have any state, it just provides some methods for
@@ -46,10 +50,92 @@ class DefaultConcretizer(object):
        default concretization strategies, or you can override all of them.
     """
 
+    def _valid_virtuals_and_externals(self, spec):
+        """Returns a list of candidate virtual dep providers and external
+           packages that coiuld be used to concretize a spec."""
+        # First construct a list of concrete candidates to replace spec with.
+        candidates = [spec]
+        if spec.virtual:
+            providers = spack.repo.providers_for(spec)
+            if not providers:
+                raise UnsatisfiableProviderSpecError(providers[0], spec)
+            spec_w_preferred_providers = find_spec(
+                spec, lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name))
+            if not spec_w_preferred_providers:
+                spec_w_preferred_providers = spec
+            provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name)
+            candidates = sorted(providers, cmp=provider_cmp)
+
+        # For each candidate package, if it has externals, add those to the usable list.
+        # if it's not buildable, then *only* add the externals.
+        usable = []
+        for cspec in candidates:
+            if is_spec_buildable(cspec):
+                usable.append(cspec)
+            externals = spec_externals(cspec)
+            for ext in externals:
+                if ext.satisfies(spec):
+                    usable.append(ext)
+
+        # If nothing is in the usable list now, it's because we aren't
+        # allowed to build anything.
+        if not usable:
+            raise NoBuildError(spec)
+
+        def cmp_externals(a, b):
+            if a.name != b.name:
+                # We're choosing between different providers, so
+                # maintain order from provider sort
+                return candidates.index(a) - candidates.index(b)
+
+            result = cmp_specs(a, b)
+            if result != 0:
+                return result
+
+            # prefer external packages to internal packages.
+            if a.external is None or b.external is None:
+                return -cmp(a.external, b.external)
+            else:
+                return cmp(a.external, b.external)
+
+        usable.sort(cmp=cmp_externals)
+        return usable
+
+
+    def choose_virtual_or_external(self, spec):
+        """Given a list of candidate virtual and external packages, try to
+           find one that is most ABI compatible.
+        """
+        candidates = self._valid_virtuals_and_externals(spec)
+        if not candidates:
+            return candidates
+
+        # Find the nearest spec in the dag that has a compiler.  We'll
+        # use that spec to calibrate compiler compatibility.
+        abi_exemplar = find_spec(spec, lambda(x): x.compiler)
+        if not abi_exemplar:
+            abi_exemplar = spec.root
+
+        # Make a list including ABI compatibility of specs with the exemplar.
+        strict = [spack.abi.compatible(c, abi_exemplar) for c in candidates]
+        loose  = [spack.abi.compatible(c, abi_exemplar, loose=True) for c in candidates]
+        keys = zip(strict, loose, candidates)
+
+        # Sort candidates from most to least compatibility.
+        # Note:
+        #   1. We reverse because True > False.
+        #   2. Sort is stable, so c's keep their order.
+        keys.sort(key=lambda k:k[:2], reverse=True)
+
+        # Pull the candidates back out and return them in order
+        candidates = [c for s,l,c in keys]
+        return candidates
+
+
     def concretize_version(self, spec):
         """If the spec is already concrete, return.  Otherwise take
-           the most recent available version, and default to the package's
+           the preferred version from spackconfig, and default to the package's
-           version if there are no avaialble versions.
+           version if there are no available versions.
 
            TODO: In many cases we probably want to look for installed
                  versions of each package and use an installed version
@@ -67,20 +153,18 @@ def concretize_version(self, spec):
         # If there are known available versions, return the most recent
         # version that satisfies the spec
         pkg = spec.package
-
+        cmp_versions = partial(spack.pkgsort.version_compare, spec.name)
-        # Key function to sort versions first by whether they were
-        # marked `preferred=True`, then by most recent.
-        def preferred_key(v):
-            prefer = pkg.versions[v].get('preferred', False)
-            return (prefer, v)
-
         valid_versions = sorted(
             [v for v in pkg.versions
              if any(v.satisfies(sv) for sv in spec.versions)],
-            key=preferred_key)
+            cmp=cmp_versions)
+
+        def prefer_key(v):
+            return pkg.versions.get(Version(v)).get('preferred', False)
+        valid_versions.sort(key=prefer_key, reverse=True)
 
         if valid_versions:
-            spec.versions = ver([valid_versions[-1]])
+            spec.versions = ver([valid_versions[0]])
         else:
             # We don't know of any SAFE versions that match the given
             # spec.  Grab the spec's versions and grab the highest
@@ -134,7 +218,7 @@ def concretize_variants(self, spec):
            the default variants from the package specification.
         """
         changed = False
-        for name, variant in spec.package.variants.items():
+        for name, variant in spec.package_class.variants.items():
             if name not in spec.variants:
                 spec.variants[name] = spack.spec.VariantSpec(name, variant.default)
                 changed = True
@@ -145,10 +229,10 @@ def concretize_compiler(self, spec):
         """If the spec already has a compiler, we're done.  If not, then take
            the compiler used for the nearest ancestor with a compiler
            spec and use that.  If the ancestor's compiler is not
-           concrete, then give it a valid version.  If there is no
+           concrete, then used the preferred compiler as specified in
-           ancestor with a compiler, use the system default compiler.
+           spackconfig.
 
-           Intuition: Use the system default if no package that depends on
+           Intuition: Use the spackconfig default if no package that depends on
            this one has a strict compiler requirement.  Otherwise, try to
            build with the compiler that will be used by libraries that
            link to this one, to maximize compatibility.
@@ -160,40 +244,91 @@ def concretize_compiler(self, spec):
             spec.compiler in all_compilers):
             return False
 
-        try:
+        #Find the another spec that has a compiler, or the root if none do
-            nearest = next(p for p in spec.traverse(direction='parents')
+        other_spec = spec if spec.compiler else find_spec(spec, lambda(x) : x.compiler)
-                           if p.compiler is not None).compiler
+        if not other_spec:
+            other_spec = spec.root
+        other_compiler = other_spec.compiler
+        assert(other_spec)
 
-            if not nearest in all_compilers:
+        # Check if the compiler is already fully specified
-                # Take the newest compiler that saisfies the spec
+        if other_compiler in all_compilers:
-                matches = sorted(spack.compilers.find(nearest))
+            spec.compiler = other_compiler.copy()
+            return True
+
+        # Filter the compilers into a sorted list based on the compiler_order from spackconfig
+        compiler_list = all_compilers if not other_compiler else spack.compilers.find(other_compiler)
+        cmp_compilers = partial(spack.pkgsort.compiler_compare, other_spec.name)
+        matches = sorted(compiler_list, cmp=cmp_compilers)
         if not matches:
-                    raise UnavailableCompilerVersionError(nearest)
+            raise UnavailableCompilerVersionError(other_compiler)
-
-                # copy concrete version into nearest spec
-                nearest.versions = matches[-1].versions.copy()
-                assert(nearest.concrete)
-
-            spec.compiler = nearest.copy()
-
-        except StopIteration:
-            spec.compiler = spack.compilers.default_compiler().copy()
 
+        # copy concrete version into other_compiler
+        spec.compiler = matches[0].copy()
+        assert(spec.compiler.concrete)
         return True  # things changed.
 
 
-    def choose_provider(self, spec, providers):
+def find_spec(spec, condition):
-        """This is invoked for virtual specs.  Given a spec with a virtual name,
+    """Searches the dag from spec in an intelligent order and looks
-           say "mpi", and a list of specs of possible providers of that spec,
+       for a spec that matches a condition"""
-           select a provider and return it.
+    # First search parents, then search children
-        """
+    dagiter = chain(spec.traverse(direction='parents',  root=False),
-        assert(spec.virtual)
+                    spec.traverse(direction='children', root=False))
-        assert(providers)
+    visited = set()
+    for relative in dagiter:
+        if condition(relative):
+            return relative
+        visited.add(id(relative))
+
+    # Then search all other relatives in the DAG *except* spec
+    for relative in spec.root.traverse():
+        if relative is spec: continue
+        if id(relative) in visited: continue
+        if condition(relative):
+            return relative
+
+    # Finally search spec itself.
+    if condition(spec):
+        return spec
+
+    return None   # Nothing matched the condition.
+
+
+def cmp_specs(lhs, rhs):
+    # Package name sort order is not configurable, always goes alphabetical
+    if lhs.name != rhs.name:
+        return cmp(lhs.name, rhs.name)
+
+    # Package version is second in compare order
+    pkgname = lhs.name
+    if lhs.versions != rhs.versions:
+        return spack.pkgsort.version_compare(
+            pkgname, lhs.versions, rhs.versions)
+
+    # Compiler is third
+    if lhs.compiler != rhs.compiler:
+        return spack.pkgsort.compiler_compare(
+            pkgname, lhs.compiler, rhs.compiler)
+
+    # Variants
+    if lhs.variants != rhs.variants:
+        return spack.pkgsort.variant_compare(
+            pkgname, lhs.variants, rhs.variants)
+
+    # Architecture
+    if lhs.architecture != rhs.architecture:
+        return spack.pkgsort.architecture_compare(
+            pkgname, lhs.architecture, rhs.architecture)
+
+    # Dependency is not configurable
+    lhash, rhash = hash(lhs), hash(rhs)
+    if lhash != rhash:
+        return -1 if lhash < rhash else 1
+
+    # Equal specs
+    return 0
 
-        index = spack.spec.index_specs(providers)
-        first_key = sorted(index.keys())[0]
-        latest_version = sorted(index[first_key])[-1]
-        return latest_version
 
 
 class UnavailableCompilerVersionError(spack.error.SpackError):
@@ -211,3 +346,11 @@ class NoValidVersionError(spack.error.SpackError):
     def __init__(self, spec):
         super(NoValidVersionError, self).__init__(
             "There are no valid versions for %s that match '%s'" % (spec.name, spec.versions))
+
+
+class NoBuildError(spack.error.SpackError):
+    """Raised when a package is configured with the buildable option False, but
+       no satisfactory external versions can be found"""
+    def __init__(self, spec):
+        super(NoBuildError, self).__init__(
+            "The spec '%s' is configured as not buildable, and no matching external installs were found" % spec.name)

lib/spack/spack/config.py

@@ -129,6 +129,7 @@
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import mkdirp
+import copy
 
 import spack
 from spack.error import SpackError
@@ -194,6 +195,49 @@
                 'default': [],
                 'items': {
                     'type': 'string'},},},},
+    'packages': {
+        '$schema': 'http://json-schema.org/schema#',
+        'title': 'Spack package configuration file schema',
+        'type': 'object',
+        'additionalProperties': False,
+        'patternProperties': {
+            r'packages:?': {
+                'type': 'object',
+                'default': {},
+                'additionalProperties': False,
+                'patternProperties': {
+                    r'\w[\w-]*': { # package name
+                        'type': 'object',
+                        'default': {},
+                        'additionalProperties': False,
+                        'properties': {
+                            'version': {
+                                'type' : 'array',
+                                'default' : [],
+                                'items' : { 'anyOf' : [ { 'type' : 'string' },
+                                                        { 'type' : 'number'}]}}, #version strings
+                            'compiler': {
+                                'type' : 'array',
+                                'default' : [],
+                                'items' : { 'type' : 'string' } }, #compiler specs
+                            'buildable': {
+                                'type':  'boolean',
+                                'default': True,
+                             },
+                            'providers': {
+                                'type':  'object',
+                                'default': {},
+                                'additionalProperties': False,
+                                'patternProperties': {
+                                    r'\w[\w-]*': {
+                                        'type' : 'array',
+                                        'default' : [],
+                                        'items' : { 'type' : 'string' },},},},
+                            'paths': {
+                                'type' : 'object',
+                                'default' : {},
+                            }
+                        },},},},},}
 }
 
 """OrderedDict of config scopes keyed by name.
@@ -205,7 +249,7 @@
 def validate_section_name(section):
     """Raise a ValueError if the section is not a valid section."""
     if section not in section_schemas:
-        raise ValueError("Invalid config section: '%s'.  Options are %s."
+        raise ValueError("Invalid config section: '%s'.  Options are %s"
                          % (section, section_schemas))
 
 
@@ -335,7 +379,7 @@ def validate_scope(scope):
         return config_scopes[scope]
 
     else:
-        raise ValueError("Invalid config scope: '%s'.  Must be one of %s."
+        raise ValueError("Invalid config scope: '%s'.  Must be one of %s"
                          % (scope, config_scopes.keys()))
 
 
@@ -350,7 +394,7 @@ def _read_config_file(filename, schema):
             "Invlaid configuration. %s exists but is not a file." % filename)
 
     elif not os.access(filename, os.R_OK):
-        raise ConfigFileError("Config file is not readable: %s." % filename)
+        raise ConfigFileError("Config file is not readable: %s" % filename)
 
     try:
         tty.debug("Reading config file %s" % filename)
@@ -494,6 +538,39 @@ def print_section(section):
         raise ConfigError("Error reading configuration: %s" % section)
 
 
+def spec_externals(spec):
+    """Return a list of external specs (with external directory path filled in),
+       one for each known external installation."""
+    allpkgs = get_config('packages')
+    name = spec.name
+
+    external_specs = []
+    pkg_paths = allpkgs.get(name, {}).get('paths', None)
+    if not pkg_paths:
+        return []
+
+    for external_spec, path in pkg_paths.iteritems():
+        if not path:
+            # skip entries without paths (avoid creating extra Specs)
+            continue
+
+        external_spec = spack.spec.Spec(external_spec, external=path)
+        if external_spec.satisfies(spec):
+            external_specs.append(external_spec)
+    return external_specs
+
+
+def is_spec_buildable(spec):
+    """Return true if the spec pkgspec is configured as buildable"""
+    allpkgs = get_config('packages')
+    name = spec.name
+    if not spec.name in allpkgs:
+        return True
+    if not 'buildable' in allpkgs[spec.name]:
+        return True
+    return allpkgs[spec.name]['buildable']
+
+
 class ConfigError(SpackError): pass
 class ConfigFileError(ConfigError): pass
 
@@ -509,7 +586,7 @@ def __init__(self, validation_error, data):
         # Try to get line number from erroneous instance and its parent
         instance_mark = getattr(validation_error.instance, '_start_mark', None)
         parent_mark = getattr(validation_error.parent, '_start_mark', None)
-        path = getattr(validation_error, 'path', None)
+        path = [str(s) for s in getattr(validation_error, 'path', None)]
 
         # Try really hard to get the parent (which sometimes is not
         # set) This digs it out of the validated structure if it's not

lib/spack/spack/database.py

@@ -330,7 +330,7 @@ def _check_ref_counts(self):
             found = rec.ref_count
             if not expected == found:
                 raise AssertionError(
-                    "Invalid ref_count: %s: %d (expected %d), in DB %s."
+                    "Invalid ref_count: %s: %d (expected %d), in DB %s"
                     % (key, found, expected, self._index_path))
 
 

lib/spack/spack/directives.py

@@ -125,7 +125,7 @@ def __init__(self, dicts=None):
             dicts = (dicts,)
         elif type(dicts) not in (list, tuple):
             raise TypeError(
-                "dicts arg must be list, tuple, or string. Found %s."
+                "dicts arg must be list, tuple, or string. Found %s"
                 % type(dicts))
 
         self.dicts = dicts
@@ -317,5 +317,5 @@ class CircularReferenceError(DirectiveError):
     def __init__(self, directive, package):
         super(CircularReferenceError, self).__init__(
             directive,
-            "Package '%s' cannot pass itself to %s." % (package, directive))
+            "Package '%s' cannot pass itself to %s" % (package, directive))
         self.package = package

lib/spack/spack/directory_layout.py

@@ -85,6 +85,16 @@ def create_install_directory(self, spec):
         raise NotImplementedError()
 
 
+    def check_installed(self, spec):
+        """Checks whether a spec is installed.
+
+        Return the spec's prefix, if it is installed, None otherwise.
+
+        Raise an exception if the install is inconsistent or corrupt.
+        """
+        raise NotImplementedError()
+
+
     def extension_map(self, spec):
         """Get a dict of currently installed extension packages for a spec.
 
@@ -140,7 +150,7 @@ def remove_install_directory(self, spec):
         if os.path.exists(path):
             try:
                 shutil.rmtree(path)
-            except exceptions.OSError, e:
+            except exceptions.OSError as e:
                 raise RemoveFailedError(spec, path, e)
 
         path = os.path.dirname(path)
@@ -188,6 +198,10 @@ def hidden_file_paths(self):
 
     def relative_path_for_spec(self, spec):
         _check_concrete(spec)
+
+        if spec.external:
+            return spec.external
+
         dir_name = "%s-%s-%s" % (
             spec.name,
             spec.version,
@@ -246,17 +260,30 @@ def build_packages_path(self, spec):
     def create_install_directory(self, spec):
         _check_concrete(spec)
 
+        prefix = self.check_installed(spec)
+        if prefix:
+            raise InstallDirectoryAlreadyExistsError(prefix)
+
+        mkdirp(self.metadata_path(spec))
+        self.write_spec(spec, self.spec_file_path(spec))
+
+
+    def check_installed(self, spec):
+        _check_concrete(spec)
         path = self.path_for_spec(spec)
         spec_file_path = self.spec_file_path(spec)
 
-        if os.path.isdir(path):
+        if not os.path.isdir(path):
+            return None
+
         if not os.path.isfile(spec_file_path):
             raise InconsistentInstallDirectoryError(
-                    'No spec file found at path %s' % spec_file_path)
+                'Inconsistent state: install prefix exists but contains no spec.yaml:',
+                "  " + path)
 
         installed_spec = self.read_spec(spec_file_path)
-            if installed_spec == self.spec:
+        if installed_spec == spec:
-                raise InstallDirectoryAlreadyExistsError(path)
+            return path
 
         if spec.dag_hash() == installed_spec.dag_hash():
             raise SpecHashCollisionError(installed_hash, spec_hash)
@@ -264,9 +291,6 @@ def create_install_directory(self, spec):
             raise InconsistentInstallDirectoryError(
                 'Spec file in %s does not match hash!' % spec_file_path)
 
-        mkdirp(self.metadata_path(spec))
-        self.write_spec(spec, spec_file_path)
-
 
     def all_specs(self):
         if not os.path.isdir(self.root):
@@ -335,7 +359,7 @@ def _extension_map(self, spec):
 
                         if not dag_hash in by_hash:
                             raise InvalidExtensionSpecError(
-                                "Spec %s not found in %s." % (dag_hash, prefix))
+                                "Spec %s not found in %s" % (dag_hash, prefix))
 
                         ext_spec = by_hash[dag_hash]
                         if not prefix == ext_spec.prefix:
@@ -399,8 +423,8 @@ def remove_extension(self, spec, ext_spec):
 
 class DirectoryLayoutError(SpackError):
     """Superclass for directory layout errors."""
-    def __init__(self, message):
+    def __init__(self, message, long_msg=None):
-        super(DirectoryLayoutError, self).__init__(message)
+        super(DirectoryLayoutError, self).__init__(message, long_msg)
 
 
 class SpecHashCollisionError(DirectoryLayoutError):
@@ -422,8 +446,8 @@ def __init__(self, installed_spec, prefix, error):
 
 class InconsistentInstallDirectoryError(DirectoryLayoutError):
     """Raised when a package seems to be installed to the wrong place."""
-    def __init__(self, message):
+    def __init__(self, message, long_msg=None):
-        super(InconsistentInstallDirectoryError, self).__init__(message)
+        super(InconsistentInstallDirectoryError, self).__init__(message, long_msg)
 
 
 class InstallDirectoryAlreadyExistsError(DirectoryLayoutError):
@@ -450,7 +474,7 @@ class ExtensionConflictError(DirectoryLayoutError):
     """Raised when an extension is added to a package that already has it."""
     def __init__(self, spec, ext_spec, conflict):
         super(ExtensionConflictError, self).__init__(
-            "%s cannot be installed in %s because it conflicts with %s."% (
+            "%s cannot be installed in %s because it conflicts with %s"% (
                 ext_spec.short_spec, spec.short_spec, conflict.short_spec))
 
 

lib/spack/spack/environment.py

@@ -0,0 +1,252 @@
+import os
+import os.path
+import collections
+import inspect
+
+
+class NameModifier(object):
+    def __init__(self, name, **kwargs):
+        self.name = name
+        self.args = {'name': name}
+        self.args.update(kwargs)
+
+
+class NameValueModifier(object):
+    def __init__(self, name, value, **kwargs):
+        self.name = name
+        self.value = value
+        self.args = {'name': name, 'value': value}
+        self.args.update(kwargs)
+
+
+class SetEnv(NameValueModifier):
+    def execute(self):
+        os.environ[self.name] = str(self.value)
+
+
+class UnsetEnv(NameModifier):
+    def execute(self):
+        os.environ.pop(self.name, None)  # Avoid throwing if the variable was not set
+
+
+class SetPath(NameValueModifier):
+    def execute(self):
+        string_path = concatenate_paths(self.value)
+        os.environ[self.name] = string_path
+
+
+class AppendPath(NameValueModifier):
+    def execute(self):
+        environment_value = os.environ.get(self.name, '')
+        directories = environment_value.split(':') if environment_value else []
+        directories.append(os.path.normpath(self.value))
+        os.environ[self.name] = ':'.join(directories)
+
+
+class PrependPath(NameValueModifier):
+    def execute(self):
+        environment_value = os.environ.get(self.name, '')
+        directories = environment_value.split(':') if environment_value else []
+        directories = [os.path.normpath(self.value)] + directories
+        os.environ[self.name] = ':'.join(directories)
+
+
+class RemovePath(NameValueModifier):
+    def execute(self):
+        environment_value = os.environ.get(self.name, '')
+        directories = environment_value.split(':') if environment_value else []
+        directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)]
+        os.environ[self.name] = ':'.join(directories)
+
+
+class EnvironmentModifications(object):
+    """
+    Keeps track of requests to modify the current environment.
+
+    Each call to a method to modify the environment stores the extra information on the caller in the request:
+    - 'filename' : filename of the module where the caller is defined
+    - 'lineno': line number where the request occurred
+    - 'context' : line of code that issued the request that failed
+    """
+
+    def __init__(self, other=None):
+        """
+        Initializes a new instance, copying commands from other if it is not None
+
+        Args:
+            other: another instance of EnvironmentModifications from which (optional)
+        """
+        self.env_modifications = []
+        if other is not None:
+            self.extend(other)
+
+    def __iter__(self):
+        return iter(self.env_modifications)
+
+    def __len__(self):
+        return len(self.env_modifications)
+
+    def extend(self, other):
+        self._check_other(other)
+        self.env_modifications.extend(other.env_modifications)
+
+    @staticmethod
+    def _check_other(other):
+        if not isinstance(other, EnvironmentModifications):
+            raise TypeError('other must be an instance of EnvironmentModifications')
+
+    def _get_outside_caller_attributes(self):
+        stack = inspect.stack()
+        try:
+            _, filename, lineno, _, context, index = stack[2]
+            context = context[index].strip()
+        except Exception:
+            filename, lineno, context = 'unknown file', 'unknown line', 'unknown context'
+        args = {
+            'filename': filename,
+            'lineno': lineno,
+            'context': context
+        }
+        return args
+
+    def set(self, name, value, **kwargs):
+        """
+        Stores in the current object a request to set an environment variable
+
+        Args:
+            name: name of the environment variable to be set
+            value: value of the environment variable
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        item = SetEnv(name, value, **kwargs)
+        self.env_modifications.append(item)
+
+    def unset(self, name, **kwargs):
+        """
+        Stores in the current object a request to unset an environment variable
+
+        Args:
+            name: name of the environment variable to be set
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        item = UnsetEnv(name, **kwargs)
+        self.env_modifications.append(item)
+
+    def set_path(self, name, elts, **kwargs):
+        """
+        Stores a request to set a path generated from a list.
+
+        Args:
+            name: name o the environment variable to be set.
+            elts: elements of the path to set.
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        item = SetPath(name, elts, **kwargs)
+        self.env_modifications.append(item)
+
+    def append_path(self, name, path, **kwargs):
+        """
+        Stores in the current object a request to append a path to a path list
+
+        Args:
+            name: name of the path list in the environment
+            path: path to be appended
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        item = AppendPath(name, path, **kwargs)
+        self.env_modifications.append(item)
+
+    def prepend_path(self, name, path, **kwargs):
+        """
+        Same as `append_path`, but the path is pre-pended
+
+        Args:
+            name: name of the path list in the environment
+            path: path to be pre-pended
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        item = PrependPath(name, path, **kwargs)
+        self.env_modifications.append(item)
+
+    def remove_path(self, name, path, **kwargs):
+        """
+        Stores in the current object a request to remove a path from a path list
+
+        Args:
+            name: name of the path list in the environment
+            path: path to be removed
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        item = RemovePath(name, path, **kwargs)
+        self.env_modifications.append(item)
+
+    def group_by_name(self):
+        """
+        Returns a dict of the modifications grouped by variable name
+
+        Returns:
+            dict mapping the environment variable name to the modifications to be done on it
+        """
+        modifications = collections.defaultdict(list)
+        for item in self:
+            modifications[item.name].append(item)
+        return modifications
+
+    def clear(self):
+        """
+        Clears the current list of modifications
+        """
+        self.env_modifications.clear()
+
+    def apply_modifications(self):
+        """
+        Applies the modifications and clears the list
+        """
+        modifications = self.group_by_name()
+        # Apply the modifications to the environment variables one variable at a time
+        for name, actions in sorted(modifications.items()):
+            for x in actions:
+                x.execute()
+
+
+def concatenate_paths(paths):
+    """
+    Concatenates an iterable of paths into a  string of column separated paths
+
+    Args:
+        paths: iterable of paths
+
+    Returns:
+        string
+    """
+    return ':'.join(str(item) for item in paths)
+
+
+def set_or_unset_not_first(variable, changes, errstream):
+    """
+    Check if we are going to set or unset something after other modifications have already been requested
+    """
+    indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]]
+    if indexes:
+        good = '\t    \t{context} at {filename}:{lineno}'
+        nogood = '\t--->\t{context} at {filename}:{lineno}'
+        errstream('Suspicious requests to set or unset the variable \'{var}\' found'.format(var=variable))
+        for ii, item in enumerate(changes):
+            print_format = nogood if ii in indexes else good
+            errstream(print_format.format(**item.args))
+
+
+def validate(env, errstream):
+    """
+    Validates the environment modifications to check for the presence of suspicious patterns. Prompts a warning for
+    everything that was found
+
+    Current checks:
+    - set or unset variables after other changes on the same variable
+
+    Args:
+        env: list of environment modifications
+    """
+    modifications = env.group_by_name()
+    for variable, list_of_changes in sorted(modifications.items()):
+        set_or_unset_not_first(variable, list_of_changes, errstream)

lib/spack/spack/fetch_strategy.py

@@ -82,7 +82,6 @@ class FetchStrategy(object):
 
     class __metaclass__(type):
         """This metaclass registers all fetch strategies in a list."""
-
         def __init__(cls, name, bases, dict):
             type.__init__(cls, name, bases, dict)
             if cls.enabled: all_strategies.append(cls)
@@ -145,6 +144,8 @@ def __init__(self, url=None, digest=None, **kwargs):
         self.digest = kwargs.get('md5', None)
         if not self.digest: self.digest = digest
 
+        self.expand_archive = kwargs.get('expand', True)
+
         if not self.url:
             raise ValueError("URLFetchStrategy requires a url for fetching.")
 
@@ -153,7 +154,7 @@ def fetch(self):
         self.stage.chdir()
 
         if self.archive_file:
-            tty.msg("Already downloaded %s." % self.archive_file)
+            tty.msg("Already downloaded %s" % self.archive_file)
             return
 
         tty.msg("Trying to fetch from %s" % self.url)
@@ -218,6 +219,10 @@ def archive_file(self):
 
     @_needs_stage
     def expand(self):
+        if not self.expand_archive:
+            tty.msg("Skipping expand step for %s" % self.archive_file)
+            return
+
         tty.msg("Staging archive: %s" % self.archive_file)
 
         self.stage.chdir()
@@ -275,8 +280,8 @@ def check(self):
         checker = crypto.Checker(self.digest)
         if not checker.check(self.archive_file):
             raise ChecksumError(
-                    "%s checksum failed for %s." % (checker.hash_name, self.archive_file),
+                    "%s checksum failed for %s" % (checker.hash_name, self.archive_file),
-                    "Expected %s but got %s." % (self.digest, checker.sum))
+                    "Expected %s but got %s" % (self.digest, checker.sum))
 
     @_needs_stage
     def reset(self):
@@ -312,7 +317,7 @@ def __init__(self, name, *rev_types, **kwargs):
         # Ensure that there's only one of the rev_types
         if sum(k in kwargs for k in rev_types) > 1:
             raise FetchStrategyError(
-                    "Supply only one of %s to fetch with %s." % (
+                    "Supply only one of %s to fetch with %s" % (
                         comma_or(rev_types), name))
 
         # Set attributes for each rev type.
@@ -321,7 +326,7 @@ def __init__(self, name, *rev_types, **kwargs):
 
     @_needs_stage
     def check(self):
-        tty.msg("No checksum needed when fetching with %s." % self.name)
+        tty.msg("No checksum needed when fetching with %s" % self.name)
 
     @_needs_stage
     def expand(self):
@@ -395,7 +400,7 @@ def fetch(self):
         self.stage.chdir()
 
         if self.stage.source_path:
-            tty.msg("Already fetched %s." % self.stage.source_path)
+            tty.msg("Already fetched %s" % self.stage.source_path)
             return
 
         args = []
@@ -505,7 +510,7 @@ def fetch(self):
         self.stage.chdir()
 
         if self.stage.source_path:
-            tty.msg("Already fetched %s." % self.stage.source_path)
+            tty.msg("Already fetched %s" % self.stage.source_path)
             return
 
         tty.msg("Trying to check out svn repository: %s" % self.url)
@@ -584,7 +589,7 @@ def fetch(self):
         self.stage.chdir()
 
         if self.stage.source_path:
-            tty.msg("Already fetched %s." % self.stage.source_path)
+            tty.msg("Already fetched %s" % self.stage.source_path)
             return
 
         args = []

lib/spack/spack/hooks/sbang.py

@@ -0,0 +1,98 @@
+##############################################################################
+# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import os
+
+from llnl.util.filesystem import *
+import llnl.util.tty as tty
+
+import spack
+import spack.modules
+
+# Character limit for shebang line.  Using Linux's 127 characters
+# here, as it is the shortest I could find on a modern OS.
+shebang_limit = 127
+
+def shebang_too_long(path):
+    """Detects whether a file has a shebang line that is too long."""
+    with open(path, 'r') as script:
+        bytes = script.read(2)
+        if bytes != '#!':
+            return False
+
+        line = bytes + script.readline()
+        return len(line) > shebang_limit
+
+
+def filter_shebang(path):
+    """Adds a second shebang line, using sbang, at the beginning of a file."""
+    with open(path, 'r') as original_file:
+        original = original_file.read()
+
+    # This line will be prepended to file
+    new_sbang_line = '#!/bin/bash %s/bin/sbang\n' % spack.spack_root
+
+    # Skip files that are already using sbang.
+    if original.startswith(new_sbang_line):
+        return
+
+    backup = path + ".shebang.bak"
+    os.rename(path, backup)
+
+    with open(path, 'w') as new_file:
+        new_file.write(new_sbang_line)
+        new_file.write(original)
+
+    copy_mode(backup, path)
+    unset_executable_mode(backup)
+
+    tty.warn("Patched overly long shebang in %s" % path)
+
+
+def filter_shebangs_in_directory(directory):
+    for file in os.listdir(directory):
+        path = os.path.join(directory, file)
+
+        # only handle files
+        if not os.path.isfile(path):
+            continue
+
+        # only handle links that resolve within THIS package's prefix.
+        if os.path.islink(path):
+            real_path = os.path.realpath(path)
+            if not real_path.startswith(directory + os.sep):
+                continue
+
+        # test the file for a long shebang, and filter
+        if shebang_too_long(path):
+            filter_shebang(path)
+
+
+def post_install(pkg):
+    """This hook edits scripts so that they call /bin/bash
+       $spack_prefix/bin/sbang instead of something longer than the
+       shebang limit."""
+    if not os.path.isdir(pkg.prefix.bin):
+        return
+    filter_shebangs_in_directory(pkg.prefix.bin)

lib/spack/spack/mirror.py

@@ -51,13 +51,20 @@ def mirror_archive_filename(spec, fetcher):
         raise ValueError("mirror.path requires spec with concrete version.")
 
     if isinstance(fetcher, fs.URLFetchStrategy):
+        if fetcher.expand_archive:
             # If we fetch this version with a URLFetchStrategy, use URL's archive type
             ext = url.downloaded_file_extension(fetcher.url)
+        else:
+            # If the archive shouldn't be expanded, don't check for its extension.
+            ext = None
     else:
         # Otherwise we'll make a .tar.gz ourselves
         ext = 'tar.gz'
 
-    return "%s-%s.%s" % (spec.package.name, spec.version, ext)
+    filename = "%s-%s" % (spec.package.name, spec.version)
+    if ext:
+        filename += ".%s" % ext
+    return filename
 
 
 def mirror_archive_path(spec, fetcher):
@@ -73,7 +80,7 @@ def get_matching_versions(specs, **kwargs):
 
         # Skip any package that has no known versions.
         if not pkg.versions:
-            tty.msg("No safe (checksummed) versions for package %s." % pkg.name)
+            tty.msg("No safe (checksummed) versions for package %s" % pkg.name)
             continue
 
         num_versions = kwargs.get('num_versions', 0)
@@ -214,7 +221,7 @@ def add_single_spec(spec, mirror_root, categories, **kwargs):
         if spack.debug:
             sys.excepthook(*sys.exc_info())
         else:
-            tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message)
+            tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message)
         categories['error'].append(spec)
 
 

lib/spack/spack/modules.py

@@ -22,14 +22,12 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-"""This module contains code for creating environment modules, which
+"""
-can include dotkits, tcl modules, lmod, and others.
+This module contains code for creating environment modules, which can include dotkits, tcl modules, lmod, and others.
 
-The various types of modules are installed by post-install hooks and
+The various types of modules are installed by post-install hooks and removed after an uninstall by post-uninstall hooks.
-removed after an uninstall by post-uninstall hooks.  This class
+This class consolidates the logic for creating an abstract description of the information that module systems need.
-consolidates the logic for creating an abstract description of the
+Currently that includes a number of directories to be appended to paths in the user's environment:
-information that module systems need.  Currently that includes a
-number of directories to be appended to paths in the user's environment:
 
   * /bin directories to be appended to PATH
   * /lib* directories for LD_LIBRARY_PATH
@@ -37,28 +35,25 @@
   * /man* and /share/man* directories for MANPATH
   * the package prefix for CMAKE_PREFIX_PATH
 
-This module also includes logic for coming up with unique names for
+This module also includes logic for coming up with unique names for the module files so that they can be found by the
-the module files so that they can be found by the various
+various shell-support files in $SPACK/share/spack/setup-env.*.
-shell-support files in $SPACK/share/spack/setup-env.*.
 
-Each hook in hooks/ implements the logic for writing its specific type
+Each hook in hooks/ implements the logic for writing its specific type of module file.
-of module file.
 """
-__all__ = ['EnvModule', 'Dotkit', 'TclModule']
-
 import os
+import os.path
 import re
-import textwrap
 import shutil
-from glob import glob
+import textwrap
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import join_path, mkdirp
-
 import spack
+from llnl.util.filesystem import join_path, mkdirp
+from spack.environment import *
 
-"""Registry of all types of modules.  Entries created by EnvModule's
+__all__ = ['EnvModule', 'Dotkit', 'TclModule']
-   metaclass."""
+
+# Registry of all types of modules.  Entries created by EnvModule's metaclass
 module_types = {}
 
 
@@ -79,8 +74,43 @@ def print_help():
             "")
 
 
+def inspect_path(prefix):
+    """
+    Inspects the prefix of an installation to search for common layouts. Issues a request to modify the environment
+    accordingly when an item is found.
+
+    Args:
+        prefix: prefix of the installation
+
+    Returns:
+        instance of EnvironmentModifications containing the requested modifications
+    """
+    env = EnvironmentModifications()
+    # Inspect the prefix to check for the existence of common directories
+    prefix_inspections = {
+        'bin': ('PATH',),
+        'man': ('MANPATH',),
+        'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'),
+        'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'),
+        'include': ('CPATH',)
+    }
+    for attribute, variables in prefix_inspections.items():
+        expected = getattr(prefix, attribute)
+        if os.path.isdir(expected):
+            for variable in variables:
+                env.prepend_path(variable, expected)
+    # PKGCONFIG
+    for expected in (join_path(prefix.lib, 'pkgconfig'), join_path(prefix.lib64, 'pkgconfig')):
+        if os.path.isdir(expected):
+            env.prepend_path('PKG_CONFIG_PATH', expected)
+    # CMake related variables
+    env.prepend_path('CMAKE_PREFIX_PATH', prefix)
+    return env
+
+
 class EnvModule(object):
     name = 'env_module'
+    formats = {}
 
     class __metaclass__(type):
         def __init__(cls, name, bases, dict):
@@ -88,66 +118,32 @@ def __init__(cls, name, bases, dict):
             if cls.name != 'env_module':
                 module_types[cls.name] = cls
 
-
     def __init__(self, spec=None):
-        # category in the modules system
-        # TODO: come up with smarter category names.
-        self.category = "spack"
-
-        # Descriptions for the module system's UI
-        self.short_description = ""
-        self.long_description = ""
-
-        # dict pathname -> list of directories to be prepended to in
-        # the module file.
-        self._paths = None
         self.spec = spec
+        self.pkg = spec.package  # Just stored for convenience
 
-
+        # short description default is just the package + version
-    @property
+        # packages can provide this optional attribute
-    def paths(self):
+        self.short_description = spec.format("$_ $@")
-        if self._paths is None:
+        if hasattr(self.pkg, 'short_description'):
-            self._paths = {}
+            self.short_description = self.pkg.short_description
-
-            def add_path(path_name, directory):
-                path = self._paths.setdefault(path_name, [])
-                path.append(directory)
-
-            # Add paths if they exist.
-            for var, directory in [
-                    ('PATH', self.spec.prefix.bin),
-                    ('MANPATH', self.spec.prefix.man),
-                    ('MANPATH', self.spec.prefix.share_man),
-                    ('LIBRARY_PATH', self.spec.prefix.lib),
-                    ('LIBRARY_PATH', self.spec.prefix.lib64),
-                    ('LD_LIBRARY_PATH', self.spec.prefix.lib),
-                    ('LD_LIBRARY_PATH', self.spec.prefix.lib64),
-                    ('CPATH', self.spec.prefix.include),
-                    ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')),
-                    ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]:
-
-                if os.path.isdir(directory):
-                    add_path(var, directory)
-
-            # Add python path unless it's an actual python installation
-            # TODO: is there a better way to do this?
-            if self.spec.name != 'python':
-                site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages"))
-                if site_packages:
-                    add_path('PYTHONPATH', site_packages[0])
-
-            if self.spec.package.extends(spack.spec.Spec('ruby')):
-              add_path('GEM_PATH', self.spec.prefix)
-
-            # short description is just the package + version
-            # TODO: maybe packages can optionally provide it.
-            self.short_description = self.spec.format("$_ $@")
 
         # long description is the docstring with reduced whitespace.
+        self.long_description = None
         if self.spec.package.__doc__:
             self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__)
 
-        return self._paths
+
+    @property
+    def category(self):
+        # Anything defined at the package level takes precedence
+        if hasattr(self.pkg, 'category'):
+            return self.pkg.category
+        # Extensions
+        for extendee in self.pkg.extendees:
+            return '{extendee} extension'.format(extendee=extendee)
+        # Not very descriptive fallback
+        return 'spack installed package'
 
 
     def write(self):
@@ -156,18 +152,41 @@ def write(self):
         if not os.path.exists(module_dir):
             mkdirp(module_dir)
 
-        # If there are no paths, no need for a dotkit.
+        # Environment modifications guessed by inspecting the
-        if not self.paths:
+        # installation prefix
+        env = inspect_path(self.spec.prefix)
+
+        # Let the extendee modify their extensions before asking for
+        # package-specific modifications
+        for extendee in self.pkg.extendees:
+            extendee_spec = self.spec[extendee]
+            extendee_spec.package.setup_dependent_package(
+                self.pkg.module, self.spec)
+
+        # Package-specific environment modifications
+        spack_env = EnvironmentModifications()
+        self.spec.package.setup_environment(spack_env, env)
+
+        # TODO : implement site-specific modifications and filters
+        if not env:
             return
 
         with open(self.file_name, 'w') as f:
-            self._write(f)
+            self.write_header(f)
+            for line in self.process_environment_command(env):
+                f.write(line)
 
-
+    def write_header(self, stream):
-    def _write(self, stream):
-        """To be implemented by subclasses."""
         raise NotImplementedError()
 
+    def process_environment_command(self, env):
+        for command in env:
+            try:
+                yield self.formats[type(command)].format(**command.args)
+            except KeyError:
+                tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command)))
+                tty.warn('{context} at {filename}:{lineno}'.format(**command.args))
+
 
     @property
     def file_name(self):
@@ -175,14 +194,12 @@ def file_name(self):
            where this module lives."""
         raise NotImplementedError()
 
-
     @property
     def use_name(self):
         """Subclasses should implement this to return the name the
            module command uses to refer to the package."""
         raise NotImplementedError()
 
-
     def remove(self):
         mod_file = self.file_name
         if os.path.exists(mod_file):
@@ -193,10 +210,14 @@ class Dotkit(EnvModule):
     name = 'dotkit'
     path = join_path(spack.share_path, "dotkit")
 
+    formats = {
+        PrependPath: 'dk_alter {name} {value}\n',
+        SetEnv: 'dk_setenv {name} {value}\n'
+    }
+
     @property
     def file_name(self):
-        return join_path(Dotkit.path, self.spec.architecture,
+        return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name)
-                         '%s.dk' % self.use_name)
 
     @property
     def use_name(self):
@@ -205,7 +226,7 @@ def use_name(self):
                                  self.spec.compiler.version,
                                  self.spec.dag_hash())
 
-    def _write(self, dk_file):
+    def write_header(self, dk_file):
         # Category
         if self.category:
             dk_file.write('#c %s\n' % self.category)
@@ -219,24 +240,23 @@ def _write(self, dk_file):
             for line in textwrap.wrap(self.long_description, 72):
                 dk_file.write("#h %s\n" % line)
 
-        # Path alterations
-        for var, dirs in self.paths.items():
-            for directory in dirs:
-                dk_file.write("dk_alter %s %s\n" % (var, directory))
-
-        # Let CMake find this package.
-        dk_file.write("dk_alter CMAKE_PREFIX_PATH %s\n" % self.spec.prefix)
-
 
 class TclModule(EnvModule):
     name = 'tcl'
     path = join_path(spack.share_path, "modules")
 
+    formats = {
+        PrependPath: 'prepend-path {name} \"{value}\"\n',
+        AppendPath: 'append-path {name} \"{value}\"\n',
+        RemovePath: 'remove-path {name} \"{value}\"\n',
+        SetEnv: 'setenv {name} \"{value}\"\n',
+        UnsetEnv: 'unsetenv {name}\n'
+    }
+
     @property
     def file_name(self):
         return join_path(TclModule.path, self.spec.architecture, self.use_name)
 
-
     @property
     def use_name(self):
       return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version,
@@ -244,25 +264,17 @@ def use_name(self):
                                  self.spec.compiler.version,
                                  self.spec.dag_hash())
 
-
+    def write_header(self, module_file):
-    def _write(self, m_file):
+        # TCL Modulefile header
-        # TODO: cateogry?
+        module_file.write('#%Module1.0\n')
-        m_file.write('#%Module1.0\n')
+        # TODO : category ?
-
         # Short description
         if self.short_description:
-            m_file.write('module-whatis \"%s\"\n\n' % self.short_description)
+            module_file.write('module-whatis \"%s\"\n\n' % self.short_description)
 
         # Long description
         if self.long_description:
-            m_file.write('proc ModulesHelp { } {\n')
+            module_file.write('proc ModulesHelp { } {\n')
             doc = re.sub(r'"', '\"', self.long_description)
-            m_file.write("puts stderr \"%s\"\n" % doc)
+            module_file.write("puts stderr \"%s\"\n" % doc)
-            m_file.write('}\n\n')
+            module_file.write('}\n\n')
-
-        # Path alterations
-        for var, dirs in self.paths.items():
-            for directory in dirs:
-                m_file.write("prepend-path %s \"%s\"\n" % (var, directory))
-
-        m_file.write("prepend-path CMAKE_PREFIX_PATH \"%s\"\n" % self.spec.prefix)

lib/spack/spack/multimethod.py

@@ -138,7 +138,7 @@ class when(object):
        methods like install() that depend on the package's spec.
        For example:
 
-       .. code-block::
+       .. code-block:: python
 
           class SomePackage(Package):
               ...
@@ -163,6 +163,8 @@ def install(self, prefix):
        if you only have part of the install that is platform specific, you
        could do this:
 
+       .. code-block:: python
+
           class SomePackage(Package):
               ...
               # virtual dependence on MPI.

lib/spack/spack/package.py

@@ -34,40 +34,34 @@
 README.
 """
 import os
-import errno
 import re
-import shutil
-import time
-import itertools
-import subprocess
-import platform as py_platform
-import multiprocessing
-from urlparse import urlparse, urljoin
 import textwrap
-from StringIO import StringIO
+import time
+import glob
 
 import llnl.util.tty as tty
-from llnl.util.tty.log import log_output
-from llnl.util.link_tree import LinkTree
-from llnl.util.filesystem import *
-from llnl.util.lang import *
-
 import spack
-import spack.error
-import spack.compilers
-import spack.mirror
-import spack.hooks
-import spack.directives
-import spack.repository
 import spack.build_environment
+import spack.compilers
+import spack.directives
+import spack.error
+import spack.fetch_strategy as fs
+import spack.hooks
+import spack.mirror
+import spack.repository
 import spack.url
 import spack.util.web
-import spack.fetch_strategy as fs
+from StringIO import StringIO
-from spack.version import *
+from llnl.util.filesystem import *
+from llnl.util.lang import *
+from llnl.util.link_tree import LinkTree
+from llnl.util.tty.log import log_output
 from spack.stage import Stage, ResourceStage, StageComposite
-from spack.util.compression import allowed_archive, extension
+from spack.util.compression import allowed_archive
-from spack.util.executable import ProcessError
 from spack.util.environment import dump_environment
+from spack.util.executable import ProcessError
+from spack.version import *
+from urlparse import urlparse
 
 """Allowed URL schemes for spack packages."""
 _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
@@ -293,6 +287,7 @@ class SomePackage(Package):
 
     .. code-block:: python
 
+       p.do_clean()              # removes the stage directory entirely
        p.do_restage()            # removes the build directory and
                                  # re-expands the archive.
 
@@ -317,6 +312,18 @@ class SomePackage(Package):
     """Most packages are NOT extendable.  Set to True if you want extensions."""
     extendable = False
 
+    """List of prefix-relative file paths (or a single path). If these do
+       not exist after install, or if they exist but are not files,
+       sanity checks fail.
+    """
+    sanity_check_is_file = []
+
+    """List of prefix-relative directory paths (or a single path). If
+       these do not exist after install, or if they exist but are not
+       directories, sanity checks will fail.
+    """
+    sanity_check_is_dir = []
+
 
     def __init__(self, spec):
         # this determines how the package should be built.
@@ -455,7 +462,7 @@ def _make_stage(self):
         # Construct a composite stage on top of the composite FetchStrategy
         composite_fetcher = self.fetcher
         composite_stage = StageComposite()
-        resources = self._get_resources()
+        resources = self._get_needed_resources()
         for ii, fetcher in enumerate(composite_fetcher):
             if ii == 0:
                 # Construct root stage first
@@ -466,6 +473,11 @@ def _make_stage(self):
                 stage = self._make_resource_stage(composite_stage[0], fetcher, resource)
             # Append the item to the composite
             composite_stage.append(stage)
+
+        # Create stage on first access.  Needed because fetch, stage,
+        # patch, and install can be called independently of each
+        # other, so `with self.stage:` in do_install isn't sufficient.
+        composite_stage.create()
         return composite_stage
 
     @property
@@ -484,12 +496,14 @@ def stage(self, stage):
 
 
     def _make_fetcher(self):
-        # Construct a composite fetcher that always contains at least one element (the root package). In case there
+        # Construct a composite fetcher that always contains at least
-        # are resources associated with the package, append their fetcher to the composite.
+        # one element (the root package). In case there are resources
+        # associated with the package, append their fetcher to the
+        # composite.
         root_fetcher = fs.for_package_version(self, self.version)
         fetcher = fs.FetchStrategyComposite()  # Composite fetcher
         fetcher.append(root_fetcher)  # Root fetcher is always present
-        resources = self._get_resources()
+        resources = self._get_needed_resources()
         for resource in resources:
             fetcher.append(resource.fetcher)
         return fetcher
@@ -686,7 +700,7 @@ def do_fetch(self, mirror_only=False):
 
             if not ignore_checksum:
                 raise FetchError(
-                    "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg)
+                    "Will not fetch %s" % self.spec.format('$_$@'), checksum_msg)
 
         self.stage.fetch(mirror_only)
 
@@ -706,6 +720,7 @@ def do_stage(self, mirror_only=False):
         self.stage.expand_archive()
         self.stage.chdir_to_source()
 
+
     def do_patch(self):
         """Calls do_stage(), then applied patches to the expanded tarball if they
            haven't been applied already."""
@@ -720,7 +735,7 @@ def do_patch(self):
 
         # If there are no patches, note it.
         if not self.patches and not has_patch_fun:
-            tty.msg("No patches needed for %s." % self.name)
+            tty.msg("No patches needed for %s" % self.name)
             return
 
         # Construct paths to special files in the archive dir used to
@@ -743,7 +758,7 @@ def do_patch(self):
             tty.msg("Already patched %s" % self.name)
             return
         elif os.path.isfile(no_patches_file):
-            tty.msg("No patches needed for %s." % self.name)
+            tty.msg("No patches needed for %s" % self.name)
             return
 
         # Apply all the patches for specs that match this one
@@ -764,10 +779,10 @@ def do_patch(self):
         if has_patch_fun:
             try:
                 self.patch()
-                tty.msg("Ran patch() for %s." % self.name)
+                tty.msg("Ran patch() for %s" % self.name)
                 patched = True
             except:
-                tty.msg("patch() function failed for %s." % self.name)
+                tty.msg("patch() function failed for %s" % self.name)
                 touch(bad_file)
                 raise
 
@@ -798,7 +813,7 @@ def do_fake_install(self):
         mkdirp(self.prefix.man1)
 
 
-    def _get_resources(self):
+    def _get_needed_resources(self):
         resources = []
         # Select the resources that are needed for this build
         for when_spec, resource_list in self.resources.items():
@@ -825,7 +840,9 @@ def do_install(self,
 
         Args:
         keep_prefix -- Keep install prefix on failure. By default, destroys it.
-        keep_stage  -- Keep stage on successful build. By default, destroys it.
+        keep_stage  -- By default, stage is destroyed only if there are no
+                       exceptions during build. Set to True to keep the stage
+                       even with exceptions.
         ignore_deps -- Do not install dependencies before installing this package.
         fake        -- Don't really build -- install fake stub files instead.
         skip_patch  -- Skip patch stage of build if True.
@@ -835,51 +852,46 @@ def do_install(self,
         if not self.spec.concrete:
             raise ValueError("Can only install concrete packages.")
 
-        if os.path.exists(self.prefix):
+        # No installation needed if package is external
-            tty.msg("%s is already installed in %s." % (self.name, self.prefix))
+        if self.spec.external:
+            tty.msg("%s is externally installed in %s" % (self.name, self.spec.external))
+            return
+
+        # Ensure package is not already installed
+        if spack.install_layout.check_installed(self.spec):
+            tty.msg("%s is already installed in %s" % (self.name, self.prefix))
             return
 
         tty.msg("Installing %s" % self.name)
 
+        # First, install dependencies recursively.
         if not ignore_deps:
             self.do_install_dependencies(
                 keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps,
-                fake=fake, skip_patch=skip_patch, verbose=verbose,
+                fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs)
-                make_jobs=make_jobs)
 
+        # Set parallelism before starting build.
+        self.make_jobs = make_jobs
+
+        # Then install the package itself.
+        def build_process():
+            """Forked for each build. Has its own process and python
+               module space set up by build_environment.fork()."""
             start_time = time.time()
-        with self.stage:
             if not fake:
                 if not skip_patch:
                     self.do_patch()
                 else:
                     self.do_stage()
 
-            # create the install directory.  The install layout
+            tty.msg("Building %s" % self.name)
-            # handles this in case so that it can use whatever
-            # package naming scheme it likes.
-            spack.install_layout.create_install_directory(self.spec)
-
-            def cleanup():
-                if not keep_prefix:
-                    # If anything goes wrong, remove the install prefix
-                    self.remove_prefix()
-                else:
-                    tty.warn("Keeping install prefix in place despite error.",
-                             "Spack will think this package is installed." +
-                             "Manually remove this directory to fix:",
-                             self.prefix, wrap=True)
-
-
-            def real_work():
-                try:
-                    tty.msg("Building %s." % self.name)
 
+            self.stage.keep = keep_stage
+            with self.stage:
                 # Run the pre-install hook in the child process after
                 # the directory is created.
                 spack.hooks.pre_install(self)
 
-                    # Set up process's build environment before running install.
                 if fake:
                     self.do_fake_install()
                 else:
@@ -889,56 +901,55 @@ def real_work():
                      # Save the build environment in a file before building.
                      env_path = join_path(os.getcwd(), 'spack-build.env')
 
-                        # This redirects I/O to a build log (and optionally to the terminal)
+                     try:
+                        # Redirect I/O to a build log (and optionally to the terminal)
                         log_path = join_path(os.getcwd(), 'spack-build.out')
                         log_file = open(log_path, 'w')
                         with log_output(log_file, verbose, sys.stdout.isatty(), True):
                             dump_environment(env_path)
                             self.install(self.spec, self.prefix)
 
-                    # Ensure that something was actually installed.
+                     except ProcessError as e:
-                    self._sanity_check_install()
+                         # Annotate ProcessErrors with the location of the build log.
+                         e.build_log = log_path
+                         raise e
 
-                    # Move build log into install directory on success
+                     # Ensure that something was actually installed.
-                    if not fake:
+                     self.sanity_check_prefix()
+
+                     # Copy provenance into the install directory on success
                      log_install_path = spack.install_layout.build_log_path(self.spec)
                      env_install_path = spack.install_layout.build_env_path(self.spec)
+                     packages_dir = spack.install_layout.build_packages_path(self.spec)
+
                      install(log_path, log_install_path)
                      install(env_path, env_install_path)
-
-                    packages_dir = spack.install_layout.build_packages_path(self.spec)
                      dump_packages(self.spec, packages_dir)
 
-                    # On successful install, remove the stage.
-                    if not keep_stage:
-                        self.stage.destroy()
-
             # Stop timer.
             self._total_time = time.time() - start_time
             build_time = self._total_time - self._fetch_time
 
-                    tty.msg("Successfully installed %s." % self.name,
+            tty.msg("Successfully installed %s" % self.name,
                     "Fetch: %s.  Build: %s.  Total: %s."
                     % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time)))
             print_pkg(self.prefix)
 
-                except ProcessError as e:
+        try:
-                    # Annotate with location of build log.
+            # Create the install prefix and fork the build process.
-                    e.build_log = log_path
+            spack.install_layout.create_install_directory(self.spec)
-                    cleanup()
+            spack.build_environment.fork(self, build_process)
-                    raise e
-
         except:
-                    # other exceptions just clean up and raise.
+            # remove the install prefix if anything went wrong during install.
-                    cleanup()
+            if not keep_prefix:
+                self.remove_prefix()
+            else:
+                tty.warn("Keeping install prefix in place despite error.",
+                         "Spack will think this package is installed. " +
+                         "Manually remove this directory to fix:",
+                         self.prefix, wrap=True)
             raise
 
-            # Set parallelism before starting build.
-            self.make_jobs = make_jobs
-
-            # Do the build.
-            spack.build_environment.fork(self, real_work)
-
         # note: PARENT of the build process adds the new package to
         # the database, so that we don't need to re-read from file.
         spack.installed_db.add(self.spec, self.prefix)
@@ -947,7 +958,21 @@ def real_work():
         spack.hooks.post_install(self)
 
 
-    def _sanity_check_install(self):
+    def sanity_check_prefix(self):
+        """This function checks whether install succeeded."""
+        def check_paths(path_list, filetype, predicate):
+            if isinstance(path_list, basestring):
+                path_list = [path_list]
+
+            for path in path_list:
+                abs_path = os.path.join(self.prefix, path)
+                if not predicate(abs_path):
+                    raise InstallError("Install failed for %s. No such %s in prefix: %s"
+                                       % (self.name, filetype, path))
+
+        check_paths(self.sanity_check_is_file, 'file', os.path.isfile)
+        check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir)
+
         installed = set(os.listdir(self.prefix))
         installed.difference_update(spack.install_layout.hidden_file_paths)
         if not installed:
@@ -977,38 +1002,127 @@ def module(self):
         return __import__(self.__class__.__module__,
                           fromlist=[self.__class__.__name__])
 
+    def setup_environment(self, spack_env, run_env):
+        """Set up the compile and runtime environemnts for a package.
 
-    def setup_dependent_environment(self, module, spec, dependent_spec):
+        `spack_env` and `run_env` are `EnvironmentModifications`
-        """Called before the install() method of dependents.
+        objects.  Package authors can call methods on them to alter
+        the environment within Spack and at runtime.
+
+        Both `spack_env` and `run_env` are applied within the build
+        process, before this package's `install()` method is called.
+
+        Modifications in `run_env` will *also* be added to the
+        generated environment modules for this package.
 
         Default implementation does nothing, but this can be
-        overridden by an extendable package to set up the install
+        overridden if the package needs a particular environment.
-        environment for its extensions.  This is useful if there are
-        some common steps to installing all extensions for a
-        certain package.
 
-        Some examples:
+        Examples:
 
-        1. Installing python modules generally requires PYTHONPATH to
+            1. Qt extensions need `QTDIR` set.
-           point to the lib/pythonX.Y/site-packages directory in the
-           module's install prefix.  This could set that variable.
 
-        2. Extensions often need to invoke the 'python' interpreter
+        Args:
-           from the Python installation being extended.  This routine can
+            spack_env (EnvironmentModifications): list of
-           put a 'python' Execuable object in the module scope for the
+                modifications to be applied when this package is built
-           extension package to simplify extension installs.
+                within Spack.
 
-        3. A lot of Qt extensions need QTDIR set.  This can be used to do that.
+            run_env (EnvironmentModifications): list of environment
+                changes to be applied when this package is run outside
+                of Spack.
 
         """
         pass
 
 
+    def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
+        """Set up the environment of packages that depend on this one.
+
+        This is similar to `setup_environment`, but it is used to
+        modify the compile and runtime environments of packages that
+        *depend* on this one. This gives packages like Python and
+        others that follow the extension model a way to implement
+        common environment or compile-time settings for dependencies.
+
+        By default, this delegates to self.setup_environment()
+
+        Example :
+
+            1. Installing python modules generally requires
+              `PYTHONPATH` to point to the lib/pythonX.Y/site-packages
+              directory in the module's install prefix.  This could
+              set that variable.
+
+        Args:
+
+            spack_env (EnvironmentModifications): list of
+                modifications to be applied when the dependent package
+                is bulit within Spack.
+
+            run_env (EnvironmentModifications): list of environment
+                changes to be applied when the dependent package is
+                run outside of Spack.
+
+            dependent_spec (Spec): The spec of the dependent package
+                about to be built. This allows the extendee (self) to
+                query the dependent's state. Note that *this*
+                package's spec is available as `self.spec`.
+
+        This is useful if there are some common steps to installing
+        all extensions for a certain package.
+
+        """
+        self.setup_environment(spack_env, run_env)
+
+
+    def setup_dependent_package(self, module, dependent_spec):
+        """Set up Python module-scope variables for dependent packages.
+
+        Called before the install() method of dependents.
+
+        Default implementation does nothing, but this can be
+        overridden by an extendable package to set up the module of
+        its extensions. This is useful if there are some common steps
+        to installing all extensions for a certain package.
+
+        Example :
+
+            1. Extensions often need to invoke the `python`
+               interpreter from the Python installation being
+               extended.  This routine can put a 'python' Executable
+               object in the module scope for the extension package to
+               simplify extension installs.
+
+            2. MPI compilers could set some variables in the
+               dependent's scope that point to `mpicc`, `mpicxx`,
+               etc., allowing them to be called by common names
+               regardless of which MPI is used.
+
+            3. BLAS/LAPACK implementations can set some variables
+               indicating the path to their libraries, since these
+               paths differ by BLAS/LAPACK implementation.
+
+        Args:
+
+            module (module): The Python `module` object of the
+                dependent package. Packages can use this to set
+                module-scope variables for the dependent to use.
+
+            dependent_spec (Spec): The spec of the dependent package
+                about to be built. This allows the extendee (self) to
+                query the dependent's state.  Note that *this*
+                package's spec is available as `self.spec`.
+
+        This is useful if there are some common steps to installing
+        all extensions for a certain package.
+
+        """
+        pass
+
     def install(self, spec, prefix):
         """Package implementations override this with their own build configuration."""
         raise InstallError("Package %s provides no install method!" % self.name)
 
-
     def do_uninstall(self, force=False):
         if not self.installed:
             raise InstallError(str(self.spec) + " is not installed.")
@@ -1024,7 +1138,7 @@ def do_uninstall(self, force=False):
         # Uninstalling in Spack only requires removing the prefix.
         self.remove_prefix()
         spack.installed_db.remove(self.spec)
-        tty.msg("Successfully uninstalled %s." % self.spec.short_spec)
+        tty.msg("Successfully uninstalled %s" % self.spec.short_spec)
 
         # Once everything else is done, run post install hooks
         spack.hooks.post_uninstall(self)
@@ -1071,7 +1185,7 @@ def do_activate(self, force=False):
         self.extendee_spec.package.activate(self, **self.extendee_args)
 
         spack.install_layout.add_extension(self.extendee_spec, self.spec)
-        tty.msg("Activated extension %s for %s."
+        tty.msg("Activated extension %s for %s"
                 % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@")))
 
 
@@ -1123,7 +1237,7 @@ def do_deactivate(self, **kwargs):
         if self.activated:
             spack.install_layout.remove_extension(self.extendee_spec, self.spec)
 
-        tty.msg("Deactivated extension %s for %s."
+        tty.msg("Deactivated extension %s for %s"
                 % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@")))
 
 
@@ -1148,6 +1262,12 @@ def do_restage(self):
         """Reverts expanded/checked out source to a pristine state."""
         self.stage.restage()
 
+
+    def do_clean(self):
+        """Removes the package's build stage and source tarball."""
+        self.stage.destroy()
+
+
     def format_doc(self, **kwargs):
         """Wrap doc string at 72 characters and format nicely"""
         indent = kwargs.get('indent', 0)
@@ -1202,8 +1322,29 @@ def rpath(self):
 
     @property
     def rpath_args(self):
-        """Get the rpath args as a string, with -Wl,-rpath= for each element."""
+        """Get the rpath args as a string, with -Wl,-rpath, for each element."""
-        return " ".join("-Wl,-rpath=%s" % p for p in self.rpath)
+        return " ".join("-Wl,-rpath,%s" % p for p in self.rpath)
+
+
+def install_dependency_symlinks(pkg, spec, prefix):
+    """Execute a dummy install and flatten dependencies"""
+    flatten_dependencies(spec, prefix)
+
+def flatten_dependencies(spec, flat_dir):
+    """Make each dependency of spec present in dir via symlink."""
+    for dep in spec.traverse(root=False):
+        name = dep.name
+
+        dep_path = spack.install_layout.path_for_spec(dep)
+        dep_files = LinkTree(dep_path)
+
+        os.mkdir(flat_dir+'/'+name)
+
+        conflict = dep_files.find_conflict(flat_dir+'/'+name)
+        if conflict:
+            raise DependencyConflictError(conflict)
+
+        dep_files.merge(flat_dir+'/'+name)
 
 
 def validate_package_url(url_string):
@@ -1293,6 +1434,10 @@ def __init__(self, message, long_msg=None):
         super(InstallError, self).__init__(message, long_msg)
 
 
+class ExternalPackageError(InstallError):
+    """Raised by install() when a package is only for external use."""
+
+
 class PackageStillNeededError(InstallError):
     """Raised when package is still needed by another on uninstall."""
     def __init__(self, spec, dependents):
@@ -1312,7 +1457,7 @@ class PackageVersionError(PackageError):
     """Raised when a version URL cannot automatically be determined."""
     def __init__(self, version):
         super(PackageVersionError, self).__init__(
-            "Cannot determine a URL automatically for version %s." % version,
+            "Cannot determine a URL automatically for version %s" % version,
             "Please provide a url for this version in the package.py file.")
 
 
@@ -1343,3 +1488,11 @@ def __init__(self, path):
 class ActivationError(ExtensionError):
     def __init__(self, msg, long_msg=None):
         super(ActivationError, self).__init__(msg, long_msg)
+
+
+class DependencyConflictError(spack.error.SpackError):
+    """Raised when the dependencies cannot be flattened as asked for."""
+    def __init__(self, conflict):
+        super(DependencyConflictError, self).__init__(
+            "%s conflicts with another file in the flattened directory." %(
+                conflict))

lib/spack/spack/preferred_packages.py

@@ -0,0 +1,175 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import spack
+from spack.version import *
+
+class PreferredPackages(object):
+    _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] } # Arbitrary, but consistent
+
+    def __init__(self):
+        self.preferred = spack.config.get_config('packages')
+        self._spec_for_pkgname_cache = {}
+
+    # Given a package name, sort component (e.g, version, compiler, ...), and
+    # a second_key (used by providers), return the list
+    def _order_for_package(self, pkgname, component, second_key, test_all=True):
+        pkglist = [pkgname]
+        if test_all:
+            pkglist.append('all')
+        for pkg in pkglist:
+            order = self.preferred.get(pkg, {}).get(component, {})
+            if type(order) is dict:
+                order = order.get(second_key, {})
+            if not order:
+                continue
+            return [str(s).strip() for s in order]
+        return []
+
+
+    # A generic sorting function. Given a package name and sort
+    # component, return less-than-0, 0, or greater-than-0 if
+    # a is respectively less-than, equal to, or greater than b.
+    def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key):
+        if a is None:
+            return -1
+        if b is None:
+            return 1
+        orderlist = self._order_for_package(pkgname, component, second_key)
+        a_in_list = str(a) in orderlist
+        b_in_list = str(b) in orderlist
+        if a_in_list and not b_in_list:
+            return -1
+        elif b_in_list and not a_in_list:
+            return 1
+
+        cmp_a = None
+        cmp_b = None
+        reverse = None
+        if not a_in_list and not b_in_list:
+            cmp_a = a
+            cmp_b = b
+            reverse = -1 if reverse_natural_compare else 1
+        else:
+            cmp_a = orderlist.index(str(a))
+            cmp_b = orderlist.index(str(b))
+            reverse = 1
+
+        if cmp_a < cmp_b:
+            return -1 * reverse
+        elif cmp_a > cmp_b:
+            return 1 * reverse
+        else:
+            return 0
+
+
+    # A sorting function for specs.  Similar to component_compare, but
+    # a and b are considered to match entries in the sorting list if they
+    # satisfy the list component.
+    def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key):
+        if not a or not a.concrete:
+            return -1
+        if not b or not b.concrete:
+            return 1
+        specs = self._spec_for_pkgname(pkgname, component, second_key)
+        a_index = None
+        b_index = None
+        reverse = -1 if reverse_natural_compare else 1
+        for i, cspec in enumerate(specs):
+            if a_index == None and (cspec.satisfies(a) or a.satisfies(cspec)):
+                a_index = i
+                if b_index:
+                    break
+            if b_index == None and (cspec.satisfies(b) or b.satisfies(cspec)):
+                b_index = i
+                if a_index:
+                    break
+
+        if   a_index != None and b_index == None: return -1
+        elif a_index == None and b_index != None: return 1
+        elif a_index != None and b_index == a_index: return -1 * cmp(a, b)
+        elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index)
+        else: return cmp(a, b) * reverse
+
+
+
+    # Given a sort order specified by the pkgname/component/second_key, return
+    # a list of CompilerSpecs, VersionLists, or Specs for that sorting list.
+    def _spec_for_pkgname(self, pkgname, component, second_key):
+        key = (pkgname, component, second_key)
+        if not key in self._spec_for_pkgname_cache:
+            pkglist = self._order_for_package(pkgname, component, second_key)
+            if not pkglist:
+                if component in self._default_order:
+                    pkglist = self._default_order[component]
+            if component == 'compiler':
+                self._spec_for_pkgname_cache[key] = [spack.spec.CompilerSpec(s) for s in pkglist]
+            elif component == 'version':
+                self._spec_for_pkgname_cache[key] = [VersionList(s) for s in pkglist]
+            else:
+                self._spec_for_pkgname_cache[key] = [spack.spec.Spec(s) for s in pkglist]
+        return self._spec_for_pkgname_cache[key]
+
+
+    def provider_compare(self, pkgname, provider_str, a, b):
+        """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or
+           greater-than b. A and b are possible implementations of provider_str.
+           One provider is less-than another if it is preferred over the other.
+           For example, provider_compare('scorep', 'mpi', 'mvapich', 'openmpi') would return -1 if
+           mvapich should be preferred over openmpi for scorep."""
+        return self._spec_compare(pkgname, 'providers', a, b, False, provider_str)
+
+
+    def spec_has_preferred_provider(self, pkgname, provider_str):
+        """Return True iff the named package has a list of preferred provider"""
+        return bool(self._order_for_package(pkgname, 'providers', provider_str, False))
+
+
+    def version_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if version a of pkgname is
+           respecively less-than, equal-to, or greater-than version b of pkgname.
+           One version is less-than another if it is preferred over the other."""
+        return self._spec_compare(pkgname, 'version', a, b, True, None)
+
+
+    def variant_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if variant a of pkgname is
+           respecively less-than, equal-to, or greater-than variant b of pkgname.
+           One variant is less-than another if it is preferred over the other."""
+        return self._component_compare(pkgname, 'variant', a, b, False, None)
+
+
+    def architecture_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is
+           respecively less-than, equal-to, or greater-than architecture b of pkgname.
+           One architecture is less-than another if it is preferred over the other."""
+        return self._component_compare(pkgname, 'architecture', a, b, False, None)
+
+
+    def compiler_compare(self, pkgname, a, b):
+        """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is
+           respecively less-than, equal-to, or greater-than compiler b of pkgname.
+           One compiler is less-than another if it is preferred over the other."""
+        return self._spec_compare(pkgname, 'compiler', a, b, False, None)

lib/spack/spack/repository.py

@@ -156,7 +156,7 @@ def _add(self, repo):
 
         if repo.namespace in self.by_namespace:
             raise DuplicateRepoError(
-                "Package repos '%s' and '%s' both provide namespace %s."
+                "Package repos '%s' and '%s' both provide namespace %s"
                 % (repo.root, self.by_namespace[repo.namespace].root, repo.namespace))
 
         # Add repo to the pkg indexes
@@ -316,6 +316,11 @@ def get(self, spec, new=False):
         return self.repo_for_pkg(spec).get(spec)
 
 
+    def get_pkg_class(self, pkg_name):
+        """Find a class for the spec's package and return the class object."""
+        return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name)
+
+
     @_autospec
     def dump_provenance(self, spec, path):
         """Dump provenance information for a spec to a particular path.
@@ -545,12 +550,12 @@ def get(self, spec, new=False):
             raise UnknownPackageError(spec.name)
 
         if spec.namespace and spec.namespace != self.namespace:
-            raise UnknownPackageError("Repository %s does not contain package %s."
+            raise UnknownPackageError("Repository %s does not contain package %s"
                                       % (self.namespace, spec.fullname))
 
         key = hash(spec)
         if new or key not in self._instances:
-            package_class = self._get_pkg_class(spec.name)
+            package_class = self.get_pkg_class(spec.name)
             try:
                 copy = spec.copy() # defensive copy.  Package owns its spec.
                 self._instances[key] = package_class(copy)
@@ -715,7 +720,7 @@ def _get_pkg_module(self, pkg_name):
         return self._modules[pkg_name]
 
 
-    def _get_pkg_class(self, pkg_name):
+    def get_pkg_class(self, pkg_name):
         """Get the class for the package out of its module.
 
         First loads (or fetches from cache) a module for the
@@ -825,7 +830,7 @@ class UnknownPackageError(PackageLoadError):
     def __init__(self, name, repo=None):
         msg = None
         if repo:
-            msg = "Package %s not found in repository %s." % (name, repo)
+            msg = "Package %s not found in repository %s" % (name, repo)
         else:
             msg = "Package %s not found." % name
         super(UnknownPackageError, self).__init__(msg)

lib/spack/spack/spec.py

@@ -353,7 +353,7 @@ def constrain(self, other):
     @property
     def concrete(self):
         return self.spec._concrete or all(
-            v in self for v in self.spec.package.variants)
+            v in self for v in self.spec.package_class.variants)
 
 
     def copy(self):
@@ -421,6 +421,9 @@ def __init__(self, spec_like, *dep_like, **kwargs):
         self._normal   = kwargs.get('normal', False)
         self._concrete = kwargs.get('concrete', False)
 
+        # Allow a spec to be constructed with an external path.
+        self.external  = kwargs.get('external', None)
+
         # This allows users to construct a spec DAG with literals.
         # Note that given two specs a and b, Spec(a) copies a, but
         # Spec(a, b) will copy a but just add b as a dep.
@@ -497,6 +500,14 @@ def package(self):
         return spack.repo.get(self)
 
 
+    @property
+    def package_class(self):
+        """Internal package call gets only the class object for a package.
+           Use this to just get package metadata.
+        """
+        return spack.repo.get_pkg_class(self.name)
+
+
     @property
     def virtual(self):
         """Right now, a spec is virtual if no package exists with its name.
@@ -770,7 +781,6 @@ def _concretize_helper(self, presets=None, visited=None):
             # Concretize virtual dependencies last.  Because they're added
             # to presets below, their constraints will all be merged, but we'll
             # still need to select a concrete package later.
-            if not self.virtual:
             changed |= any(
                 (spack.concretizer.concretize_architecture(self),
                  spack.concretizer.concretize_compiler(self),
@@ -786,10 +796,32 @@ def _replace_with(self, concrete):
         """Replace this virtual spec with a concrete spec."""
         assert(self.virtual)
         for name, dependent in self.dependents.items():
+            # remove self from all dependents.
             del dependent.dependencies[self.name]
+
+            # add the replacement, unless it is already a dep of dependent.
+            if concrete.name not in dependent.dependencies:
                 dependent._add_dependency(concrete)
 
 
+    def _replace_node(self, replacement):
+        """Replace this spec with another.
+
+        Connects all dependents of this spec to its replacement, and
+        disconnects this spec from any dependencies it has. New spec
+        will have any dependencies the replacement had, and may need
+        to be normalized.
+
+        """
+        for name, dependent in self.dependents.items():
+            del dependent.dependencies[self.name]
+            dependent._add_dependency(replacement)
+
+        for name, dep in self.dependencies.items():
+            del dep.dependents[self.name]
+            del self.dependencies[dep.name]
+
+
     def _expand_virtual_packages(self):
         """Find virtual packages in this spec, replace them with providers,
            and normalize again to include the provider's (potentially virtual)
@@ -807,22 +839,81 @@ def _expand_virtual_packages(self):
               this are infrequent, but should implement this before it is
               a problem.
         """
-        changed = False
+        # Make an index of stuff this spec already provides
-        while True:
+        self_index = ProviderIndex(self.traverse(), restrict=True)
-            virtuals =[v for v in self.traverse() if v.virtual]
-            if not virtuals:
-                return changed
 
-            for spec in virtuals:
+        changed = False
-                providers = spack.repo.providers_for(spec)
+        done = False
-                concrete = spack.concretizer.choose_provider(spec, providers)
+        while not done:
-                concrete = concrete.copy()
+            done = True
-                spec._replace_with(concrete)
+            for spec in list(self.traverse()):
+                replacement = None
+                if spec.virtual:
+                    replacement = self._find_provider(spec, self_index)
+                    if replacement:
+                        # TODO: may break if in-place on self but
+                        # shouldn't happen if root is traversed first.
+                        spec._replace_with(replacement)
+                        done=False
+                        break
+
+                if not replacement:
+                    # Get a list of possible replacements in order of preference.
+                    candidates = spack.concretizer.choose_virtual_or_external(spec)
+
+                    # Try the replacements in order, skipping any that cause
+                    # satisfiability problems.
+                    for replacement in candidates:
+                        if replacement is spec:
+                            break
+
+                        # Replace spec with the candidate and normalize
+                        copy = self.copy()
+                        copy[spec.name]._dup(replacement.copy(deps=False))
+
+                        try:
+                            # If there are duplicate providers or duplicate provider
+                            # deps, consolidate them and merge constraints.
+                            copy.normalize(force=True)
+                            break
+                        except SpecError as e:
+                            # On error, we'll try the next replacement.
+                            continue
+
+                # If replacement is external then trim the dependencies
+                if replacement.external:
+                    if (spec.dependencies):
+                        changed = True
+                        spec.dependencies = DependencyMap()
+                    replacement.dependencies = DependencyMap()
+
+                # TODO: could this and the stuff in _dup be cleaned up?
+                def feq(cfield, sfield):
+                    return (not cfield) or (cfield == sfield)
+
+                if replacement is spec or (feq(replacement.name, spec.name) and
+                    feq(replacement.versions, spec.versions) and
+                    feq(replacement.compiler, spec.compiler) and
+                    feq(replacement.architecture, spec.architecture) and
+                    feq(replacement.dependencies, spec.dependencies) and
+                    feq(replacement.variants, spec.variants) and
+                    feq(replacement.external, spec.external)):
+                    continue
+
+                # Refine this spec to the candidate. This uses
+                # replace_with AND dup so that it can work in
+                # place. TODO: make this more efficient.
+                if spec.virtual:
+                    spec._replace_with(replacement)
+                    changed = True
+                if spec._dup(replacement, deps=False, cleardeps=False):
                     changed = True
 
-            # If there are duplicate providers or duplicate provider deps, this
+                self_index.update(spec)
-            # consolidates them and merge constraints.
+                done=False
-            changed |= self.normalize(force=True)
+                break
+
+        return changed
 
 
     def concretize(self):
@@ -837,6 +928,7 @@ def concretize(self):
            with requirements of its pacakges.  See flatten() and normalize() for
            more details on this.
         """
+
         if self._concrete:
             return
 
@@ -844,7 +936,7 @@ def concretize(self):
         force = False
 
         while changed:
-            changes = (self.normalize(force=force),
+            changes = (self.normalize(force),
                        self._expand_virtual_packages(),
                        self._concretize_helper())
             changed = any(changes)
@@ -1012,17 +1104,14 @@ def _merge_dependency(self, dep, visited, spec_deps, provider_index):
         """
         changed = False
 
-        # If it's a virtual dependency, try to find a provider and
+        # If it's a virtual dependency, try to find an existing
-        # merge that.
+        # provider in the spec, and merge that.
         if dep.virtual:
             visited.add(dep.name)
             provider = self._find_provider(dep, provider_index)
             if provider:
                 dep = provider
-
         else:
-            # if it's a real dependency, check whether it provides
-            # something already required in the spec.
             index = ProviderIndex([dep], restrict=True)
             for vspec in (v for v in spec_deps.values() if v.virtual):
                 if index.providers_for(vspec):
@@ -1069,7 +1158,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index):
 
         # if we descend into a virtual spec, there's nothing more
         # to normalize.  Concretize will finish resolving it later.
-        if self.virtual:
+        if self.virtual or self.external:
             return False
 
         # Combine constraints from package deps with constraints from
@@ -1119,13 +1208,14 @@ def normalize(self, force=False):
         # Get all the dependencies into one DependencyMap
         spec_deps = self.flat_dependencies(copy=False)
 
-        # Initialize index of virtual dependency providers
+        # Initialize index of virtual dependency providers if
-        index = ProviderIndex(spec_deps.values(), restrict=True)
+        # concretize didn't pass us one already
+        provider_index = ProviderIndex(spec_deps.values(), restrict=True)
 
         # traverse the package DAG and fill out dependencies according
         # to package files & their 'when' specs
         visited = set()
-        any_change = self._normalize_helper(visited, spec_deps, index)
+        any_change = self._normalize_helper(visited, spec_deps, provider_index)
 
         # If there are deps specified but not visited, they're not
         # actually deps of this package.  Raise an error.
@@ -1163,7 +1253,7 @@ def validate_names(self):
 
             # Ensure that variants all exist.
             for vname, variant in spec.variants.items():
-                if vname not in spec.package.variants:
+                if vname not in spec.package_class.variants:
                     raise UnknownVariantError(spec.name, vname)
 
 
@@ -1404,15 +1494,25 @@ def _dup(self, other, **kwargs):
                Whether deps should be copied too.  Set to false to copy a
                spec but not its dependencies.
         """
+        # We don't count dependencies as changes here
+        changed = True
+        if hasattr(self, 'name'):
+            changed = (self.name != other.name and self.versions != other.versions and
+                       self.architecture != other.architecture and self.compiler != other.compiler and
+                       self.variants != other.variants and self._normal != other._normal and
+                       self.concrete != other.concrete and self.external != other.external)
+
         # Local node attributes get copied first.
         self.name = other.name
         self.versions = other.versions.copy()
         self.architecture = other.architecture
         self.compiler = other.compiler.copy() if other.compiler else None
+        if kwargs.get('cleardeps', True):
             self.dependents = DependencyMap()
             self.dependencies = DependencyMap()
         self.variants = other.variants.copy()
         self.variants.spec = self
+        self.external = other.external
         self.namespace = other.namespace
 
         # If we copy dependencies, preserve DAG structure in the new spec
@@ -1431,6 +1531,8 @@ def _dup(self, other, **kwargs):
         # Since we preserved structure, we can copy _normal safely.
         self._normal = other._normal
         self._concrete = other._concrete
+        self.external = other.external
+        return changed
 
 
     def copy(self, **kwargs):
@@ -1571,14 +1673,28 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs):
 
                $_   Package name
                $.   Full package name (with namespace)
-               $@   Version
+               $@   Version with '@' prefix
-               $%   Compiler
+               $%   Compiler with '%' prefix
-               $%@  Compiler & compiler version
+               $%@  Compiler with '%' prefix & compiler version with '@' prefix
                $+   Options
-               $=   Architecture
+               $=   Architecture with '=' prefix
-               $#   7-char prefix of DAG hash
+               $#   7-char prefix of DAG hash with '-' prefix
                $$   $
 
+               You can also use full-string versions, which leave off the prefixes:
+
+               ${PACKAGE}       Package name
+               ${VERSION}       Version
+               ${COMPILER}      Full compiler string
+               ${COMPILERNAME}  Compiler name
+               ${COMPILERVER}   Compiler version
+               ${OPTIONS}       Options
+               ${ARCHITECTURE}  Architecture
+               ${SHA1}          Dependencies 8-char sha1 prefix
+
+               ${SPACK_ROOT}    The spack root directory
+               ${SPACK_INSTALL} The default spack install directory, ${SPACK_PREFIX}/opt
+
            Optionally you can provide a width, e.g. $20_ for a 20-wide name.
            Like printf, you can provide '-' for left justification, e.g.
            $-20_ for a left-justified name.
@@ -1594,7 +1710,8 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs):
         color    = kwargs.get('color', False)
         length = len(format_string)
         out = StringIO()
-        escape = compiler = False
+        named = escape = compiler = False
+        named_str = fmt = ''
 
         def write(s, c):
             if color:
@@ -1636,9 +1753,12 @@ def write(s, c):
                 elif c == '#':
                     out.write('-' + fmt % (self.dag_hash(7)))
                 elif c == '$':
-                    if fmt != '':
+                    if fmt != '%s':
                         raise ValueError("Can't use format width with $$.")
                     out.write('$')
+                elif c == '{':
+                    named = True
+                    named_str = ''
                 escape = False
 
             elif compiler:
@@ -1652,6 +1772,43 @@ def write(s, c):
                     out.write(c)
                 compiler = False
 
+            elif named:
+                if not c == '}':
+                    if i == length - 1:
+                        raise ValueError("Error: unterminated ${ in format: '%s'"
+                                         % format_string)
+                    named_str += c
+                    continue;
+                if named_str == 'PACKAGE':
+                    write(fmt % self.name, '@')
+                if named_str == 'VERSION':
+                    if self.versions and self.versions != _any_version:
+                        write(fmt % str(self.versions), '@')
+                elif named_str == 'COMPILER':
+                    if self.compiler:
+                        write(fmt % self.compiler, '%')
+                elif named_str == 'COMPILERNAME':
+                    if self.compiler:
+                        write(fmt % self.compiler.name, '%')
+                elif named_str == 'COMPILERVER':
+                    if self.compiler:
+                        write(fmt % self.compiler.versions, '%')
+                elif named_str == 'OPTIONS':
+                    if self.variants:
+                        write(fmt % str(self.variants), '+')
+                elif named_str == 'ARCHITECTURE':
+                    if self.architecture:
+                        write(fmt % str(self.architecture), '=')
+                elif named_str == 'SHA1':
+                    if self.dependencies:
+                        out.write(fmt % str(self.dag_hash(7)))
+                elif named_str == 'SPACK_ROOT':
+                    out.write(fmt % spack.prefix)
+                elif named_str == 'SPACK_INSTALL':
+                    out.write(fmt % spack.install_path)
+
+                named = False
+
             elif c == '$':
                 escape = True
                 if i == length - 1:
@@ -1782,6 +1939,7 @@ def spec(self):
         spec.variants = VariantMap(spec)
         spec.architecture = None
         spec.compiler = None
+        spec.external = None
         spec.dependents   = DependencyMap()
         spec.dependencies = DependencyMap()
         spec.namespace = spec_namespace

lib/spack/spack/stage.py

@@ -42,29 +42,54 @@
 
 
 class Stage(object):
-    """
+    """Manages a temporary stage directory for building.
-    A Stage object is a context manager that handles a directory where some source code is downloaded and built
+
-    before being installed.  It handles fetching the source code, either as an archive to be expanded or by checking
+    A Stage object is a context manager that handles a directory where
-    it out of a repository.  A stage's lifecycle looks like this:
+    some source code is downloaded and built before being installed.
+    It handles fetching the source code, either as an archive to be
+    expanded or by checking it out of a repository.  A stage's
+    lifecycle looks like this:
 
     ```
     with Stage() as stage:      # Context manager creates and destroys the stage directory
-        fetch()  # Fetch a source archive into the stage.
+        stage.fetch()           # Fetch a source archive into the stage.
-        expand_archive()  # Expand the source archive.
+        stage.expand_archive()  # Expand the source archive.
-        <install>  # Build and install the archive.  This is handled by the Package class.
+        <install>               # Build and install the archive. (handled by user of Stage)
     ```
 
-    If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory.
+    When used as a context manager, the stage is automatically
-    Otherwise, stages are created directly in spack.stage_path.
+    destroyed if no exception is raised by the context. If an
+    excpetion is raised, the stage is left in the filesystem and NOT
+    destroyed, for potential reuse later.
 
-    There are two kinds of stages: named and unnamed.  Named stages can persist between runs of spack, e.g. if you
+    You can also use the stage's create/destroy functions manually,
-    fetched a tarball but didn't finish building it, you won't have to fetch it again.
+    like this:
 
-    Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for
+    ```
-    only one run of spack.
+    stage = Stage()
+    try:
+        stage.create()          # Explicitly create the stage directory.
+        stage.fetch()           # Fetch a source archive into the stage.
+        stage.expand_archive()  # Expand the source archive.
+        <install>               # Build and install the archive. (handled by user of Stage)
+    finally:
+        stage.destroy()         # Explicitly destroy the stage directory.
+    ```
+
+    If spack.use_tmp_stage is True, spack will attempt to create
+    stages in a tmp directory.  Otherwise, stages are created directly
+    in spack.stage_path.
+
+    There are two kinds of stages: named and unnamed.  Named stages
+    can persist between runs of spack, e.g. if you fetched a tarball
+    but didn't finish building it, you won't have to fetch it again.
+
+    Unnamed stages are created using standard mkdtemp mechanisms or
+    similar, and are intended to persist for only one run of spack.
     """
 
-    def __init__(self, url_or_fetch_strategy, **kwargs):
+    def __init__(self, url_or_fetch_strategy,
+                 name=None, mirror_path=None, keep=False):
         """Create a stage object.
            Parameters:
              url_or_fetch_strategy
@@ -76,6 +101,17 @@ def __init__(self, url_or_fetch_strategy, **kwargs):
                  and will persist between runs (or if you construct another
                  stage object later).  If name is not provided, then this
                  stage will be given a unique name automatically.
+
+             mirror_path
+                 If provided, Stage will search Spack's mirrors for
+                 this archive at the mirror_path, before using the
+                 default fetch strategy.
+
+             keep
+                 By default, when used as a context manager, the Stage
+                 is deleted on exit when no exceptions are raised.
+                 Pass True to keep the stage intact even if no
+                 exceptions are raised.
         """
         # TODO: fetch/stage coupling needs to be reworked -- the logic
         # TODO: here is convoluted and not modular enough.
@@ -91,15 +127,19 @@ def __init__(self, url_or_fetch_strategy, **kwargs):
 
         # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name
         # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root
-        self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names())
+        self.name = name
-        self.mirror_path = kwargs.get('mirror_path')
+        if name is None:
+            self.name = STAGE_PREFIX + next(tempfile._get_candidate_names())
+        self.mirror_path = mirror_path
         self.tmp_root = find_tmp_root()
 
         # Try to construct here a temporary name for the stage directory
         # If this is a named stage, then construct a named path.
         self.path = join_path(spack.stage_path, self.name)
+
         # Flag to decide whether to delete the stage folder on exit or not
-        self.delete_on_exit = True
+        self.keep = keep
+
 
     def __enter__(self):
         """
@@ -111,6 +151,7 @@ def __enter__(self):
         self.create()
         return self
 
+
     def __exit__(self, exc_type, exc_val, exc_tb):
         """
         Exiting from a stage context will delete the stage directory unless:
@@ -125,11 +166,11 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         Returns:
             Boolean
         """
-        self.delete_on_exit = False if exc_type is not None else self.delete_on_exit
+        # Delete when there are no exceptions, unless asked to keep.
-
+        if exc_type is None and not self.keep:
-        if self.delete_on_exit:
             self.destroy()
 
+
     def _need_to_create_path(self):
         """Makes sure nothing weird has happened since the last time we
            looked at path.  Returns True if path already exists and is ok.
@@ -150,8 +191,8 @@ def _need_to_create_path(self):
             real_tmp = os.path.realpath(self.tmp_root)
 
             if spack.use_tmp_stage:
-                # If we're using a tmp dir, it's a link, and it points at the right spot,
+                # If we're using a tmp dir, it's a link, and it points at the
-                # then keep it.
+                # right spot, then keep it.
                 if (real_path.startswith(real_tmp) and os.path.exists(real_path)):
                     return False
                 else:
@@ -184,13 +225,22 @@ def archive_file(self):
 
     @property
     def source_path(self):
-        """Returns the path to the expanded/checked out source code
+        """Returns the path to the expanded/checked out source code.
-           within this fetch strategy's path.
 
-           This assumes nothing else is going ot be put in the
+        To find the source code, this method searches for the first
-           FetchStrategy's path.  It searches for the first
+        subdirectory of the stage that it can find, and returns it.
-           subdirectory of the path it can find, then returns that.
+        This assumes nothing besides the archive file will be in the
+        stage path, but it has the advantage that we don't need to
+        know the name of the archive or its contents.
+
+        If the fetch strategy is not supposed to expand the downloaded
+        file, it will just return the stage path. If the archive needs
+        to be expanded, it will return None when no archive is found.
         """
+        if isinstance(self.fetcher, fs.URLFetchStrategy):
+            if not self.fetcher.expand_archive:
+                return self.path
+
         for p in [os.path.join(self.path, f) for f in os.listdir(self.path)]:
             if os.path.isdir(p):
                 return p
@@ -201,7 +251,7 @@ def chdir(self):
         if os.path.isdir(self.path):
             os.chdir(self.path)
         else:
-            tty.die("Setup failed: no such directory: " + self.path)
+            raise ChdirError("Setup failed: no such directory: " + self.path)
 
     def fetch(self, mirror_only=False):
         """Downloads an archive or checks out code from a repository."""
@@ -276,9 +326,9 @@ def expand_archive(self):
         archive_dir = self.source_path
         if not archive_dir:
             self.fetcher.expand()
-            tty.msg("Created stage in %s." % self.path)
+            tty.msg("Created stage in %s" % self.path)
         else:
-            tty.msg("Already staged %s in %s." % (self.name, self.path))
+            tty.msg("Already staged %s in %s" % (self.name, self.path))
 
     def chdir_to_source(self):
         """Changes directory to the expanded archive directory.
@@ -302,11 +352,14 @@ def create(self):
         """
         Creates the stage directory
 
-        If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise
+        If self.tmp_root evaluates to False, the stage directory is
-        this will attempt to create a stage in a temporary directory and link it into spack.stage_path.
+        created directly under spack.stage_path, otherwise this will
+        attempt to create a stage in a temporary directory and link it
+        into spack.stage_path.
 
-        Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location
+        Spack will use the first writable location in spack.tmp_dirs
-        in tmp_dirs, fall back to making the stage inside spack.stage_path.
+        to create a stage. If there is no valid location in tmp_dirs,
+        fall back to making the stage inside spack.stage_path.
         """
         # Create the top-level stage directory
         mkdirp(spack.stage_path)
@@ -323,9 +376,7 @@ def create(self):
         ensure_access(self.path)
 
     def destroy(self):
-        """
+        """Removes this stage directory."""
-        Removes this stage directory
-        """
         remove_linked_tree(self.path)
 
         # Make sure we don't end up in a removed directory
@@ -370,13 +421,28 @@ def expand_archive(self):
                 shutil.move(source_path, destination_path)
 
 
-@pattern.composite(method_list=['fetch', 'check', 'expand_archive', 'restage', 'destroy'])
+@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive',  'restage', 'destroy'])
 class StageComposite:
     """
     Composite for Stage type objects. The first item in this composite is considered to be the root package, and
     operations that return a value are forwarded to it.
     """
+    #
+    # __enter__ and __exit__ delegate to all stages in the composite.
+    #
+    def __enter__(self):
+        for item in self:
+            item.__enter__()
+        return self
 
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        for item in reversed(self):
+            item.keep = getattr(self, 'keep', False)
+            item.__exit__(exc_type, exc_val, exc_tb)
+
+    #
+    # Below functions act only on the *first* stage in the composite.
+    #
     @property
     def source_path(self):
         return self[0].source_path
@@ -385,18 +451,13 @@ def source_path(self):
     def path(self):
         return self[0].path
 
-    def __enter__(self):
-        for item in self:
-            item.__enter__()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        for item in reversed(self):
-            item.__exit__(exc_type, exc_val, exc_tb)
-
     def chdir_to_source(self):
         return self[0].chdir_to_source()
 
+    @property
+    def archive_file(self):
+        return self[0].archive_file
+
 
 class DIYStage(object):
     """Simple class that allows any directory to be a spack stage."""
@@ -410,12 +471,16 @@ def chdir(self):
         if os.path.isdir(self.path):
             os.chdir(self.path)
         else:
-            tty.die("Setup failed: no such directory: " + self.path)
+            raise ChdirError("Setup failed: no such directory: " + self.path)
+
+    # DIY stages do nothing as context managers.
+    def __enter__(self): pass
+    def __exit__(self, exc_type, exc_val, exc_tb): pass
 
     def chdir_to_source(self):
         self.chdir()
 
-    def fetch(self):
+    def fetch(self, mirror_only):
         tty.msg("No need to fetch for DIY.")
 
     def check(self):
@@ -472,19 +537,15 @@ def find_tmp_root():
 
 
 class StageError(spack.error.SpackError):
-    def __init__(self, message, long_message=None):
+    """"Superclass for all errors encountered during staging."""
-        super(self, StageError).__init__(message, long_message)
 
 
 class RestageError(StageError):
-    def __init__(self, message, long_msg=None):
+    """"Error encountered during restaging."""
-        super(RestageError, self).__init__(message, long_msg)
 
 
 class ChdirError(StageError):
-    def __init__(self, message, long_msg=None):
+    """Raised when Spack can't change directories."""
-        super(ChdirError, self).__init__(message, long_msg)
-
 
 # Keep this in namespace for convenience
 FailedDownloadError = fs.FailedDownloadError

lib/spack/spack/test/__init__.py

@@ -65,7 +65,9 @@
               'lock',
               'database',
               'namespace_trie',
-              'yaml']
+              'yaml',
+              'sbang',
+              'environment']
 
 
 def list_tests():

lib/spack/spack/test/cc.py

@@ -39,11 +39,11 @@
     'arg1',
     '-Wl,--start-group',
     'arg2',
-    '-Wl,-rpath=/first/rpath', 'arg3', '-Wl,-rpath', '-Wl,/second/rpath',
+    '-Wl,-rpath,/first/rpath', 'arg3', '-Wl,-rpath', '-Wl,/second/rpath',
     '-llib1', '-llib2',
     'arg4',
     '-Wl,--end-group',
-    '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath=/fourth/rpath',
+    '-Xlinker', '-rpath', '-Xlinker', '/third/rpath', '-Xlinker', '-rpath', '-Xlinker', '/fourth/rpath',
     '-llib3', '-llib4',
     'arg5', 'arg6']
 
@@ -95,13 +95,13 @@ def test_cpp_mode(self):
     def test_ccld_mode(self):
         self.check_cc('dump-mode', [], "ccld")
         self.check_cc('dump-mode', ['foo.c', '-o', 'foo'], "ccld")
-        self.check_cc('dump-mode', ['foo.c', '-o', 'foo', '-Wl,-rpath=foo'], "ccld")
+        self.check_cc('dump-mode', ['foo.c', '-o', 'foo', '-Wl,-rpath,foo'], "ccld")
-        self.check_cc('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath=foo'], "ccld")
+        self.check_cc('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ccld")
 
 
     def test_ld_mode(self):
         self.check_ld('dump-mode', [], "ld")
-        self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath=foo'], "ld")
+        self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ld")
 
 
     def test_includes(self):

lib/spack/spack/test/concretize.py

@@ -24,6 +24,8 @@
 ##############################################################################
 import spack
 from spack.spec import Spec, CompilerSpec
+from spack.version import ver
+from spack.concretize import find_spec
 from spack.test.mock_packages_test import *
 
 class ConcretizeTest(MockPackagesTest):
@@ -76,6 +78,14 @@ def test_concretize_variant(self):
         self.check_concretize('mpich')
 
 
+    def test_concretize_preferred_version(self):
+        spec = self.check_concretize('python')
+        self.assertEqual(spec.versions, ver('2.7.11'))
+
+        spec = self.check_concretize('python@3.5.1')
+        self.assertEqual(spec.versions, ver('3.5.1'))
+
+
     def test_concretize_with_virtual(self):
         self.check_concretize('mpileaks ^mpi')
         self.check_concretize('mpileaks ^mpi@:1.1')
@@ -141,6 +151,34 @@ def test_concretize_with_provides_when(self):
                                 for spec in spack.repo.providers_for('mpi@3')))
 
 
+    def test_concretize_two_virtuals(self):
+        """Test a package with multiple virtual dependencies."""
+        s = Spec('hypre').concretize()
+
+
+    def test_concretize_two_virtuals_with_one_bound(self):
+        """Test a package with multiple virtual dependencies and one preset."""
+        s = Spec('hypre ^openblas').concretize()
+
+
+    def test_concretize_two_virtuals_with_two_bound(self):
+        """Test a package with multiple virtual dependencies and two of them preset."""
+        s = Spec('hypre ^openblas ^netlib-lapack').concretize()
+
+
+    def test_concretize_two_virtuals_with_dual_provider(self):
+        """Test a package with multiple virtual dependencies and force a provider
+           that provides both."""
+        s = Spec('hypre ^openblas-with-lapack').concretize()
+
+
+    def test_concretize_two_virtuals_with_dual_provider_and_a_conflict(self):
+        """Test a package with multiple virtual dependencies and force a provider
+           that provides both, and another conflicting package that provides one."""
+        s = Spec('hypre ^openblas-with-lapack ^netlib-lapack')
+        self.assertRaises(spack.spec.MultipleProviderError, s.concretize)
+
+
     def test_virtual_is_fully_expanded_for_callpath(self):
         # force dependence on fake "zmpi" by asking for MPI 10.0
         spec = Spec('callpath ^mpi@10.0')
@@ -190,3 +228,100 @@ def test_compiler_inheritance(self):
         # TODO: not exactly the syntax I would like.
         self.assertTrue(spec['libdwarf'].compiler.satisfies('clang'))
         self.assertTrue(spec['libelf'].compiler.satisfies('clang'))
+
+
+    def test_external_package(self):
+        spec = Spec('externaltool')
+        spec.concretize()
+
+        self.assertEqual(spec['externaltool'].external, '/path/to/external_tool')
+        self.assertFalse('externalprereq' in spec)
+        self.assertTrue(spec['externaltool'].compiler.satisfies('gcc'))
+
+
+    def test_nobuild_package(self):
+        got_error = False
+        spec = Spec('externaltool%clang')
+        try:
+            spec.concretize()
+        except spack.concretize.NoBuildError:
+            got_error = True
+        self.assertTrue(got_error)
+
+
+    def test_external_and_virtual(self):
+        spec = Spec('externaltest')
+        spec.concretize()
+        self.assertEqual(spec['externaltool'].external, '/path/to/external_tool')
+        self.assertEqual(spec['stuff'].external, '/path/to/external_virtual_gcc')
+        self.assertTrue(spec['externaltool'].compiler.satisfies('gcc'))
+        self.assertTrue(spec['stuff'].compiler.satisfies('gcc'))
+
+
+    def test_find_spec_parents(self):
+        """Tests the spec finding logic used by concretization. """
+        s = Spec('a +foo',
+                 Spec('b +foo',
+                      Spec('c'),
+                      Spec('d +foo')),
+                 Spec('e +foo'))
+
+        self.assertEqual('a', find_spec(s['b'], lambda s: '+foo' in s).name)
+
+
+    def test_find_spec_children(self):
+        s = Spec('a',
+                 Spec('b +foo',
+                      Spec('c'),
+                      Spec('d +foo')),
+                 Spec('e +foo'))
+        self.assertEqual('d', find_spec(s['b'], lambda s: '+foo' in s).name)
+        s = Spec('a',
+                 Spec('b +foo',
+                      Spec('c +foo'),
+                      Spec('d')),
+                 Spec('e +foo'))
+        self.assertEqual('c', find_spec(s['b'], lambda s: '+foo' in s).name)
+
+
+    def test_find_spec_sibling(self):
+        s = Spec('a',
+                 Spec('b +foo',
+                      Spec('c'),
+                      Spec('d')),
+                 Spec('e +foo'))
+        self.assertEqual('e', find_spec(s['b'], lambda s: '+foo' in s).name)
+        self.assertEqual('b', find_spec(s['e'], lambda s: '+foo' in s).name)
+
+        s = Spec('a',
+                 Spec('b +foo',
+                      Spec('c'),
+                      Spec('d')),
+                 Spec('e',
+                      Spec('f +foo')))
+        self.assertEqual('f', find_spec(s['b'], lambda s: '+foo' in s).name)
+
+
+    def test_find_spec_self(self):
+        s = Spec('a',
+                 Spec('b +foo',
+                      Spec('c'),
+                      Spec('d')),
+                 Spec('e'))
+        self.assertEqual('b', find_spec(s['b'], lambda s: '+foo' in s).name)
+
+
+    def test_find_spec_none(self):
+        s = Spec('a',
+                 Spec('b',
+                      Spec('c'),
+                      Spec('d')),
+                 Spec('e'))
+        self.assertEqual(None, find_spec(s['b'], lambda s: '+foo' in s))
+
+
+    def test_compiler_child(self):
+        s = Spec('mpileaks%clang ^dyninst%gcc')
+        s.concretize()
+        self.assertTrue(s['mpileaks'].satisfies('%clang'))
+        self.assertTrue(s['dyninst'].satisfies('%gcc'))

lib/spack/spack/test/database.py

@@ -26,6 +26,7 @@
 These tests check the database is functioning properly,
 both in memory and in its file
 """
+import os.path
 import multiprocessing
 import shutil
 import tempfile

lib/spack/spack/test/directory_layout.py

@@ -66,6 +66,9 @@ def test_read_and_write_spec(self):
         packages = list(spack.repo.all_packages())[:max_packages]
 
         for pkg in packages:
+            if pkg.name.startswith('external'):
+                #External package tests cannot be installed
+                continue            
             spec = pkg.spec
 
             # If a spec fails to concretize, just skip it.  If it is a
@@ -171,6 +174,9 @@ def test_find(self):
         # Create install prefixes for all packages in the list
         installed_specs = {}
         for pkg in packages:
+            if pkg.name.startswith('external'):
+                #External package tests cannot be installed
+                continue
             spec = pkg.spec.concretized()
             installed_specs[spec.name] = spec
             self.layout.create_install_directory(spec)

lib/spack/spack/test/environment.py

@@ -0,0 +1,73 @@
+import unittest
+import os
+from spack.environment import EnvironmentModifications
+
+
+class EnvironmentTest(unittest.TestCase):
+    def setUp(self):
+        os.environ.clear()
+        os.environ['UNSET_ME'] = 'foo'
+        os.environ['EMPTY_PATH_LIST'] = ''
+        os.environ['PATH_LIST'] = '/path/second:/path/third'
+        os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g'
+
+    def test_set(self):
+        env = EnvironmentModifications()
+        env.set('A', 'dummy value')
+        env.set('B', 3)
+        env.apply_modifications()
+        self.assertEqual('dummy value', os.environ['A'])
+        self.assertEqual(str(3), os.environ['B'])
+
+    def test_unset(self):
+        env = EnvironmentModifications()
+        self.assertEqual('foo', os.environ['UNSET_ME'])
+        env.unset('UNSET_ME')
+        env.apply_modifications()
+        self.assertRaises(KeyError, os.environ.__getitem__, 'UNSET_ME')
+
+    def test_set_path(self):
+        env = EnvironmentModifications()
+        env.set_path('A', ['foo', 'bar', 'baz'])
+        env.apply_modifications()
+        self.assertEqual('foo:bar:baz', os.environ['A'])
+
+    def test_path_manipulation(self):
+        env = EnvironmentModifications()
+
+        env.append_path('PATH_LIST', '/path/last')
+        env.prepend_path('PATH_LIST', '/path/first')
+
+        env.append_path('EMPTY_PATH_LIST', '/path/middle')
+        env.append_path('EMPTY_PATH_LIST', '/path/last')
+        env.prepend_path('EMPTY_PATH_LIST', '/path/first')
+
+        env.append_path('NEWLY_CREATED_PATH_LIST', '/path/middle')
+        env.append_path('NEWLY_CREATED_PATH_LIST', '/path/last')
+        env.prepend_path('NEWLY_CREATED_PATH_LIST', '/path/first')
+
+        env.remove_path('REMOVE_PATH_LIST', '/remove/this')
+        env.remove_path('REMOVE_PATH_LIST', '/duplicate/')
+
+        env.apply_modifications()
+        self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST'])
+        self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST'])
+        self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST'])
+        self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST'])
+
+    def test_extra_arguments(self):
+        env = EnvironmentModifications()
+        env.set('A', 'dummy value', who='Pkg1')
+        for x in env:
+            assert 'who' in x.args
+        env.apply_modifications()
+        self.assertEqual('dummy value', os.environ['A'])
+
+    def test_extend(self):
+        env = EnvironmentModifications()
+        env.set('A', 'dummy value')
+        env.set('B', 3)
+        copy_construct = EnvironmentModifications(env)
+        self.assertEqual(len(copy_construct), 2)
+        for x, y in zip(env, copy_construct):
+            assert x is y

lib/spack/spack/test/mock_packages_test.py

@@ -49,6 +49,19 @@
       fc: /path/to/gfortran
 """
 
+mock_packages_config = """\
+packages:
+  externaltool:
+    buildable: False
+    paths:
+      externaltool@1.0%gcc@4.5.0: /path/to/external_tool
+  externalvirtual:
+    buildable: False
+    paths:
+      externalvirtual@2.0%clang@3.3: /path/to/external_virtual_clang
+      externalvirtual@1.0%gcc@4.5.0: /path/to/external_virtual_gcc
+"""
+
 class MockPackagesTest(unittest.TestCase):
     def initmock(self):
         # Use the mock packages database for these tests.  This allows
@@ -66,9 +79,10 @@ def initmock(self):
         self.mock_user_config = os.path.join(self.temp_config, 'user')
         mkdirp(self.mock_site_config)
         mkdirp(self.mock_user_config)
-        comp_yaml = os.path.join(self.mock_site_config, 'compilers.yaml')
+        for confs in [('compilers.yaml', mock_compiler_config), ('packages.yaml', mock_packages_config)]:
-        with open(comp_yaml, 'w') as f:
+            conf_yaml = os.path.join(self.mock_site_config, confs[0])
-            f.write(mock_compiler_config)
+            with open(conf_yaml, 'w') as f:
+                f.write(confs[1])
 
         # TODO: Mocking this up is kind of brittle b/c ConfigScope
         # TODO: constructor modifies config_scopes.  Make it cleaner.

lib/spack/spack/test/sbang.py

@@ -0,0 +1,93 @@
+##############################################################################
+# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""\
+Test that Spack's shebang filtering works correctly.
+"""
+import os
+import unittest
+import tempfile
+import shutil
+
+from llnl.util.filesystem import *
+from spack.hooks.sbang import filter_shebangs_in_directory
+import spack
+
+short_line = "#!/this/is/short/bin/bash\n"
+long_line  = "#!/this/" + ('x' * 200) + "/is/long\n"
+sbang_line = '#!/bin/bash %s/bin/sbang\n' % spack.spack_root
+last_line = "last!\n"
+
+class SbangTest(unittest.TestCase):
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+
+        # make sure we can ignore non-files
+        directory = os.path.join(self.tempdir, 'dir')
+        mkdirp(directory)
+
+        # Script with short shebang
+        self.short_shebang = os.path.join(self.tempdir, 'short')
+        with open(self.short_shebang, 'w') as f:
+            f.write(short_line)
+            f.write(last_line)
+
+        # Script with long shebang
+        self.long_shebang = os.path.join(self.tempdir, 'long')
+        with open(self.long_shebang, 'w') as f:
+            f.write(long_line)
+            f.write(last_line)
+
+        # Script already using sbang.
+        self.has_shebang = os.path.join(self.tempdir, 'shebang')
+        with open(self.has_shebang, 'w') as f:
+            f.write(sbang_line)
+            f.write(long_line)
+            f.write(last_line)
+
+
+    def tearDown(self):
+         shutil.rmtree(self.tempdir, ignore_errors=True)
+
+
+
+    def test_shebang_handling(self):
+        filter_shebangs_in_directory(self.tempdir)
+
+        # Make sure this is untouched
+        with open(self.short_shebang, 'r') as f:
+            self.assertEqual(f.readline(), short_line)
+            self.assertEqual(f.readline(), last_line)
+
+        # Make sure this got patched.
+        with open(self.long_shebang, 'r') as f:
+            self.assertEqual(f.readline(), sbang_line)
+            self.assertEqual(f.readline(), long_line)
+            self.assertEqual(f.readline(), last_line)
+
+        # Make sure this is untouched
+        with open(self.has_shebang, 'r') as f:
+            self.assertEqual(f.readline(), sbang_line)
+            self.assertEqual(f.readline(), long_line)
+            self.assertEqual(f.readline(), last_line)

lib/spack/spack/test/stage.py

@@ -277,3 +277,38 @@ def test_restage(self):
             self.check_chdir_to_source(stage, stage_name)
             self.assertFalse('foobar' in os.listdir(stage.source_path))
         self.check_destroy(stage, stage_name)
+
+
+    def test_no_keep_without_exceptions(self):
+        with Stage(archive_url, name=stage_name, keep=False) as stage:
+            pass
+        self.check_destroy(stage, stage_name)
+
+
+    def test_keep_without_exceptions(self):
+        with Stage(archive_url, name=stage_name, keep=True) as stage:
+            pass
+        path = self.get_stage_path(stage, stage_name)
+        self.assertTrue(os.path.isdir(path))
+
+
+    def test_no_keep_with_exceptions(self):
+        try:
+            with Stage(archive_url, name=stage_name, keep=False) as stage:
+                raise Exception()
+
+            path = self.get_stage_path(stage, stage_name)
+            self.assertTrue(os.path.isdir(path))
+        except:
+            pass # ignore here.
+
+
+    def test_keep_exceptions(self):
+        try:
+            with Stage(archive_url, name=stage_name, keep=True) as stage:
+                raise Exception()
+
+            path = self.get_stage_path(stage, stage_name)
+            self.assertTrue(os.path.isdir(path))
+        except:
+            pass # ignore here.

lib/spack/spack/url.py

@@ -142,7 +142,7 @@ def split_url_extension(path):
 
 def downloaded_file_extension(path):
     """This returns the type of archive a URL refers to.  This is
-       sometimes confusing becasue of URLs like:
+       sometimes confusing because of URLs like:
 
            (1) https://github.com/petdance/ack/tarball/1.93_02
 
@@ -225,7 +225,7 @@ def parse_version_offset(path):
         (r'_((\d+\.)+\d+[a-z]?)[.]orig$', stem),
 
         # e.g. http://www.openssl.org/source/openssl-0.9.8s.tar.gz
-        (r'-([^-]+(-alpha|-beta)?)', stem),
+        (r'-v?([^-]+(-alpha|-beta)?)', stem),
 
         # e.g. astyle_1.23_macosx.tar.gz
         (r'_([^_]+(_alpha|_beta)?)', stem),

lib/spack/spack/util/compression.py

@@ -27,13 +27,12 @@
 from itertools import product
 from spack.util.executable import which
 
-# Supported archvie extensions.
+# Supported archive extensions.
 PRE_EXTS = ["tar"]
 EXTS     = ["gz", "bz2", "xz", "Z", "zip", "tgz"]
 
-# Add EXTS last so that .tar.gz is matched *before* tar.gz
+# Add PRE_EXTS and EXTS last so that .tar.gz is matched *before* .tar or .gz
-ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS
+ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + PRE_EXTS + EXTS
-
 
 def allowed_archive(path):
     return any(path.endswith(t) for t in ALLOWED_ARCHIVE_TYPES)

lib/spack/spack/util/environment.py

@@ -59,12 +59,6 @@ def path_put_first(var_name, directories):
     path_set(var_name, new_path)
 
 
-def pop_keys(dictionary, *keys):
-    for key in keys:
-        if key in dictionary:
-            dictionary.pop(key)
-
-
 def dump_environment(path):
     """Dump the current environment out to a file."""
     with open(path, 'w') as env_file:

share/spack/setup-env.sh

@@ -141,7 +141,7 @@ function _spack_pathadd {
     fi
 
     # Do the actual prepending here.
-    eval "_pa_oldvalue=\$${_pa_varname}"
+    eval "_pa_oldvalue=\${${_pa_varname}:-}"
 
     if [ -d "$_pa_new_path" ] && [[ ":$_pa_oldvalue:" != *":$_pa_new_path:"* ]]; then
         if [ -n "$_pa_oldvalue" ]; then

var/spack/repos/builtin.mock/packages/externalprereq/package.py

@@ -0,0 +1,34 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externalprereq(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/prereq-1.0.tar.gz"
+
+    version('1.4', 'f1234567890abcdef1234567890abcde')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin.mock/packages/externaltest/package.py

@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externaltest(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/test-1.0.tar.gz"
+
+    version('1.0', '1234567890abcdef1234567890abcdef')
+
+    depends_on('stuff')
+    depends_on('externaltool')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin.mock/packages/externaltool/package.py

@@ -0,0 +1,36 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externaltool(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/tool-1.0.tar.gz"
+
+    version('1.0', '1234567890abcdef1234567890abcdef')
+
+    depends_on('externalprereq')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin.mock/packages/externalvirtual/package.py

@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Externalvirtual(Package):
+    homepage = "http://somewhere.com"
+    url      = "http://somewhere.com/stuff-1.0.tar.gz"
+
+    version('1.0', '1234567890abcdef1234567890abcdef')
+    version('2.0', '234567890abcdef1234567890abcdef1')
+
+    provides('stuff')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin.mock/packages/hypre/package.py

@@ -0,0 +1,39 @@
+##############################################################################
+# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Hypre(Package):
+    """Hypre is included here as an example of a package that depends on
+       both LAPACK and BLAS."""
+    homepage = "http://www.openblas.net"
+    url      = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz"
+
+    version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9')
+
+    depends_on('lapack')
+    depends_on('blas')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin.mock/packages/mpich/package.py

@@ -38,6 +38,7 @@ class Mpich(Package):
     version('3.0.2', 'foobarbaz')
     version('3.0.1', 'foobarbaz')
     version('3.0', 'foobarbaz')
+    version('1.0', 'foobarbas')
 
     provides('mpi@:3', when='@3:')
     provides('mpi@:1', when='@:1')

var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py

@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class OpenblasWithLapack(Package):
+    """Dummy version of OpenBLAS that also provides LAPACK, for testing."""
+    homepage = "http://www.openblas.net"
+    url      = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz"
+
+    version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9')
+
+    provides('lapack')
+    provides('blas')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin.mock/packages/python/package.py

@@ -0,0 +1,43 @@
+##############################################################################
+# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Python(Package):
+    """Dummy Python package to demonstrate preferred versions."""
+    homepage = "http://www.python.org"
+    url      = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz"
+
+    extendable = True
+
+    version('3.5.1', 'be78e48cdfc1a7ad90efff146dce6cfe')
+    version('3.5.0', 'a56c0c0b45d75a0ec9c6dee933c41c36')
+    version('2.7.11', '6b6076ec9e93f05dd63e47eb9c15728b', preferred=True)
+    version('2.7.10', 'd7547558fd673bd9d38e2108c6b42521')
+    version('2.7.9', '5eebcaa0030dc4061156d3429657fb83')
+    version('2.7.8', 'd4bca0159acb0b44a781292b5231936f')
+
+    def install(self, spec, prefix):
+        pass
+

var/spack/repos/builtin/packages/arpack-ng/package.py

@@ -35,6 +35,10 @@ class ArpackNg(Package):
     variant('shared', default=True, description='Enables the build of shared libraries')
     variant('mpi', default=False, description='Activates MPI support')
 
+    # The function pdlamch10 does not set the return variable. This is fixed upstream
+    # see https://github.com/opencollab/arpack-ng/issues/34
+    patch('pdlamch10.patch', when='@3.3:')
+
     depends_on('blas')
     depends_on('lapack')
     depends_on('mpi', when='+mpi')
@@ -46,7 +50,10 @@ def install(self, spec, prefix):
         options = ['--prefix=%s' % prefix]
 
         if '+mpi' in spec:
-            options.append('--enable-mpi')
+            options.extend([
+                '--enable-mpi',
+                'F77=mpif77' #FIXME: avoid hardcoding MPI wrapper names
+            ])
 
         if '~shared' in spec:
             options.append('--enable-shared=no')

var/spack/repos/builtin/packages/arpack-ng/pdlamch10.patch

@@ -0,0 +1,15 @@
+diff --git a/PARPACK/SRC/MPI/pdlamch10.f b/PARPACK/SRC/MPI/pdlamch10.f
+index 6571da9..2882c2e 100644
+--- a/PARPACK/SRC/MPI/pdlamch10.f
++++ b/PARPACK/SRC/MPI/pdlamch10.f
+@@ -86,8 +86,8 @@
+           TEMP = TEMP1
+       END IF
+ *
+-      PDLAMCH = TEMP
++      PDLAMCH10 = TEMP
+ *
+-*     End of PDLAMCH
++*     End of PDLAMCH10
+ *
+       END

var/spack/repos/builtin/packages/autoconf/package.py

@@ -6,6 +6,7 @@ class Autoconf(Package):
     url      = "http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz"
 
     version('2.69', '82d05e03b93e45f5a39b828dc9c6c29b')
+    version('2.62', '6c1f3b3734999035d77da5024aab4fbd')
 
     def install(self, spec, prefix):
         configure("--prefix=%s" % prefix)

var/spack/repos/builtin/packages/automake/package.py

@@ -5,7 +5,9 @@ class Automake(Package):
     homepage = "http://www.gnu.org/software/automake/"
     url      = "http://ftp.gnu.org/gnu/automake/automake-1.14.tar.gz"
 
+    version('1.15',   '716946a105ca228ab545fc37a70df3a3')
     version('1.14.1', 'd052a3e884631b9c7892f2efce542d75')
+    version('1.11.6', '0286dc30295b62985ca51919202ecfcc')
 
     depends_on('autoconf')
 

var/spack/repos/builtin/packages/binutils/package.py

@@ -4,10 +4,13 @@ class Binutils(Package):
     """GNU binutils, which contain the linker, assembler, objdump and others"""
     homepage = "http://www.gnu.org/software/binutils/"
 
-    version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2")
+    url="https://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2"
-    version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.24.tar.bz2")
+
-    version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.23.2.tar.bz2")
+    version('2.26', '64146a0faa3b411ba774f47d41de239f')
-    version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.20.1.tar.bz2")
+    version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66')
+    version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b')
+    version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e')
+    version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764')
 
     # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell
     variant('krellpatch', default=False, description="build with openspeedshop based patch.")

var/spack/repos/builtin/packages/blitz/package.py

@@ -0,0 +1,15 @@
+from spack import *
+
+class Blitz(Package):
+    """N-dimensional arrays for C++"""
+    homepage = "http://github.com/blitzpp/blitz"
+    url      = "https://github.com/blitzpp/blitz/tarball/1.0.0"
+
+    version('1.0.0', '9f040b9827fe22228a892603671a77af')
+
+    # No dependencies
+
+    def install(self, spec, prefix):
+        configure('--prefix=%s' % prefix)
+        make()
+        make("install")

var/spack/repos/builtin/packages/cmake/package.py

@@ -30,6 +30,7 @@ class Cmake(Package):
     homepage  = 'https://www.cmake.org'
     url       = 'https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz'
 
+    version('3.5.0',    '33c5d09d4c33d4ffcc63578a6ba8777e')
     version('3.4.3',    '4cb3ff35b2472aae70f542116d616e63')
     version('3.4.0',    'cd3034e0a44256a0917e254167217fc8')
     version('3.3.1',    '52638576f4e1e621fed6c3410d3a1b12')
@@ -37,16 +38,48 @@ class Cmake(Package):
     version('2.8.10.2', '097278785da7182ec0aea8769d06860c')
 
     variant('ncurses', default=True, description='Enables the build of the ncurses gui')
+    variant('qt', default=False, description='Enables the build of cmake-gui')
+    variant('doc', default=False, description='Enables the generation of html and man page documentation')
+
     depends_on('ncurses', when='+ncurses')
+    depends_on('qt', when='+qt')
+    depends_on('python@2.7.11:', when='+doc')
+    depends_on('py-sphinx', when='+doc')
 
     def url_for_version(self, version):
         """Handle CMake's version-based custom URLs."""
         return 'https://cmake.org/files/v%s/cmake-%s.tar.gz' % (version.up_to(2), version)
 
+    def validate(self, spec):
+        """
+        Checks if incompatible versions of qt were specified
+
+        :param spec: spec of the package
+        :raises RuntimeError: in case of inconsistencies
+        """
+
+        if '+qt' in spec and spec.satisfies('^qt@5.4.0'):
+            msg = 'qt-5.4.0 has broken CMake modules.'
+            raise RuntimeError(msg)
 
     def install(self, spec, prefix):
-        configure('--prefix='   + prefix,
+        # Consistency check
-                  '--parallel=' + str(make_jobs),
+        self.validate(spec)
-                  '--', '-DCMAKE_USE_OPENSSL=ON')
+
+        # configure, build, install:
+        options = ['--prefix=%s' % prefix]
+        options.append('--parallel=%s' % str(make_jobs))
+
+        if '+qt' in spec:
+            options.append('--qt-gui')
+
+        if '+doc' in spec:
+            options.append('--sphinx-html')
+            options.append('--sphinx-man')
+
+        options.append('--')
+        options.append('-DCMAKE_USE_OPENSSL=ON')
+
+        configure(*options)
         make()
         make('install')

var/spack/repos/builtin/packages/cryptopp/package.py

@@ -8,8 +8,8 @@ class Cryptopp(Package):
        public-key encryption (RSA, DSA), and a few obsolete/historical encryption
        algorithms (MD5, Panama)."""
 
-    homepage = "http://www.cryptopp.com/"
+    homepage = "http://www.cryptopp.com"
-    url      = "http://www.cryptopp.com/cryptopp563.zip"
+    base_url = "http://www.cryptopp.com"
 
     version('5.6.3', '3c5b70e2ec98b7a24988734446242d07')
     version('5.6.2', '7ed022585698df48e65ce9218f6c6a67')
@@ -25,7 +25,5 @@ def install(self, spec, prefix):
         install('libcryptopp.a', prefix.lib)
 
     def url_for_version(self, version):
-        version_tuple = tuple(v for v in iter(version))
+        version_string = str(version).replace('.', '')
-        version_string = reduce(lambda vs, nv: vs + str(nv), version_tuple, "")
+        return '%s/cryptopp%s.zip' % (Cryptopp.base_url, version_string)
-
-        return "%scryptopp%s.zip" % (Cryptopp.homepage, version_string)

var/spack/repos/builtin/packages/curl/package.py

@@ -7,6 +7,7 @@ class Curl(Package):
     homepage = "http://curl.haxx.se"
     url      = "http://curl.haxx.se/download/curl-7.46.0.tar.bz2"
 
+    version('7.47.1', '9ea3123449439bbd960cd25cf98796fb')
     version('7.46.0', '9979f989a2a9930d10f1b3deeabc2148')
     version('7.45.0', '62c1a352b28558f25ba6209214beadc8')
     version('7.44.0', '6b952ca00e5473b16a11f05f06aa8dae')

var/spack/repos/builtin/packages/eigen/package.py

@@ -48,7 +48,7 @@ class Eigen(Package):
     depends_on('metis', when='+metis')
     depends_on('scotch', when='+scotch')
     depends_on('fftw', when='+fftw')
-    depends_on('SuiteSparse', when='+suitesparse')
+    depends_on('suite-sparse', when='+suitesparse')
     depends_on('mpfr@2.3.0:')  # Eigen 3.2.7 requires at least 2.3.0
     depends_on('gmp')
 

var/spack/repos/builtin/packages/emacs/package.py

@@ -0,0 +1,21 @@
+from spack import *
+
+class Emacs(Package):
+    """The Emacs programmable text editor."""
+    homepage = "https://www.gnu.org/software/emacs"
+    url      = "http://ftp.gnu.org/gnu/emacs/emacs-24.5.tar.gz"
+
+    version('24.5', 'd74b597503a68105e61b5b9f6d065b44')
+
+    depends_on('ncurses')
+    # Emacs also depends on:
+    #     GTK or other widget library
+    #     libtiff, png, etc.
+    # For now, we assume the system provides all that stuff.
+    # For Ubuntu 14.04 LTS:
+    #     sudo apt-get install libgtk-3-dev libxpm-dev libtiff5-dev libjpeg8-dev libgif-dev libpng12-dev
+
+    def install(self, spec, prefix):
+        configure('--prefix=%s' % prefix)
+        make()
+        make("install")

var/spack/repos/builtin/packages/gettext/package.py

@@ -0,0 +1,30 @@
+from spack import *
+
+class Gettext(Package):
+    """GNU internationalization (i18n) and localization (l10n) library."""
+    homepage = "https://www.gnu.org/software/gettext/"
+    url      = "http://ftpmirror.gnu.org/gettext/gettext-0.19.7.tar.xz"
+
+    version('0.19.7', 'f81e50556da41b44c1d59ac93474dca5')
+
+    def install(self, spec, prefix):
+        options = ['--disable-dependency-tracking',
+                   '--disable-silent-rules',
+                   '--disable-debug',
+                   '--prefix=%s' % prefix,
+                   '--with-included-gettext',
+                   '--with-included-glib',
+                   '--with-included-libcroco',
+                   '--with-included-libunistring',
+                   '--with-emacs',
+                   '--with-lispdir=%s/emacs/site-lisp/gettext' % prefix.share,
+                   '--disable-java',
+                   '--disable-csharp',
+                   '--without-git', # Don't use VCS systems to create these archives
+                   '--without-cvs',
+                   '--without-xz']
+
+        configure(*options)
+
+        make()
+        make("install")

var/spack/repos/builtin/packages/git/package.py

@@ -5,14 +5,22 @@ class Git(Package):
        system designed to handle everything from small to very large
        projects with speed and efficiency."""
     homepage = "http://git-scm.com"
-    url      = "https://www.kernel.org/pub/software/scm/git/git-2.2.1.tar.gz"
+    url      = "https://github.com/git/git/tarball/v2.7.1"
 
-    version('2.6.3', 'b711be7628a4a2c25f38d859ee81b423')
+    version('2.8.0-rc2', 'c2cf9f2cc70e35f2fafbaf9258f82e4c')
-    version('2.6.2', 'da293290da69f45a86a311ad3cd43dc8')
+    version('2.7.3', 'fa1c008b56618c355a32ba4a678305f6')
-    version('2.6.1', '4c62ee9c5991fe93d99cf2a6b68397fd')
+    version('2.7.1', 'bf0706b433a8dedd27a63a72f9a66060')
-    version('2.6.0', 'eb76a07148d94802a1745d759716a57e')
+
-    version('2.5.4', '3eca2390cf1fa698b48e2a233563a76b')
+
-    version('2.2.1', 'ff41fdb094eed1ec430aed8ee9b9849c')
+    # See here for info on vulnerable Git versions:
+    # http://www.theregister.co.uk/2016/03/16/git_server_client_patch_now/
+    # All the following are vulnerable
+    #version('2.6.3', 'b711be7628a4a2c25f38d859ee81b423')
+    #version('2.6.2', 'da293290da69f45a86a311ad3cd43dc8')
+    #version('2.6.1', '4c62ee9c5991fe93d99cf2a6b68397fd')
+    #version('2.6.0', 'eb76a07148d94802a1745d759716a57e')
+    #version('2.5.4', '3eca2390cf1fa698b48e2a233563a76b')
+    #version('2.2.1', 'ff41fdb094eed1ec430aed8ee9b9849c')
 
 
     # Git compiles with curl support by default on but if your system
@@ -24,6 +32,7 @@ class Git(Package):
     variant("expat", default=False, description="Add the internal support of expat for https push")
 
     depends_on("openssl")
+    depends_on("autoconf")
     depends_on("curl", when="+curl")
     depends_on("expat", when="+expat")
 
@@ -47,6 +56,7 @@ def install(self, spec, prefix):
         if '+expat' in spec:
             configure_args.append("--with-expat=%s" % spec['expat'].prefix)
 
+        which('autoreconf')('-i')
         configure(*configure_args)
         make()
         make("install")

var/spack/repos/builtin/packages/gl2ps/package.py

@@ -0,0 +1,18 @@
+from spack import *
+
+class Gl2ps(Package):
+    """GL2PS is a C library providing high quality vector output for any
+    OpenGL application."""
+
+    homepage = "http://www.geuz.org/gl2ps/"
+    url      = "http://geuz.org/gl2ps/src/gl2ps-1.3.9.tgz"
+
+    version('1.3.9', '377b2bcad62d528e7096e76358f41140')
+
+    depends_on("libpng")
+
+    def install(self, spec, prefix):
+        cmake('.', *std_cmake_args)
+
+        make()
+        make("install")

var/spack/repos/builtin/packages/hdf5/package.py

@@ -46,7 +46,6 @@ class Hdf5(Package):
 
     variant('cxx', default=True, description='Enable C++ support')
     variant('fortran', default=True, description='Enable Fortran support')
-    variant('unsupported', default=True, description='Enables unsupported configuration options')
 
     variant('mpi', default=False, description='Enable MPI support')
     variant('szip', default=False, description='Enable szip support')
@@ -74,6 +73,13 @@ def install(self, spec, prefix):
         self.validate(spec)
         # Handle compilation after spec validation
         extra_args = []
+
+        # Always enable this option. This does not actually enable any
+        # features: it only *allows* the user to specify certain
+        # combinations of other arguments. Enabling it just skips a
+        # sanity check in configure, so this doesn't merit a variant.
+        extra_args.append("--enable-unsupported")
+
         if '+debug' in spec:
             extra_args.append('--enable-debug=all')
         else:
@@ -84,9 +90,6 @@ def install(self, spec, prefix):
         else:
             extra_args.append('--enable-static-exec')
 
-        if '+unsupported' in spec:
-            extra_args.append("--enable-unsupported")
-
         if '+cxx' in spec:
             extra_args.append('--enable-cxx')
 

var/spack/repos/builtin/packages/hypre/package.py

@@ -1,4 +1,5 @@
 from spack import *
+import os, sys
 
 class Hypre(Package):
     """Hypre is a library of high performance preconditioners that
@@ -8,8 +9,12 @@ class Hypre(Package):
     homepage = "http://computation.llnl.gov/project/linear_solvers/software.php"
     url      = "http://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz"
 
+    version('2.10.1', 'dc048c4cabb3cd549af72591474ad674')
     version('2.10.0b', '768be38793a35bb5d055905b271f5b8e')
 
+    # hypre does not know how to build shared libraries on Darwin
+    variant('shared', default=sys.platform!='darwin', description="Build shared library version (disables static library)")
+
     depends_on("mpi")
     depends_on("blas")
     depends_on("lapack")
@@ -17,16 +22,26 @@ class Hypre(Package):
     def install(self, spec, prefix):
         blas_dir = spec['blas'].prefix
         lapack_dir = spec['lapack'].prefix
+        mpi_dir = spec['mpi'].prefix
+
+        os.environ['CC'] = os.path.join(mpi_dir, 'bin', 'mpicc')
+        os.environ['CXX'] = os.path.join(mpi_dir, 'bin', 'mpicxx')
+        os.environ['F77'] = os.path.join(mpi_dir, 'bin', 'mpif77')
+
+
+        configure_args = [
+                "--prefix=%s" % prefix,
+                "--with-lapack-libs=lapack",
+                "--with-lapack-lib-dirs=%s/lib" % lapack_dir,
+                "--with-blas-libs=blas",
+                "--with-blas-lib-dirs=%s/lib" % blas_dir]
+        if '+shared' in self.spec:
+            configure_args.append("--enable-shared")
 
         # Hypre's source is staged under ./src so we'll have to manually
         # cd into it.
         with working_dir("src"):
-            configure(
+            configure(*configure_args)
-                "--prefix=%s" % prefix,
+
-                "--with-blas-libs=blas",
-                "--with-blas-lib-dirs=%s/lib" % blas_dir,
-                "--with-lapack-libs=\"lapack blas\"",
-                "--with-lapack-lib-dirs=%s/lib" % lapack_dir,
-                "--with-MPI")
             make()
             make("install")

var/spack/repos/builtin/packages/jdk/package.py

@@ -28,7 +28,7 @@ class Jdk(Package):
         '-H', # specify required License Agreement cookie
         'Cookie: oraclelicense=accept-securebackup-cookie']
 
-    def do_fetch(self):
+    def do_fetch(self, mirror_only=False):
         # Add our custom curl commandline options
         tty.msg(
             "[Jdk] Adding required commandline options to curl " +
@@ -39,7 +39,7 @@ def do_fetch(self):
             spack.curl.add_default_arg(option)
 
         # Now perform the actual fetch
-        super(Jdk, self).do_fetch()
+        super(Jdk, self).do_fetch(mirror_only)
 
 
     def install(self, spec, prefix):

var/spack/repos/builtin/packages/libelf/package.py

@@ -38,6 +38,8 @@ class Libelf(Package):
 
     provides('elf')
 
+    sanity_check_is_file = 'include/libelf.h'
+
     def install(self, spec, prefix):
         configure("--prefix=" + prefix,
                   "--enable-shared",

var/spack/repos/builtin/packages/libevent/package.py

@@ -22,9 +22,16 @@ class Libevent(Package):
     version('2.0.13', 'af786b4b3f790c9d3279792edf7867fc')
     version('2.0.12', '42986228baf95e325778ed328a93e070')
 
+    variant('openssl', default=True, description="Build with encryption enabled at the libevent level.")
+    depends_on('openssl', when='+openssl')
 
     def install(self, spec, prefix):
-        configure("--prefix=%s" % prefix)
+        configure_args = []
+        if '+openssl' in spec:
+            configure_args.append('--enable-openssl')
+        else:
+            configure_args.append('--enable-openssl')
 
+        configure("--prefix=%s" % prefix, *configure_args)
         make()
         make("install")

var/spack/repos/builtin/packages/libsigsegv/package.py

@@ -0,0 +1,15 @@
+from spack import *
+
+class Libsigsegv(Package):
+    """GNU libsigsegv is a library for handling page faults in user mode."""
+    homepage = "https://www.gnu.org/software/libsigsegv/"
+    url      = "ftp://ftp.gnu.org/gnu/libsigsegv/libsigsegv-2.10.tar.gz"
+
+    version('2.10', '7f96fb1f65b3b8cbc1582fb7be774f0f')
+
+    def install(self, spec, prefix):
+        configure('--prefix=%s' % prefix,
+                  '--enable-shared')
+
+        make()
+        make("install")

var/spack/repos/builtin/packages/llvm/package.py

@@ -1,5 +1,5 @@
 ##############################################################################
-# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2016, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory.
 #
 # This file is part of Spack.
@@ -34,7 +34,7 @@ class Llvm(Package):
        it is the full name of the project.
     """
     homepage = 'http://llvm.org/'
-    url = 'http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz'
+    url = 'http://llvm.org/releases/3.7.1/llvm-3.7.1.src.tar.xz'
 
     version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package
 
@@ -52,7 +52,7 @@ class Llvm(Package):
     depends_on('cmake @2.8.12.2:')
 
     # Universal dependency
-    depends_on('python@2.7:')
+    depends_on('python@2.7:2.8')  # Seems not to support python 3.X.Y
 
     # lldb dependencies
     depends_on('ncurses', when='+lldb')
@@ -132,6 +132,36 @@ class Llvm(Package):
                         'llvm-libunwind' : 'http://llvm.org/svn/llvm-project/libunwind/trunk',
                         }
                   },
+                  {
+                    'version' : '3.8.0',
+                    'md5':'07a7a74f3c6bd65de4702bf941b511a0',
+                    'resources' : {
+                        'compiler-rt' : 'd6fcbe14352ffb708e4d1ac2e48bb025',
+                        'openmp' : '8fd7cc35d48051613cf1e750e9f22e40',
+                        'polly' : '1b3b20f52d34a4024e21a4ea7112caa7',
+                        'libcxx' : 'd6e0bdbbee39f7907ad74fd56d03b88a',
+                        'libcxxabi' : 'bbe6b4d72c7c5978550d370af529bcf7',
+                        'clang' : 'cc99e7019bb74e6459e80863606250c5',
+                        'clang-tools-extra' : 'c2344f50e0eea0b402f0092a80ddc036',
+                        'lldb' : 'a5da35ed9cc8c8817ee854e3dbfba00e',
+                        'llvm-libunwind' : '162ade468607f153cca12be90b5194fa',
+                        }
+                  },
+                  {
+                    'version' : '3.7.1',
+                    'md5':'bf8b3a2c79e61212c5409041dfdbd319',
+                    'resources' : {
+                        'compiler-rt' : '1c6975daf30bb3b0473b53c3a1a6ff01',
+                        'openmp' : 'b4ad08cda4e5c22e42b66062b140438e',
+                        'polly' : '3a2a7367002740881637f4d47bca4dc3',
+                        'libcxx' : 'f9c43fa552a10e14ff53b94d04bea140',
+                        'libcxxabi' : '52d925afac9f97e9dcac90745255c169',
+                        'clang' : '0acd026b5529164197563d135a8fd83e',
+                        'clang-tools-extra' : '5d49ff745037f061a7c86aeb6a24c3d2',
+                        'lldb' : 'a106d8a0d21fc84d76953822fbaf3398',
+                        'llvm-libunwind' : '814bd52c9247c5d04629658fbcb3ab8c',
+                        }
+                  },
                   {
                     'version' : '3.7.0',
                     'md5':'b98b9495e5655a672d6cb83e1a180f8e',

var/spack/repos/builtin/packages/m4/package.py

@@ -7,7 +7,19 @@ class M4(Package):
 
     version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76')
 
+    patch('pgi.patch', when='@1.4.17')
+
+    variant('sigsegv', default=True, description="Build the libsigsegv dependency")
+
+    depends_on('libsigsegv', when='+sigsegv')
+
     def install(self, spec, prefix):
-        configure("--prefix=%s" % prefix)
+        configure_args = []
+        if 'libsigsegv' in spec:
+            configure_args.append('--with-libsigsegv-prefix=%s' % spec['libsigsegv'].prefix)
+        else:
+            configure_args.append('--without-libsigsegv-prefix')
+
+        configure("--prefix=%s" % prefix, *configure_args)
         make()
         make("install")

var/spack/repos/builtin/packages/m4/pgi.patch

@@ -0,0 +1,10 @@
+--- a/lib/config.hin
++++ b/lib/config.hin
+@@ -1510,6 +1510,7 @@
+       ? defined __GNUC_STDC_INLINE__ && __GNUC_STDC_INLINE__ \
+       : (199901L <= __STDC_VERSION__ \
+          && !defined __HP_cc \
++         && !defined __PGI \
+          && !(defined __SUNPRO_C && __STDC__))) \
+      && !defined _GL_EXTERN_INLINE_APPLE_BUG)
+ # define _GL_INLINE inline

var/spack/repos/builtin/packages/mpc/package.py

@@ -37,6 +37,12 @@ class Mpc(Package):
     depends_on("gmp")
     depends_on("mpfr")
 
+    def url_for_version(self, version):
+        if version < Version("1.0.1"):
+            return "http://www.multiprecision.org/mpc/download/mpc-%s.tar.gz" % version
+        else:
+            return "ftp://ftp.gnu.org/gnu/mpc/mpc-%s.tar.gz" % version
+
     def install(self, spec, prefix):
         configure("--prefix=%s" % prefix)
         make()

var/spack/repos/builtin/packages/mpfr/package.py

@@ -28,8 +28,9 @@ class Mpfr(Package):
     """The MPFR library is a C library for multiple-precision
        floating-point computations with correct rounding."""
     homepage = "http://www.mpfr.org"
-    url      = "http://www.mpfr.org/mpfr-current/mpfr-3.1.3.tar.bz2"
+    url      = "https://gforge.inria.fr/frs/download.php/latestfile/159/mpfr-3.1.2.tar.bz2"
 
+    version('3.1.4', 'b8a2f6b0e68bef46e53da2ac439e1cf4')
     version('3.1.3', '5fdfa3cfa5c86514ee4a241a1affa138')
     version('3.1.2', 'ee2c3ac63bf0c2359bf08fc3ee094c19')
 

var/spack/repos/builtin/packages/mpich/package.py

@@ -25,6 +25,7 @@
 from spack import *
 import os
 
+
 class Mpich(Package):
     """MPICH is a high performance and widely portable implementation of
        the Message Passing Interface (MPI) standard."""
@@ -46,14 +47,17 @@ class Mpich(Package):
     provides('mpi@:3.0', when='@3:')
     provides('mpi@:1.3', when='@1:')
 
-    def setup_dependent_environment(self, module, spec, dep_spec):
+    def setup_dependent_environment(self, env, dependent_spec):
-        """For dependencies, make mpicc's use spack wrapper."""
+        env.set('MPICH_CC', spack_cc)
-        os.environ['MPICH_CC']  = os.environ['CC']
+        env.set('MPICH_CXX', spack_cxx)
-        os.environ['MPICH_CXX'] = os.environ['CXX']
+        env.set('MPICH_F77', spack_f77)
-        os.environ['MPICH_F77'] = os.environ['F77']
+        env.set('MPICH_F90', spack_f90)
-        os.environ['MPICH_F90'] = os.environ['FC']
+        env.set('MPICH_FC', spack_fc)
-        os.environ['MPICH_FC'] = os.environ['FC']
 
+    def setup_dependent_package(self, module, dep_spec):
+        """For dependencies, make mpicc's use spack wrapper."""
+        # FIXME : is this necessary ? Shouldn't this be part of a contract with MPI providers?
+        module.mpicc = join_path(self.prefix.bin, 'mpicc')
 
     def install(self, spec, prefix):
         config_args = ["--prefix=" + prefix,

var/spack/repos/builtin/packages/mumps/package.py

@@ -23,7 +23,7 @@ class Mumps(Package):
 
     depends_on('scotch + esmumps', when='~ptscotch+scotch')
     depends_on('scotch + esmumps + mpi', when='+ptscotch')
-    depends_on('metis', when='~parmetis+metis')
+    depends_on('metis', when='+metis')
     depends_on('parmetis', when="+parmetis")
     depends_on('blas')
     depends_on('lapack')
@@ -54,15 +54,19 @@ def write_makefile_inc(self):
             if '+ptscotch' in self.spec:
                 orderings.append('-Dptscotch')
 
-        if '+parmetis' in self.spec or '+metis' in self.spec:
+        if '+parmetis' in self.spec and '+metis' in self.spec:
             libname = 'parmetis' if '+parmetis' in self.spec else 'metis'
             makefile_conf.extend(
-                ["IMETIS = -I%s" % self.spec[libname].prefix.include,
+                ["IMETIS = -I%s" % self.spec['parmetis'].prefix.include,
-                 "LMETIS = -L%s -l%s" % (self.spec[libname].prefix.lib, libname)])
+                 "LMETIS = -L%s -l%s -L%s -l%s" % (self.spec['parmetis'].prefix.lib, 'parmetis',self.spec['metis'].prefix.lib, 'metis')])
+
+            orderings.append('-Dparmetis')
+        elif '+metis' in self.spec:
+            makefile_conf.extend(
+                ["IMETIS = -I%s" % self.spec['metis'].prefix.include,
+                 "LMETIS = -L%s -l%s" % (self.spec['metis'].prefix.lib, 'metis')])
 
             orderings.append('-Dmetis')
-            if '+parmetis' in self.spec:
-                orderings.append('-Dparmetis')
 
         makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings)))
 

var/spack/repos/builtin/packages/muparser/package.py

@@ -0,0 +1,18 @@
+from spack import *
+
+class Muparser(Package):
+    """C++ math expression parser library."""
+    homepage = "http://muparser.beltoforion.de/"
+    url      = "https://github.com/beltoforion/muparser/archive/v2.2.5.tar.gz"
+
+    version('2.2.5', '02dae671aa5ad955fdcbcd3fee313fb7')
+
+    def install(self, spec, prefix):
+        options = ['--disable-debug',
+                   '--disable-dependency-tracking',
+                   '--prefix=%s' % prefix]
+
+        configure(*options)
+
+        make()
+        make("install")

var/spack/repos/builtin/packages/mvapich2/package.py

@@ -123,7 +123,7 @@ def set_network_type(self, spec, configure_args):
                 count += 1
         if count > 1:
             raise RuntimeError('network variants are mutually exclusive (only one can be selected at a time)')
-
+        network_options = []
         # From here on I can suppose that only one variant has been selected
         if self.enabled(Mvapich2.PSM) in spec:
             network_options = ["--with-device=ch3:psm"]

var/spack/repos/builtin/packages/netcdf-cxx4/package.py

@@ -0,0 +1,15 @@
+from spack import *
+
+class NetcdfCxx4(Package):
+    """C++ interface for NetCDF4"""
+    homepage = "http://www.unidata.ucar.edu/software/netcdf"
+    url      = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz"
+
+    version('4.2', 'd019853802092cf686254aaba165fc81')
+
+    depends_on('netcdf')
+
+    def install(self, spec, prefix):
+        configure('--prefix=%s' % prefix)
+        make()
+        make("install")

var/spack/repos/builtin/packages/netcdf-fortran/package.py

@@ -0,0 +1,16 @@
+from spack import *
+
+class NetcdfFortran(Package):
+    """Fortran interface for NetCDF4"""
+
+    homepage = "http://www.unidata.ucar.edu/software/netcdf"
+    url      = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-fortran-4.4.3.tar.gz"
+
+    version('4.4.3', 'bfd4ae23a34635b273d3eb0d91cbde9e')
+
+    depends_on('netcdf')
+
+    def install(self, spec, prefix):
+        configure("--prefix=%s" % prefix)
+        make()
+        make("install")