Allow arbitrary Prefix attributes (#4591)

* Allow arbitrary Prefix attributes
* Test attribute type as well

* Flake8 fixes

* Remove __new__ method

* Fewer uses of join_path in the docs

lib/spack/docs/packaging_guide.rst

@@ -2408,15 +2408,21 @@ is handy when a package supports additional variants like
 Blas and Lapack libraries
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Different packages provide implementation of ``Blas`` and ``Lapack``
+Multiple packages provide implementations of ``Blas`` and ``Lapack``
 routines.  The names of the resulting static and/or shared libraries
 differ from package to package. In order to make the ``install()`` method
 independent of the choice of ``Blas`` implementation, each package which
 provides it sets up ``self.spec.blas_libs`` to point to the correct
 ``Blas`` libraries.  The same applies to packages which provide
 ``Lapack``. Package developers are advised to use these variables, for
-example ``spec['blas'].blas_libs.joined()`` instead of hard-coding
+example ``spec['blas'].blas_libs.joined()`` instead of hard-coding them:
-``join_path(spec['blas'].prefix.lib, 'libopenblas.so')``.
+
+.. code-block:: python
+
+   if 'openblas' in spec:
+       libs = join_path(spec['blas'].prefix.lib, 'libopenblas.so')
+   elif 'intel-mkl' in spec:
+       ...
 
 .. _prefix-objects:
 
@@ -2430,7 +2436,7 @@ e.g.:
 
 .. code-block:: python
 
-   configure('--prefix=' + prefix)
+   configure('--prefix={0}'.format(prefix))
 
 For the most part, prefix objects behave exactly like strings.  For
 packages that do not have their own install target, or for those that
@@ -2451,29 +2457,27 @@ yourself, e.g.:
        mkdirp(prefix.lib)
        install('libfoo.a', prefix.lib)
 
-Most of the standard UNIX directory names are attributes on the
-``prefix`` object.  Here is a full list:
 
-  =========================  ================================================
+Attributes of this object are created on the fly when you request them,
-  Prefix Attribute           Location
+so any of the following will work:
-  =========================  ================================================
-  ``prefix.bin``             ``$prefix/bin``
-  ``prefix.sbin``            ``$prefix/sbin``
-  ``prefix.etc``             ``$prefix/etc``
-  ``prefix.include``         ``$prefix/include``
-  ``prefix.lib``             ``$prefix/lib``
-  ``prefix.lib64``           ``$prefix/lib64``
-  ``prefix.libexec``         ``$prefix/libexec``
-  ``prefix.share``           ``$prefix/share``
-  ``prefix.doc``             ``$prefix/doc``
-  ``prefix.info``            ``$prefix/info``
 
-  ``prefix.man``             ``$prefix/man``
+======================  =======================
-  ``prefix.man[1-8]``        ``$prefix/man/man[1-8]``
+Prefix Attribute        Location
+======================  =======================
+``prefix.bin``          ``$prefix/bin``
+``prefix.lib64``        ``$prefix/lib64``
+``prefix.share.man``    ``$prefix/share/man``
+``prefix.foo.bar.baz``  ``$prefix/foo/bar/baz``
+======================  =======================
+
+Of course, this only works if your file or directory is a valid Python
+variable name. If your file or directory contains dashes or dots, use
+``join_path`` instead:
+
+.. code-block:: python
+
+   join_path(prefix.lib, 'libz.a')
 
-  ``prefix.share_man``       ``$prefix/share/man``
-  ``prefix.share_man[1-8]``  ``$prefix/share/man[1-8]``
-  =========================  ================================================
 
 .. _spec-objects:
 
@@ -2572,23 +2576,25 @@ of its dependencies satisfy the provided spec.
 Accessing Dependencies
 ^^^^^^^^^^^^^^^^^^^^^^
 
-You may need to get at some file or binary that's in the prefix of one
+You may need to get at some file or binary that's in the installation
-of your dependencies.  You can do that by sub-scripting the spec:
+prefix of one of your dependencies. You can do that by sub-scripting
+the spec:
 
 .. code-block:: python
 
-   my_mpi = spec['mpi']
+   spec['mpi']
 
 The value in the brackets needs to be some package name, and spec
 needs to depend on that package, or the operation will fail.  For
 example, the above code will fail if the ``spec`` doesn't depend on
-``mpi``.  The value returned and assigned to ``my_mpi``, is itself
+``mpi``.  The value returned is itself just another ``Spec`` object,
-just another ``Spec`` object, so you can do all the same things you
+so you can do all the same things you would do with the package's
-would do with the package's own spec:
+own spec:
 
 .. code-block:: python
 
-   mpicc = join_path(my_mpi.prefix.bin, 'mpicc')
+   spec['mpi'].prefix.bin
+   spec['mpi'].version
 
 .. _multimethods:
 
@@ -3086,7 +3092,7 @@ Filtering functions
      .. code-block:: python
 
         filter_file(r'#!/usr/bin/perl',
-                    '#!/usr/bin/env perl', join_path(prefix.bin, 'bib2xhtml'))
+                    '#!/usr/bin/env perl', prefix.bin.bib2xhtml)
 
   #. Switching the compilers used by ``mpich``'s MPI wrapper scripts from
      ``cc``, etc. to the compilers used by the Spack build:
@@ -3094,10 +3100,10 @@ Filtering functions
      .. code-block:: python
 
         filter_file('CC="cc"', 'CC="%s"' % self.compiler.cc,
-                    join_path(prefix.bin, 'mpicc'))
+                    prefix.bin.mpicc)
 
         filter_file('CXX="c++"', 'CXX="%s"' % self.compiler.cxx,
-                    join_path(prefix.bin, 'mpicxx'))
+                    prefix.bin.mpicxx)
 
 :py:func:`change_sed_delimiter(old_delim, new_delim, *filenames) <spack.change_sed_delim>`
     Some packages, like TAU, have a build system that can't install
@@ -3134,12 +3140,10 @@ File functions
 
   .. code-block:: python
 
-     install('my-header.h', join_path(prefix.include))
+     install('my-header.h', prefix.include)
 
-:py:func:`join_path(prefix, *args) <spack.join_path>`
+:py:func:`join_path(*paths) <spack.join_path>`
-  Like ``os.path.join``, this joins paths using the OS path separator.
+  An alias for ``os.path.join``. This joins paths using the OS path separator.
-  However, this version allows an arbitrary number of arguments, so
-  you can string together many path components.
 
 :py:func:`mkdirp(*paths) <spack.mkdirp>`
   Create each of the directories in ``paths``, creating any parent

lib/spack/spack/package.py

@@ -1045,7 +1045,7 @@ def do_fake_install(self):
         touch(join_path(self.prefix.lib, library_name + dso_suffix))
         touch(join_path(self.prefix.lib, library_name + '.a'))
 
-        mkdirp(self.prefix.man1)
+        mkdirp(self.prefix.man.man1)
 
         packages_dir = spack.store.layout.build_packages_path(self.spec)
         dump_packages(self.spec, packages_dir)

lib/spack/spack/test/util/prefix.py

@@ -0,0 +1,66 @@
+##############################################################################
+# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""Tests various features of :py:class:`spack.util.prefix.Prefix`"""
+
+from spack.util.prefix import Prefix
+
+
+def test_prefix_attributes():
+    """Test normal prefix attributes like ``prefix.bin``"""
+    prefix = Prefix('/usr')
+
+    assert prefix.bin     == '/usr/bin'
+    assert prefix.lib     == '/usr/lib'
+    assert prefix.include == '/usr/include'
+
+
+def test_multilevel_attributes():
+    """Test attributes of attributes, like ``prefix.share.man``"""
+    prefix = Prefix('/usr/')
+
+    assert prefix.share.man   == '/usr/share/man'
+    assert prefix.man.man8    == '/usr/man/man8'
+    assert prefix.foo.bar.baz == '/usr/foo/bar/baz'
+
+    share = prefix.share
+
+    assert isinstance(share, Prefix)
+    assert share.man == '/usr/share/man'
+
+
+def test_string_like_behavior():
+    """Test string-like behavior of the prefix object"""
+    prefix = Prefix('/usr')
+
+    assert prefix == '/usr'
+    assert isinstance(prefix, str)
+
+    assert prefix + '/bin' == '/usr/bin'
+    assert '--prefix=%s' % prefix == '--prefix=/usr'
+    assert '--prefix={0}'.format(prefix) == '--prefix=/usr'
+
+    assert prefix.find('u', 1)
+    assert prefix.upper() == '/USR'
+    assert prefix.lstrip('/') == 'usr'

lib/spack/spack/util/prefix.py

@@ -23,74 +23,35 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 """
-This file contains utilities to help with installing packages.
+This file contains utilities for managing the installation prefix of a package.
 """
-from llnl.util.filesystem import join_path
+import os
 
 
 class Prefix(str):
     """This class represents an installation prefix, but provides useful
-       attributes for referring to directories inside the prefix.
+    attributes for referring to directories inside the prefix.
 
-       For example, you can do something like this::
+    Attributes of this object are created on the fly when you request them,
+    so any of the following is valid:
 
-           prefix = Prefix('/usr')
+    >>> prefix = Prefix('/usr')
-           print(prefix.lib)
+    >>> prefix.bin
-           print(prefix.lib64)
+    /usr/bin
-           print(prefix.bin)
+    >>> prefix.lib64
-           print(prefix.share)
+    /usr/lib64
-           print(prefix.man4)
+    >>> prefix.share.man
+    /usr/share/man
+    >>> prefix.foo.bar.baz
+    /usr/foo/bar/baz
 
-       This program would print:
+    Prefix objects behave identically to strings. In fact, they
+    subclass ``str``. So operators like ``+`` are legal::
 
-           /usr/lib
+        print('foobar ' + prefix)
-           /usr/lib64
-           /usr/bin
-           /usr/share
-           /usr/share/man/man4
 
-       Prefix objects behave identically to strings.  In fact, they
+    This prints ``foobar /usr``. All of this is meant to make custom
-       subclass str.  So operators like + are legal:
+    installs easy.
-
-           print("foobar " + prefix)
-
-       This prints 'foobar /usr". All of this is meant to make custom
-       installs easy.
     """
-
+    def __getattr__(self, attr):
-    def __new__(cls, path):
+        return Prefix(os.path.join(self, attr))
-        s = super(Prefix, cls).__new__(cls, path)
-        s.bin       = join_path(s, 'bin')
-        s.bin64     = join_path(s, 'bin64')
-        s.sbin      = join_path(s, 'sbin')
-        s.etc       = join_path(s, 'etc')
-        s.include   = join_path(s, 'include')
-        s.include64 = join_path(s, 'include64')
-        s.lib       = join_path(s, 'lib')
-        s.lib64     = join_path(s, 'lib64')
-        s.libexec   = join_path(s, 'libexec')
-        s.share     = join_path(s, 'share')
-        s.doc       = join_path(s.share, 'doc')
-        s.info      = join_path(s.share, 'info')
-
-        s.man  = join_path(s, 'man')
-        s.man1 = join_path(s.man, 'man1')
-        s.man2 = join_path(s.man, 'man2')
-        s.man3 = join_path(s.man, 'man3')
-        s.man4 = join_path(s.man, 'man4')
-        s.man5 = join_path(s.man, 'man5')
-        s.man6 = join_path(s.man, 'man6')
-        s.man7 = join_path(s.man, 'man7')
-        s.man8 = join_path(s.man, 'man8')
-
-        s.share_man  = join_path(s.share, 'man')
-        s.share_man1 = join_path(s.share_man, 'man1')
-        s.share_man2 = join_path(s.share_man, 'man2')
-        s.share_man3 = join_path(s.share_man, 'man3')
-        s.share_man4 = join_path(s.share_man, 'man4')
-        s.share_man5 = join_path(s.share_man, 'man5')
-        s.share_man6 = join_path(s.share_man, 'man6')
-        s.share_man7 = join_path(s.share_man, 'man7')
-        s.share_man8 = join_path(s.share_man, 'man8')
-
-        return s

var/spack/repos/builtin/packages/bwa/package.py

@@ -51,5 +51,5 @@ def install(self, spec, prefix):
         mkdirp(prefix.doc)
         install('README.md', prefix.doc)
         install('NEWS.md', prefix.doc)
-        mkdirp(prefix.man1)
+        mkdirp(prefix.man.man1)
-        install('bwa.1', prefix.man1)
+        install('bwa.1', prefix.man.man1)

var/spack/repos/builtin/packages/git/package.py

@@ -186,6 +186,6 @@ def install_manpages(self):
         prefix = self.prefix
 
         with working_dir('git-manpages'):
-            install_tree('man1', prefix.share_man1)
+            install_tree('man1', prefix.share.man.man1)
-            install_tree('man5', prefix.share_man5)
+            install_tree('man5', prefix.share.man.man5)
-            install_tree('man7', prefix.share_man7)
+            install_tree('man7', prefix.share.man.man7)

var/spack/repos/builtin/packages/libdwarf/package.py

@@ -68,7 +68,7 @@ def install(self, spec, prefix):
             make.add_default_arg('ARFLAGS=rcs')
 
             # Dwarf doesn't provide an install, so we have to do it.
-            mkdirp(prefix.bin, prefix.include, prefix.lib, prefix.man1)
+            mkdirp(prefix.bin, prefix.include, prefix.lib, prefix.man.man1)
 
             with working_dir('libdwarf'):
                 extra_config_args = []
@@ -101,4 +101,4 @@ def install(self, spec, prefix):
 
                 install('dwarfdump',      prefix.bin)
                 install('dwarfdump.conf', prefix.lib)
-                install('dwarfdump.1',    prefix.man1)
+                install('dwarfdump.1',    prefix.man.man1)

var/spack/repos/builtin/packages/mercurial/package.py

@@ -57,14 +57,14 @@ def post_install(self):
         prefix = self.prefix
 
         # Install man pages
-        mkdirp(prefix.man1)
+        mkdirp(prefix.man.man1)
-        mkdirp(prefix.man5)
+        mkdirp(prefix.man.man5)
-        mkdirp(prefix.man8)
+        mkdirp(prefix.man.man8)
         with working_dir('doc'):
-            install('hg.1', prefix.man1)
+            install('hg.1', prefix.man.man1)
-            install('hgignore.5', prefix.man5)
+            install('hgignore.5', prefix.man.man5)
-            install('hgrc.5', prefix.man5)
+            install('hgrc.5', prefix.man.man5)
-            install('hg-ssh.8', prefix.man8)
+            install('hg-ssh.8', prefix.man.man8)
 
         # Install completion scripts
         contrib = join_path(prefix, 'contrib')

var/spack/repos/builtin/packages/pigz/package.py

@@ -41,6 +41,6 @@ def build(self, spec, prefix):
 
     def install(self, spec, prefix):
         mkdirp(prefix.bin)
-        mkdirp(prefix.man1)
+        mkdirp(prefix.man.man1)
         install('pigz', "%s/pigz" % prefix.bin)
-        install('pigz.1', "%s/pigz.1" % prefix.man1)
+        install('pigz.1', "%s/pigz.1" % prefix.man.man1)

var/spack/repos/builtin/packages/scotch/package.py

@@ -249,4 +249,4 @@ def install(self, spec, prefix):
         install_tree('bin', prefix.bin)
         install_tree('lib', prefix.lib)
         install_tree('include', prefix.include)
-        install_tree('man/man1', prefix.share_man1)
+        install_tree('man/man1', prefix.share.man.man1)