Merge pull request #208 from epfl-scitas/features/resource_directive

resource directive : implementation + clang / llvm use case

lib/spack/llnl/util/lang.py

@@ -112,7 +112,7 @@ def partition_list(elements, predicate):
 
 def caller_locals():
     """This will return the locals of the *parent* of the caller.
-       This allows a fucntion to insert variables into its caller's
+       This allows a function to insert variables into its caller's
        scope.  Yes, this is some black magic, and yes it's useful
        for implementing things like depends_on and provides.
     """

lib/spack/spack/directives.py

@@ -42,15 +42,19 @@ class OpenMpi(Package):
   * ``extends``
   * ``patch``
   * ``variant``
+  * ``resource``
 
 """
-__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version',
+__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version',
-            'variant' ]
+           'variant', 'resource']
 
 import re
 import inspect
+import os.path
+import functools
 
 from llnl.util.lang import *
+from llnl.util.filesystem import join_path
 
 import spack
 import spack.spec
@@ -60,7 +64,8 @@ class OpenMpi(Package):
 from spack.patch import Patch
 from spack.variant import Variant
 from spack.spec import Spec, parse_anonymous_spec
-
+from spack.resource import Resource
+from spack.fetch_strategy import from_kwargs
 
 #
 # This is a list of all directives, built up as they are defined in
@@ -79,8 +84,8 @@ class directive(object):
     """Decorator for Spack directives.
 
     Spack directives allow you to modify a package while it is being
-    defined, e.g. to add version or depenency information.  Directives
+    defined, e.g. to add version or dependency information.  Directives
-    are one of the key pieces of Spack's package "langauge", which is
+    are one of the key pieces of Spack's package "language", which is
     embedded in python.
 
     Here's an example directive:
@@ -141,6 +146,7 @@ def ensure_dicts(self, pkg):
     def __call__(self, directive_function):
         directives[directive_function.__name__] = self
 
+        @functools.wraps(directive_function)
         def wrapped(*args, **kwargs):
             pkg = DictWrapper(caller_locals())
             self.ensure_dicts(pkg)
@@ -259,6 +265,43 @@ def variant(pkg, name, default=False, description=""):
     pkg.variants[name] = Variant(default, description)
 
 
+@directive('resources')
+def resource(pkg, **kwargs):
+    """
+    Define an external resource to be fetched and staged when building the package. Based on the keywords present in the
+    dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their
+    own folder inside spack stage area, and then linked into the stage area of the package that needs them.
+
+    List of recognized keywords:
+
+    * 'when' : (optional) represents the condition upon which the resource is needed
+    * 'destination' : (optional) path where to link the resource. This path must be relative to the main package stage
+    area.
+    * 'placement' : (optional) gives the possibility to fine tune how the resource is linked into the main package stage
+    area.
+    """
+    when = kwargs.get('when', pkg.name)
+    destination = kwargs.get('destination', "")
+    placement = kwargs.get('placement', None)
+    # Check if the path is relative
+    if os.path.isabs(destination):
+        message = "The destination keyword of a resource directive can't be an absolute path.\n"
+        message += "\tdestination : '{dest}\n'".format(dest=destination)
+        raise RuntimeError(message)
+    # Check if the path falls within the main package stage area
+    test_path = 'stage_folder_root/'
+    normalized_destination = os.path.normpath(join_path(test_path, destination))  # Normalized absolute path
+    if test_path not in normalized_destination:
+        message = "The destination folder of a resource must fall within the main package stage directory.\n"
+        message += "\tdestination : '{dest}'\n".format(dest=destination)
+        raise RuntimeError(message)
+    when_spec = parse_anonymous_spec(when, pkg.name)
+    resources = pkg.resources.setdefault(when_spec, [])
+    fetcher = from_kwargs(**kwargs)
+    name = kwargs.get('name')
+    resources.append(Resource(name, fetcher, destination, placement))
+
+
 class DirectiveError(spack.error.SpackError):
     """This is raised when something is wrong with a package directive."""
     def __init__(self, directive, message):

lib/spack/spack/fetch_strategy.py

@@ -634,6 +634,22 @@ def from_url(url):
     return URLFetchStrategy(url)
 
 
+def from_kwargs(**kwargs):
+    """
+    Construct the appropriate FetchStrategy from the given keyword arguments.
+
+    :param kwargs: dictionary of keyword arguments
+    :return: fetcher or raise a FetchError exception
+    """
+    for fetcher in all_strategies:
+        if fetcher.matches(kwargs):
+            return fetcher(**kwargs)
+    # Raise an error in case we can't instantiate any known strategy
+    message = "Cannot instantiate any FetchStrategy"
+    long_message = message + " from the given arguments : {arguments}".format(srguments=kwargs)
+    raise FetchError(message, long_message)
+
+
 def args_are_for(args, fetcher):
     fetcher.matches(args)
 

lib/spack/spack/mirror.py

@@ -26,7 +26,7 @@
 This file contains code for creating spack mirror directories.  A
 mirror is an organized hierarchy containing specially named archive
 files.  This enabled spack to know where to find files in a mirror if
-the main server for a particualr package is down.  Or, if the computer
+the main server for a particular package is down.  Or, if the computer
 where spack is run is not connected to the internet, it allows spack
 to download packages directly from a mirror (e.g., on an intranet).
 """
@@ -42,7 +42,7 @@
 from spack.spec import Spec
 from spack.stage import Stage
 from spack.version import *
-from spack.util.compression import extension
+from spack.util.compression import extension, allowed_archive
 
 
 def mirror_archive_filename(spec):
@@ -87,11 +87,26 @@ def get_matching_versions(specs, **kwargs):
             if v.satisfies(spec.versions):
                 s = Spec(pkg.name)
                 s.versions = VersionList([v])
+                s.variants = spec.variants.copy()
                 matching.append(s)
 
     return matching
 
 
+def suggest_archive_basename(resource):
+    """
+    Return a tentative basename for an archive. Raise an exception if the name is among the allowed archive types.
+
+    :param fetcher:
+    :return:
+    """
+    basename = os.path.basename(resource.fetcher.url)
+    if not allowed_archive(basename):
+        raise RuntimeError("%s is not an allowed archive tye" % basename)
+    return basename
+
+
+
 def create(path, specs, **kwargs):
     """Create a directory to be used as a spack mirror, and fill it with
        package archives.
@@ -108,7 +123,7 @@ def create(path, specs, **kwargs):
 
        Return Value:
          Returns a tuple of lists: (present, mirrored, error)
-         * present:  Package specs that were already prsent.
+         * present:  Package specs that were already present.
          * mirrored: Package specs that were successfully mirrored.
          * error:    Package specs that failed to mirror due to some error.
 
@@ -140,6 +155,7 @@ def create(path, specs, **kwargs):
     error    = []
 
     # Iterate through packages and download all the safe tarballs for each of them
+    everything_already_exists = True
     for spec in version_specs:
         pkg = spec.package
 
@@ -152,9 +168,8 @@ def create(path, specs, **kwargs):
 
             if os.path.exists(archive_path):
                 tty.msg("Already added %s" % spec.format("$_$@"))
-                present.append(spec)
+            else:
-                continue
+                everything_already_exists = False
-
                 # Set up a stage and a fetcher for the download
                 unique_fetch_name = spec.format("$_$@")
                 fetcher = fs.for_package_version(pkg, pkg.version)
@@ -171,6 +186,28 @@ def create(path, specs, **kwargs):
                 # that to move/copy/create an archive in the mirror.
                 fetcher.archive(archive_path)
                 tty.msg("Added %s." % spec.format("$_$@"))
+
+            # Fetch resources if they are associated with the spec
+            resources = pkg._get_resources()
+            for resource in resources:
+                resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
+                if os.path.exists(resource_archive_path):
+                    tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+                    continue
+                everything_already_exists = False
+                resource_stage_folder = pkg._resource_stage(resource)
+                resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
+                resource.fetcher.set_stage(resource_stage)
+                resource.fetcher.fetch()
+                if not kwargs.get('no_checksum', False):
+                    resource.fetcher.check()
+                    tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
+                resource.fetcher.archive(resource_archive_path)
+                tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+
+            if everything_already_exists:
+                present.append(spec)
+            else:
                 mirrored.append(spec)
 
         except Exception, e:

lib/spack/spack/package.py

@@ -655,26 +655,62 @@ def do_fetch(self):
                     "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg)
 
         self.stage.fetch()
+
+        ##########
+        # Fetch resources
+        resources = self._get_resources()
+        for resource in resources:
+            resource_stage_folder = self._resource_stage(resource)
+            # FIXME : works only for URLFetchStrategy
+            resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
+            resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
+            resource.fetcher.set_stage(resource_stage)
+            # Delegate to stage object to trigger mirror logic
+            resource_stage.fetch()
+            resource_stage.check()
+        ##########
+
         self._fetch_time = time.time() - start_time
 
         if spack.do_checksum and self.version in self.versions:
             self.stage.check()
 
-
     def do_stage(self):
         """Unpacks the fetched tarball, then changes into the expanded tarball
            directory."""
         if not self.spec.concrete:
             raise ValueError("Can only stage concrete packages.")
 
-        self.do_fetch()
+        def _expand_archive(stage, name=self.name):
-
+            archive_dir = stage.source_path
-        archive_dir = self.stage.source_path
             if not archive_dir:
-            self.stage.expand_archive()
+                stage.expand_archive()
-            tty.msg("Created stage in %s." % self.stage.path)
+                tty.msg("Created stage in %s." % stage.path)
             else:
-            tty.msg("Already staged %s in %s." % (self.name, self.stage.path))
+                tty.msg("Already staged %s in %s." % (name, stage.path))
+
+
+        self.do_fetch()
+        _expand_archive(self.stage)
+
+        ##########
+        # Stage resources in appropriate path
+        resources = self._get_resources()
+        for resource in resources:
+            stage = resource.fetcher.stage
+            _expand_archive(stage, resource.name)
+            # Turn placement into a dict with relative paths
+            placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement
+            if not isinstance(placement, dict):
+                placement = {'': placement}
+            # Make the paths in the dictionary absolute and link
+            for key, value in placement.iteritems():
+                link_path = join_path(self.stage.source_path, resource.destination, value)
+                source_path = join_path(stage.source_path, key)
+                if not os.path.exists(link_path):
+                    # Create a symlink
+                    os.symlink(source_path, link_path)
+        ##########
         self.stage.chdir_to_source()
 
 
@@ -746,6 +782,19 @@ def do_fake_install(self):
         mkdirp(self.prefix.man1)
 
 
+    def _get_resources(self):
+        resources = []
+        # Select the resources that are needed for this build
+        for when_spec, resource_list in self.resources.items():
+            if when_spec in self.spec:
+                resources.extend(resource_list)
+        return resources
+
+    def _resource_stage(self, resource):
+        pieces = ['resource', resource.name, self.spec.dag_hash()]
+        resource_stage_folder = '-'.join(pieces)
+        return resource_stage_folder
+
     def _build_logger(self, log_path):
         """Create a context manager to log build output."""
 

lib/spack/spack/resource.py

@@ -0,0 +1,39 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""
+Describes an optional resource needed for a build. Typically a bunch of sources that can be built in-tree within another
+package to enable optional features.
+"""
+
+
+class Resource(object):
+    """
+    Represents an optional resource. Aggregates a name, a fetcher, a destination and a placement
+    """
+    def __init__(self, name, fetcher, destination, placement):
+        self.name = name
+        self.fetcher = fetcher
+        self.destination = destination
+        self.placement = placement

var/spack/packages/clang/package.py

@@ -22,8 +22,13 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+
+
 from spack import *
 
+import os
+import os.path
+
 class Clang(Package):
     """The goal of the Clang project is to create a new C, C++,
        Objective C and Objective C++ front-end for the LLVM compiler.
@@ -39,13 +44,52 @@ class Clang(Package):
     version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz')
     version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz')
 
+    ##########
+    # @3.7.0
+    resource(name='clang-tools-extra',
+             url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz',
+             md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0')
+    ##########
+
     def install(self, spec, prefix):
         env['CXXFLAGS'] = self.compiler.cxx11_flag
 
         with working_dir('spack-build', create=True):
+
+            options = []
+            if '@3.7.0:' in spec:
+                options.append('-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp')
+            options.extend(std_cmake_args)
+
             cmake('..',
-                  '-DCLANG_PATH_TO_LLVM_BUILD=%s' % spec['llvm'].prefix,
+                  '-DCLANG_PATH_TO_LLVM_BUILD:PATH=%s' % spec['llvm'].prefix,
-                  '-DLLVM_MAIN_SRC_DIR=%s' % spec['llvm'].prefix,
+                  '-DLLVM_MAIN_SRC_DIR:PATH=%s' % spec['llvm'].prefix,
-                  *std_cmake_args)
+                  *options)
             make()
             make("install")
+            # CLang doesn't look in llvm folders for system headers...
+            self.link_llvm_directories(spec)
+
+    def link_llvm_directories(self, spec):
+
+        def clang_include_dir_at(root):
+            return join_path(root, 'include')
+
+        def clang_lib_dir_at(root):
+            return join_path(root, 'lib/clang/', str(self.version), 'include')
+
+        def do_link(source_dir, destination_dir):
+            if os.path.exists(source_dir):
+                for name in os.listdir(source_dir):
+                    source = join_path(source_dir, name)
+                    link = join_path(destination_dir, name)
+                    os.symlink(source, link)
+
+        # Link folder and files in include
+        llvm_dir = clang_include_dir_at(spec['llvm'].prefix)
+        clang_dir = clang_include_dir_at(self.prefix)
+        do_link(llvm_dir, clang_dir)
+        # Link folder and files in lib
+        llvm_dir = clang_lib_dir_at(spec['llvm'].prefix)
+        clang_dir = clang_lib_dir_at(self.prefix)
+        do_link(llvm_dir, clang_dir)

var/spack/packages/llvm/package.py

@@ -42,13 +42,31 @@ class Llvm(Package):
 
     depends_on('python@2.7:')
 
+    variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11")
+
+    ##########
+    # @3.7.0
+    resource(name='compiler-rt',
+             url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5',
+             destination='projects', when='@3.7.0')
+    resource(name='openmp',
+             url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f',
+             destination='projects', when='@3.7.0')
+    resource(name='libcxx',
+             url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd',
+             destination='projects', placement='libcxx', when='+libcxx@3.7.0')
+    resource(name='libcxxabi',
+             url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772',
+             destination='projects', placement='libcxxabi', when='+libcxx@3.7.0')
+    ##########
+
     def install(self, spec, prefix):
         env['CXXFLAGS'] = self.compiler.cxx11_flag
 
         with working_dir('spack-build', create=True):
             cmake('..',
-                  '-DLLVM_REQUIRES_RTTI=1',
+                  '-DLLVM_REQUIRES_RTTI:BOOL=ON',
-                  '-DPYTHON_EXECUTABLE=%s/bin/python' % spec['python'].prefix,
+                  '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix,
                   *std_cmake_args)
             make()
             make("install")