resource directive : sketch of implementation + clang / llvm use case

lib/spack/llnl/util/lang.py

@@ -112,7 +112,7 @@ def partition_list(elements, predicate):
 
 def caller_locals():
     """This will return the locals of the *parent* of the caller.
-       This allows a fucntion to insert variables into its caller's
+       This allows a function to insert variables into its caller's
        scope.  Yes, this is some black magic, and yes it's useful
        for implementing things like depends_on and provides.
     """

lib/spack/spack/directives.py

@@ -44,11 +44,12 @@ class OpenMpi(Package):
   * ``variant``
 
 """
-__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version',
+__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version',
-            'variant' ]
+           'variant', 'resource']
 
 import re
 import inspect
+import functools
 
 from llnl.util.lang import *
 
@@ -60,7 +61,8 @@ class OpenMpi(Package):
 from spack.patch import Patch
 from spack.variant import Variant
 from spack.spec import Spec, parse_anonymous_spec
-
+from spack.resource import Resource
+from spack.fetch_strategy import URLFetchStrategy
 
 #
 # This is a list of all directives, built up as they are defined in
@@ -79,8 +81,8 @@ class directive(object):
     """Decorator for Spack directives.
 
     Spack directives allow you to modify a package while it is being
-    defined, e.g. to add version or depenency information.  Directives
+    defined, e.g. to add version or dependency information.  Directives
-    are one of the key pieces of Spack's package "langauge", which is
+    are one of the key pieces of Spack's package "language", which is
     embedded in python.
 
     Here's an example directive:
@@ -141,6 +143,7 @@ def ensure_dicts(self, pkg):
     def __call__(self, directive_function):
         directives[directive_function.__name__] = self
 
+        @functools.wraps(directive_function)
         def wrapped(*args, **kwargs):
             pkg = DictWrapper(caller_locals())
             self.ensure_dicts(pkg)
@@ -259,6 +262,29 @@ def variant(pkg, name, default=False, description=""):
     pkg.variants[name] = Variant(default, description)
 
 
+@directive('resources')
+def resource(pkg, **kwargs):
+    """
+    Define an external resource to be fetched and staged when building the package. Based on the keywords present in the
+    dictionary the appropriate FetchStrategy will be used for the resource.
+
+    List of recognized keywords:
+
+    * 'when' : represents the condition upon which the resource is needed (optional)
+    * 'destination' : path where to extract / checkout the resource (optional)
+
+    """
+    when = kwargs.get('when', pkg.name)
+    # FIXME : currently I assume destination to be a relative path (rooted at pkg.stage.source_path)
+    destination = kwargs.get('destination', "")
+    when_spec = parse_anonymous_spec(when, pkg.name)
+    resources = pkg.resources.setdefault(when_spec, [])
+    # FIXME : change URLFetchStrategy with a factory that selects based on kwargs
+    fetcher = URLFetchStrategy(**kwargs)
+    # FIXME : should we infer the name somehow if not passed ?
+    name = kwargs.get('name')
+    resources.append(Resource(name, fetcher, destination))
+
 class DirectiveError(spack.error.SpackError):
     """This is raised when something is wrong with a package directive."""
     def __init__(self, directive, message):

lib/spack/spack/package.py

@@ -639,26 +639,53 @@ def do_fetch(self):
                     "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg)
 
         self.stage.fetch()
+
+        ##########
+        # Fetch resources
+        resources = self._get_resources()
+        # FIXME : choose the unique name appropriately. Is there a function somewhere for the base name ?
+        pieces = [self.name, str(self.version), self.spec.dag_hash()]
+        for resource in resources:
+            resource_stage_folder = '-'.join(pieces + [resource.name])
+            stage = Stage(resource.fetcher, name=resource_stage_folder)
+            resource.fetcher.set_stage(stage)
+            resource.fetcher.fetch()
+        ##########
+
         self._fetch_time = time.time() - start_time
 
         if spack.do_checksum and self.version in self.versions:
             self.stage.check()
 
-
     def do_stage(self):
         """Unpacks the fetched tarball, then changes into the expanded tarball
            directory."""
         if not self.spec.concrete:
             raise ValueError("Can only stage concrete packages.")
 
-        self.do_fetch()
+        def _expand_archive(stage, name=self.name):
-
+            archive_dir = stage.source_path
-        archive_dir = self.stage.source_path
             if not archive_dir:
-            self.stage.expand_archive()
+                stage.expand_archive()
-            tty.msg("Created stage in %s." % self.stage.path)
+                tty.msg("Created stage in %s." % stage.path)
             else:
-            tty.msg("Already staged %s in %s." % (self.name, self.stage.path))
+                tty.msg("Already staged %s in %s." % (name, stage.path))
+
+
+        self.do_fetch()
+        _expand_archive(self.stage)
+
+        ##########
+        # Stage resources in appropriate path
+        resources = self._get_resources()
+        for resource in resources:
+            stage = resource.fetcher.stage
+            _expand_archive(stage, resource.name)
+            link_path = join_path(self.stage.source_path, resource.destination, os.path.basename(stage.source_path))
+            if not os.path.exists(link_path):
+                # Create a symlink
+                os.symlink(stage.source_path, link_path)
+        ##########
         self.stage.chdir_to_source()
 
 
@@ -730,6 +757,14 @@ def do_fake_install(self):
         mkdirp(self.prefix.man1)
 
 
+    def _get_resources(self):
+        resources = []
+        # Select the resources that are needed for this build
+        for when_spec, resource_list in self.resources.items():
+            if when_spec in self.spec:
+                resources.extend(resource_list)
+        return resources
+
     def _build_logger(self, log_path):
         """Create a context manager to log build output."""
 

lib/spack/spack/resource.py

@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""
+Describes an optional resource needed for a build. Typically a bunch of sources that can be built in-tree within another
+package to enable optional features.
+"""
+
+class Resource(object):
+    """
+    Represents an optional resource. Aggregates a name, a fetcher and a destination.
+    """
+    def __init__(self, name, fetcher, destination):
+        self.name = name
+        self.fetcher = fetcher
+        self.destination = destination

var/spack/packages/clang/package.py

@@ -39,6 +39,13 @@ class Clang(Package):
     version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz')
     version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz')
 
+    ##########
+    # @3.7.0
+    resource(name='clang-tools-extra',
+             url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz',
+             md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0')
+    ##########
+
     def install(self, spec, prefix):
         env['CXXFLAGS'] = self.compiler.cxx11_flag
 

var/spack/packages/llvm/package.py

@@ -41,13 +41,24 @@ class Llvm(Package):
 
     depends_on('python@2.7:')
 
+    ##########
+    # @3.7.0
+    # TODO : Add support for libc++ <- libc++ABI <- libunwind with variant?
+    resource(name='compiler-rt',
+             url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5',
+             destination='projects', when='@3.7.0')
+    resource(name='openmp',
+             url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f',
+             destination='projects', when='@3.7.0')
+    ##########
+
     def install(self, spec, prefix):
         env['CXXFLAGS'] = self.compiler.cxx11_flag
 
         with working_dir('spack-build', create=True):
             cmake('..',
-                  '-DLLVM_REQUIRES_RTTI=1',
+                  '-DLLVM_REQUIRES_RTTI:BOOL=ON',
-                  '-DPYTHON_EXECUTABLE=%s/bin/python' % spec['python'].prefix,
+                  '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix,
                   *std_cmake_args)
             make()
             make("install")