resource directive : sketch of implementation + clang / llvm use case

lib/spack/llnl/util/lang.py

@@ -112,7 +112,7 @@ def partition_list(elements, predicate):
 
 def caller_locals():
     """This will return the locals of the *parent* of the caller.
-       This allows a fucntion to insert variables into its caller's
+       This allows a function to insert variables into its caller's
        scope.  Yes, this is some black magic, and yes it's useful
        for implementing things like depends_on and provides.
     """

lib/spack/spack/directives.py

@@ -44,11 +44,12 @@ class OpenMpi(Package):
   * ``variant``
 
 """
-__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version',
-            'variant' ]
+__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version',
+           'variant', 'resource']
 
 import re
 import inspect
+import functools
 
 from llnl.util.lang import *
 
@@ -60,7 +61,8 @@ class OpenMpi(Package):
 from spack.patch import Patch
 from spack.variant import Variant
 from spack.spec import Spec, parse_anonymous_spec
-
+from spack.resource import Resource
+from spack.fetch_strategy import URLFetchStrategy
 
 #
 # This is a list of all directives, built up as they are defined in
@@ -79,8 +81,8 @@ class directive(object):
     """Decorator for Spack directives.
 
     Spack directives allow you to modify a package while it is being
-    defined, e.g. to add version or depenency information.  Directives
-    are one of the key pieces of Spack's package "langauge", which is
+    defined, e.g. to add version or dependency information.  Directives
+    are one of the key pieces of Spack's package "language", which is
     embedded in python.
 
     Here's an example directive:
@@ -141,6 +143,7 @@ def ensure_dicts(self, pkg):
     def __call__(self, directive_function):
         directives[directive_function.__name__] = self
 
+        @functools.wraps(directive_function)
         def wrapped(*args, **kwargs):
             pkg = DictWrapper(caller_locals())
             self.ensure_dicts(pkg)
@@ -259,6 +262,29 @@ def variant(pkg, name, default=False, description=""):
     pkg.variants[name] = Variant(default, description)
 
 
+@directive('resources')
+def resource(pkg, **kwargs):
+    """
+    Define an external resource to be fetched and staged when building the package. Based on the keywords present in the
+    dictionary the appropriate FetchStrategy will be used for the resource.
+
+    List of recognized keywords:
+
+    * 'when' : represents the condition upon which the resource is needed (optional)
+    * 'destination' : path where to extract / checkout the resource (optional)
+
+    """
+    when = kwargs.get('when', pkg.name)
+    # FIXME : currently I assume destination to be a relative path (rooted at pkg.stage.source_path)
+    destination = kwargs.get('destination', "")
+    when_spec = parse_anonymous_spec(when, pkg.name)
+    resources = pkg.resources.setdefault(when_spec, [])
+    # FIXME : change URLFetchStrategy with a factory that selects based on kwargs
+    fetcher = URLFetchStrategy(**kwargs)
+    # FIXME : should we infer the name somehow if not passed ?
+    name = kwargs.get('name')
+    resources.append(Resource(name, fetcher, destination))
+
 class DirectiveError(spack.error.SpackError):
     """This is raised when something is wrong with a package directive."""
     def __init__(self, directive, message):

lib/spack/spack/package.py

@@ -639,26 +639,53 @@ def do_fetch(self):
                     "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg)
 
         self.stage.fetch()
+
+        ##########
+        # Fetch resources
+        resources = self._get_resources()
+        # FIXME : choose the unique name appropriately. Is there a function somewhere for the base name ?
+        pieces = [self.name, str(self.version), self.spec.dag_hash()]
+        for resource in resources:
+            resource_stage_folder = '-'.join(pieces + [resource.name])
+            stage = Stage(resource.fetcher, name=resource_stage_folder)
+            resource.fetcher.set_stage(stage)
+            resource.fetcher.fetch()
+        ##########
+
         self._fetch_time = time.time() - start_time
 
         if spack.do_checksum and self.version in self.versions:
             self.stage.check()
 
-
     def do_stage(self):
         """Unpacks the fetched tarball, then changes into the expanded tarball
            directory."""
         if not self.spec.concrete:
             raise ValueError("Can only stage concrete packages.")
 
-        self.do_fetch()
+        def _expand_archive(stage, name=self.name):
+            archive_dir = stage.source_path
+            if not archive_dir:
+                stage.expand_archive()
+                tty.msg("Created stage in %s." % stage.path)
+            else:
+                tty.msg("Already staged %s in %s." % (name, stage.path))
 
-        archive_dir = self.stage.source_path
-        if not archive_dir:
-            self.stage.expand_archive()
-            tty.msg("Created stage in %s." % self.stage.path)
-        else:
-            tty.msg("Already staged %s in %s." % (self.name, self.stage.path))
+
+        self.do_fetch()
+        _expand_archive(self.stage)
+
+        ##########
+        # Stage resources in appropriate path
+        resources = self._get_resources()
+        for resource in resources:
+            stage = resource.fetcher.stage
+            _expand_archive(stage, resource.name)
+            link_path = join_path(self.stage.source_path, resource.destination, os.path.basename(stage.source_path))
+            if not os.path.exists(link_path):
+                # Create a symlink
+                os.symlink(stage.source_path, link_path)
+        ##########
         self.stage.chdir_to_source()
 
 
@@ -730,6 +757,14 @@ def do_fake_install(self):
         mkdirp(self.prefix.man1)
 
 
+    def _get_resources(self):
+        resources = []
+        # Select the resources that are needed for this build
+        for when_spec, resource_list in self.resources.items():
+            if when_spec in self.spec:
+                resources.extend(resource_list)
+        return resources
+
     def _build_logger(self, log_path):
         """Create a context manager to log build output."""
 

lib/spack/spack/resource.py

@@ -0,0 +1,37 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""
+Describes an optional resource needed for a build. Typically a bunch of sources that can be built in-tree within another
+package to enable optional features.
+"""
+
+class Resource(object):
+    """
+    Represents an optional resource. Aggregates a name, a fetcher and a destination.
+    """
+    def __init__(self, name, fetcher, destination):
+        self.name = name
+        self.fetcher = fetcher
+        self.destination = destination