zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

resource directive : sketch of implementation + clang / llvm use case

Browse Source
This commit is contained in:
Massimiliano Culpo 2015-11-26 17:53:33 +01:00
parent f8ffb005c8
commit 1fe626ec7c
6 changed files with 132 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/spack/llnl/util/lang.py
View File

@ -112,7 +112,7 @@ def partition_list(elements, predicate):
def caller_locals():
"""This will return the locals of the *parent* of the caller.
This allows a fucntion to insert variables into its caller's
This allows a function to insert variables into its caller's
scope. Yes, this is some black magic, and yes it's useful
for implementing things like depends_on and provides.
"""

36
lib/spack/spack/directives.py
View File

@ -44,11 +44,12 @@ class OpenMpi(Package):
* ``variant``
"""
__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version',
'variant' ]
__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version',
'variant', 'resource']
import re
import inspect
import functools
from llnl.util.lang import *
@ -60,7 +61,8 @@ class OpenMpi(Package):
from spack.patch import Patch
from spack.variant import Variant
from spack.spec import Spec, parse_anonymous_spec
from spack.resource import Resource
from spack.fetch_strategy import URLFetchStrategy
#
# This is a list of all directives, built up as they are defined in
@ -79,8 +81,8 @@ class directive(object):
"""Decorator for Spack directives.
Spack directives allow you to modify a package while it is being
defined, e.g. to add version or depenency information. Directives
are one of the key pieces of Spack's package "langauge", which is
defined, e.g. to add version or dependency information. Directives
are one of the key pieces of Spack's package "language", which is
embedded in python.
Here's an example directive:
@ -141,6 +143,7 @@ def ensure_dicts(self, pkg):
def __call__(self, directive_function):
directives[directive_function.__name__] = self
@functools.wraps(directive_function)
def wrapped(*args, **kwargs):
pkg = DictWrapper(caller_locals())
self.ensure_dicts(pkg)
@ -259,6 +262,29 @@ def variant(pkg, name, default=False, description=""):
pkg.variants[name] = Variant(default, description)
@directive('resources')
def resource(pkg, **kwargs):
"""
Define an external resource to be fetched and staged when building the package. Based on the keywords present in the
dictionary the appropriate FetchStrategy will be used for the resource.
List of recognized keywords:
* 'when' : represents the condition upon which the resource is needed (optional)
* 'destination' : path where to extract / checkout the resource (optional)
"""
when = kwargs.get('when', pkg.name)
# FIXME : currently I assume destination to be a relative path (rooted at pkg.stage.source_path)
destination = kwargs.get('destination', "")
when_spec = parse_anonymous_spec(when, pkg.name)
resources = pkg.resources.setdefault(when_spec, [])
# FIXME : change URLFetchStrategy with a factory that selects based on kwargs
fetcher = URLFetchStrategy(**kwargs)
# FIXME : should we infer the name somehow if not passed ?
name = kwargs.get('name')
resources.append(Resource(name, fetcher, destination))
class DirectiveError(spack.error.SpackError):
"""This is raised when something is wrong with a package directive."""
def __init__(self, directive, message):

51
lib/spack/spack/package.py
View File

@ -639,26 +639,53 @@ def do_fetch(self):
"Will not fetch %s." % self.spec.format('$_$@'), checksum_msg)
self.stage.fetch()
##########
# Fetch resources
resources = self._get_resources()
# FIXME : choose the unique name appropriately. Is there a function somewhere for the base name ?
pieces = [self.name, str(self.version), self.spec.dag_hash()]
for resource in resources:
resource_stage_folder = '-'.join(pieces + [resource.name])
stage = Stage(resource.fetcher, name=resource_stage_folder)
resource.fetcher.set_stage(stage)
resource.fetcher.fetch()
##########
self._fetch_time = time.time() - start_time
if spack.do_checksum and self.version in self.versions:
self.stage.check()
def do_stage(self):
"""Unpacks the fetched tarball, then changes into the expanded tarball
directory."""
if not self.spec.concrete:
raise ValueError("Can only stage concrete packages.")
self.do_fetch()
def _expand_archive(stage, name=self.name):
archive_dir = stage.source_path
if not archive_dir:
stage.expand_archive()
tty.msg("Created stage in %s." % stage.path)
else:
tty.msg("Already staged %s in %s." % (name, stage.path))
archive_dir = self.stage.source_path
if not archive_dir:
self.stage.expand_archive()
tty.msg("Created stage in %s." % self.stage.path)
else:
tty.msg("Already staged %s in %s." % (self.name, self.stage.path))
self.do_fetch()
_expand_archive(self.stage)
##########
# Stage resources in appropriate path
resources = self._get_resources()
for resource in resources:
stage = resource.fetcher.stage
_expand_archive(stage, resource.name)
link_path = join_path(self.stage.source_path, resource.destination, os.path.basename(stage.source_path))
if not os.path.exists(link_path):
# Create a symlink
os.symlink(stage.source_path, link_path)
##########
self.stage.chdir_to_source()
@ -730,6 +757,14 @@ def do_fake_install(self):
mkdirp(self.prefix.man1)
def _get_resources(self):
resources = []
# Select the resources that are needed for this build
for when_spec, resource_list in self.resources.items():
if when_spec in self.spec:
resources.extend(resource_list)
return resources
def _build_logger(self, log_path):
"""Create a context manager to log build output."""

37
lib/spack/spack/resource.py Normal file
View File

@ -0,0 +1,37 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""
Describes an optional resource needed for a build. Typically a bunch of sources that can be built in-tree within another
package to enable optional features.
"""
class Resource(object):
"""
Represents an optional resource. Aggregates a name, a fetcher and a destination.
"""
def __init__(self, name, fetcher, destination):
self.name = name
self.fetcher = fetcher
self.destination = destination

7
var/spack/packages/clang/package.py
View File

@ -39,6 +39,13 @@ class Clang(Package):
version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz')
version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz')
##########
# @3.7.0
resource(name='clang-tools-extra',
url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz',
md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0')
##########
def install(self, spec, prefix):
env['CXXFLAGS'] = self.compiler.cxx11_flag

15
var/spack/packages/llvm/package.py
View File

@ -41,13 +41,24 @@ class Llvm(Package):
depends_on('python@2.7:')
##########
# @3.7.0
# TODO : Add support for libc++ <- libc++ABI <- libunwind with variant?
resource(name='compiler-rt',
url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5',
destination='projects', when='@3.7.0')
resource(name='openmp',
url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f',
destination='projects', when='@3.7.0')
##########
def install(self, spec, prefix):
env['CXXFLAGS'] = self.compiler.cxx11_flag
with working_dir('spack-build', create=True):
cmake('..',
'-DLLVM_REQUIRES_RTTI=1',
'-DPYTHON_EXECUTABLE=%s/bin/python' % spec['python'].prefix,
'-DLLVM_REQUIRES_RTTI:BOOL=ON',
'-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix,
*std_cmake_args)
make()
make("install")