zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #298 from epfl-scitas/refactoring/resource_directive

Browse Source 
refactoring proposal : composite for Stage and FetchStrategy
This commit is contained in:
Todd Gamblin 2016-02-18 23:13:12 -08:00
commit 3c1aa9a4ad
11 changed files with 461 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/spack/spack/directives.py
View File

@ -296,8 +296,8 @@ def resource(pkg, **kwargs):
raise RuntimeError(message) raise RuntimeError(message)
when_spec = parse_anonymous_spec(when, pkg.name) when_spec = parse_anonymous_spec(when, pkg.name)
resources = pkg.resources.setdefault(when_spec, []) resources = pkg.resources.setdefault(when_spec, [])
fetcher = from_kwargs(**kwargs)
name = kwargs.get('name') name = kwargs.get('name')
fetcher = from_kwargs(**kwargs)
resources.append(Resource(name, fetcher, destination, placement)) resources.append(Resource(name, fetcher, destination, placement))

131
lib/spack/spack/fetch_strategy.py
View File

@ -44,6 +44,7 @@
import sys import sys
import re import re
import shutil import shutil
import copy
from functools import wraps from functools import wraps
import llnl.util.tty as tty import llnl.util.tty as tty
from llnl.util.filesystem import * from llnl.util.filesystem import *
@ -55,53 +56,59 @@
from spack.version import Version, ver from spack.version import Version, ver
from spack.util.compression import decompressor_for, extension from spack.util.compression import decompressor_for, extension
import spack.util.pattern as pattern
"""List of all fetch strategies, created by FetchStrategy metaclass.""" """List of all fetch strategies, created by FetchStrategy metaclass."""
all_strategies = [] all_strategies = []
def _needs_stage(fun): def _needs_stage(fun):
"""Many methods on fetch strategies require a stage to be set """Many methods on fetch strategies require a stage to be set
using set_stage(). This decorator adds a check for self.stage.""" using set_stage(). This decorator adds a check for self.stage."""
@wraps(fun) @wraps(fun)
def wrapper(self, *args, **kwargs): def wrapper(self, *args, **kwargs):
if not self.stage: if not self.stage:
raise NoStageError(fun) raise NoStageError(fun)
return fun(self, *args, **kwargs) return fun(self, *args, **kwargs)
return wrapper return wrapper
class FetchStrategy(object): class FetchStrategy(object):
"""Superclass of all fetch strategies.""" """Superclass of all fetch strategies."""
enabled = False # Non-abstract subclasses should be enabled. enabled = False # Non-abstract subclasses should be enabled.
required_attributes = None # Attributes required in version() args. required_attributes = None # Attributes required in version() args.
class __metaclass__(type): class __metaclass__(type):
"""This metaclass registers all fetch strategies in a list.""" """This metaclass registers all fetch strategies in a list."""
def __init__(cls, name, bases, dict): def __init__(cls, name, bases, dict):
type.__init__(cls, name, bases, dict) type.__init__(cls, name, bases, dict)
if cls.enabled: all_strategies.append(cls) if cls.enabled: all_strategies.append(cls)
def __init__(self): def __init__(self):
# The stage is initialized late, so that fetch strategies can be constructed # The stage is initialized late, so that fetch strategies can be constructed
# at package construction time. This is where things will be fetched. # at package construction time. This is where things will be fetched.
self.stage = None self.stage = None
def set_stage(self, stage): def set_stage(self, stage):
"""This is called by Stage before any of the fetching """This is called by Stage before any of the fetching
methods are called on the stage.""" methods are called on the stage."""
self.stage = stage self.stage = stage
# Subclasses need to implement these methods # Subclasses need to implement these methods
def fetch(self): pass # Return True on success, False on fail. def fetch(self): pass # Return True on success, False on fail.
def check(self): pass # Do checksum. def check(self): pass # Do checksum.
def expand(self): pass # Expand archive. def expand(self): pass # Expand archive.
def reset(self): pass # Revert to freshly downloaded state. def reset(self): pass # Revert to freshly downloaded state.
def archive(self, destination): pass # Used to create tarball for mirror. def archive(self, destination): pass # Used to create tarball for mirror.
def __str__(self): # Should be human readable URL. def __str__(self): # Should be human readable URL.
return "FetchStrategy.__str___" return "FetchStrategy.__str___"
# This method is used to match fetch strategies to version() # This method is used to match fetch strategies to version()
@ -111,6 +118,15 @@ def matches(cls, args):
return any(k in args for k in cls.required_attributes) return any(k in args for k in cls.required_attributes)
@pattern.composite(interface=FetchStrategy)
class FetchStrategyComposite(object):
"""
Composite for a FetchStrategy object. Implements the GoF composite pattern.
"""
matches = FetchStrategy.matches
set_stage = FetchStrategy.set_stage
class URLFetchStrategy(FetchStrategy): class URLFetchStrategy(FetchStrategy):
"""FetchStrategy that pulls source code from a URL for an archive, """FetchStrategy that pulls source code from a URL for an archive,
checks the archive against a checksum,and decompresses the archive. checks the archive against a checksum,and decompresses the archive.
@ -142,15 +158,15 @@ def fetch(self):
tty.msg("Trying to fetch from %s" % self.url) tty.msg("Trying to fetch from %s" % self.url)
curl_args = ['-O', # save file to disk curl_args = ['-O', # save file to disk
'-f', # fail on >400 errors '-f', # fail on >400 errors
'-D', '-', # print out HTML headers '-D', '-', # print out HTML headers
'-L', self.url,] '-L', self.url, ]
if sys.stdout.isatty(): if sys.stdout.isatty():
curl_args.append('-#') # status bar when using a tty curl_args.append('-#') # status bar when using a tty
else: else:
curl_args.append('-sS') # just errors when not. curl_args.append('-sS') # just errors when not.
# Run curl but grab the mime type from the http headers # Run curl but grab the mime type from the http headers
headers = spack.curl( headers = spack.curl(
@ -164,24 +180,23 @@ def fetch(self):
if spack.curl.returncode == 22: if spack.curl.returncode == 22:
# This is a 404. Curl will print the error. # This is a 404. Curl will print the error.
raise FailedDownloadError( raise FailedDownloadError(
self.url, "URL %s was not found!" % self.url) self.url, "URL %s was not found!" % self.url)
elif spack.curl.returncode == 60: elif spack.curl.returncode == 60:
# This is a certificate error. Suggest spack -k # This is a certificate error. Suggest spack -k
raise FailedDownloadError( raise FailedDownloadError(
self.url, self.url,
"Curl was unable to fetch due to invalid certificate. " "Curl was unable to fetch due to invalid certificate. "
"This is either an attack, or your cluster's SSL configuration " "This is either an attack, or your cluster's SSL configuration "
"is bad. If you believe your SSL configuration is bad, you " "is bad. If you believe your SSL configuration is bad, you "
"can try running spack -k, which will not check SSL certificates." "can try running spack -k, which will not check SSL certificates."
"Use this at your own risk.") "Use this at your own risk.")
else: else:
# This is some other curl error. Curl will print the # This is some other curl error. Curl will print the
# error, but print a spack message too # error, but print a spack message too
raise FailedDownloadError( raise FailedDownloadError(
self.url, "Curl failed with error %d" % spack.curl.returncode) self.url, "Curl failed with error %d" % spack.curl.returncode)
# Check if we somehow got an HTML file rather than the archive we # Check if we somehow got an HTML file rather than the archive we
# asked for. We only look at the last content type, to handle # asked for. We only look at the last content type, to handle
@ -196,7 +211,6 @@ def fetch(self):
if not self.archive_file: if not self.archive_file:
raise FailedDownloadError(self.url) raise FailedDownloadError(self.url)
@property @property
def archive_file(self): def archive_file(self):
"""Path to the source archive within this stage directory.""" """Path to the source archive within this stage directory."""
@ -209,7 +223,7 @@ def expand(self):
self.stage.chdir() self.stage.chdir()
if not self.archive_file: if not self.archive_file:
raise NoArchiveFileError("URLFetchStrategy couldn't find archive file", raise NoArchiveFileError("URLFetchStrategy couldn't find archive file",
"Failed on expand() for URL %s" % self.url) "Failed on expand() for URL %s" % self.url)
decompress = decompressor_for(self.archive_file) decompress = decompressor_for(self.archive_file)
@ -241,7 +255,6 @@ def expand(self):
# Set the wd back to the stage when done. # Set the wd back to the stage when done.
self.stage.chdir() self.stage.chdir()
def archive(self, destination): def archive(self, destination):
"""Just moves this archive to the destination.""" """Just moves this archive to the destination."""
if not self.archive_file: if not self.archive_file:
@ -252,7 +265,6 @@ def archive(self, destination):
shutil.move(self.archive_file, destination) shutil.move(self.archive_file, destination)
@_needs_stage @_needs_stage
def check(self): def check(self):
"""Check the downloaded archive against a checksum digest. """Check the downloaded archive against a checksum digest.
@ -263,9 +275,8 @@ def check(self):
checker = crypto.Checker(self.digest) checker = crypto.Checker(self.digest)
if not checker.check(self.archive_file): if not checker.check(self.archive_file):
raise ChecksumError( raise ChecksumError(
"%s checksum failed for %s." % (checker.hash_name, self.archive_file), "%s checksum failed for %s." % (checker.hash_name, self.archive_file),
"Expected %s but got %s." % (self.digest, checker.sum)) "Expected %s but got %s." % (self.digest, checker.sum))
@_needs_stage @_needs_stage
def reset(self): def reset(self):
@ -277,12 +288,10 @@ def reset(self):
shutil.rmtree(self.stage.source_path, ignore_errors=True) shutil.rmtree(self.stage.source_path, ignore_errors=True)
self.expand() self.expand()
def __repr__(self): def __repr__(self):
url = self.url if self.url else "no url" url = self.url if self.url else "no url"
return "URLFetchStrategy<%s>" % url return "URLFetchStrategy<%s>" % url
def __str__(self): def __str__(self):
if self.url: if self.url:
return self.url return self.url
@ -298,33 +307,30 @@ def __init__(self, name, *rev_types, **kwargs):
# Set a URL based on the type of fetch strategy. # Set a URL based on the type of fetch strategy.
self.url = kwargs.get(name, None) self.url = kwargs.get(name, None)
if not self.url: raise ValueError( if not self.url: raise ValueError(
"%s requires %s argument." % (self.__class__, name)) "%s requires %s argument." % (self.__class__, name))
# Ensure that there's only one of the rev_types # Ensure that there's only one of the rev_types
if sum(k in kwargs for k in rev_types) > 1: if sum(k in kwargs for k in rev_types) > 1:
raise FetchStrategyError( raise FetchStrategyError(
"Supply only one of %s to fetch with %s." % ( "Supply only one of %s to fetch with %s." % (
comma_or(rev_types), name)) comma_or(rev_types), name))
# Set attributes for each rev type. # Set attributes for each rev type.
for rt in rev_types: for rt in rev_types:
setattr(self, rt, kwargs.get(rt, None)) setattr(self, rt, kwargs.get(rt, None))
@_needs_stage @_needs_stage
def check(self): def check(self):
tty.msg("No checksum needed when fetching with %s." % self.name) tty.msg("No checksum needed when fetching with %s." % self.name)
@_needs_stage @_needs_stage
def expand(self): def expand(self):
tty.debug("Source fetched with %s is already expanded." % self.name) tty.debug("Source fetched with %s is already expanded." % self.name)
@_needs_stage @_needs_stage
def archive(self, destination, **kwargs): def archive(self, destination, **kwargs):
assert(extension(destination) == 'tar.gz') assert (extension(destination) == 'tar.gz')
assert(self.stage.source_path.startswith(self.stage.path)) assert (self.stage.source_path.startswith(self.stage.path))
tar = which('tar', required=True) tar = which('tar', required=True)
@ -338,16 +344,13 @@ def archive(self, destination, **kwargs):
self.stage.chdir() self.stage.chdir()
tar('-czf', destination, os.path.basename(self.stage.source_path)) tar('-czf', destination, os.path.basename(self.stage.source_path))
def __str__(self): def __str__(self):
return "VCS: %s" % self.url return "VCS: %s" % self.url
def __repr__(self): def __repr__(self):
return "%s<%s>" % (self.__class__, self.url) return "%s<%s>" % (self.__class__, self.url)
class GitFetchStrategy(VCSFetchStrategy): class GitFetchStrategy(VCSFetchStrategy):
"""Fetch strategy that gets source code from a git repository. """Fetch strategy that gets source code from a git repository.
Use like this in a package: Use like this in a package:
@ -368,24 +371,25 @@ class GitFetchStrategy(VCSFetchStrategy):
required_attributes = ('git',) required_attributes = ('git',)
def __init__(self, **kwargs): def __init__(self, **kwargs):
super(GitFetchStrategy, self).__init__( # Discards the keywords in kwargs that may conflict with the next call to __init__
'git', 'tag', 'branch', 'commit', **kwargs) forwarded_args = copy.copy(kwargs)
self._git = None forwarded_args.pop('name', None)
super(GitFetchStrategy, self).__init__(
'git', 'tag', 'branch', 'commit', **forwarded_args)
self._git = None
@property @property
def git_version(self): def git_version(self):
vstring = self.git('--version', output=str).lstrip('git version ') vstring = self.git('--version', output=str).lstrip('git version ')
return Version(vstring) return Version(vstring)
@property @property
def git(self): def git(self):
if not self._git: if not self._git:
self._git = which('git', required=True) self._git = which('git', required=True)
return self._git return self._git
@_needs_stage @_needs_stage
def fetch(self): def fetch(self):
self.stage.chdir() self.stage.chdir()
@ -418,7 +422,7 @@ def fetch(self):
if self.branch: if self.branch:
args.extend(['--branch', self.branch]) args.extend(['--branch', self.branch])
elif self.tag and self.git_version >= ver('1.8.5.2'): elif self.tag and self.git_version >= ver('1.8.5.2'):
args.extend(['--branch', self.tag]) args.extend(['--branch', self.tag])
# Try to be efficient if we're using a new enough git. # Try to be efficient if we're using a new enough git.
# This checks out only one branch's history # This checks out only one branch's history
@ -429,7 +433,7 @@ def fetch(self):
# Yet more efficiency, only download a 1-commit deep tree # Yet more efficiency, only download a 1-commit deep tree
if self.git_version >= ver('1.7.1'): if self.git_version >= ver('1.7.1'):
try: try:
self.git(*(args + ['--depth','1', self.url])) self.git(*(args + ['--depth', '1', self.url]))
cloned = True cloned = True
except spack.error.SpackError: except spack.error.SpackError:
# This will fail with the dumb HTTP transport # This will fail with the dumb HTTP transport
@ -452,18 +456,15 @@ def fetch(self):
self.git('pull', '--tags', ignore_errors=1) self.git('pull', '--tags', ignore_errors=1)
self.git('checkout', self.tag) self.git('checkout', self.tag)
def archive(self, destination): def archive(self, destination):
super(GitFetchStrategy, self).archive(destination, exclude='.git') super(GitFetchStrategy, self).archive(destination, exclude='.git')
@_needs_stage @_needs_stage
def reset(self): def reset(self):
self.stage.chdir_to_source() self.stage.chdir_to_source()
self.git('checkout', '.') self.git('checkout', '.')
self.git('clean', '-f') self.git('clean', '-f')
def __str__(self): def __str__(self):
return "[git] %s" % self.url return "[git] %s" % self.url
@ -483,20 +484,22 @@ class SvnFetchStrategy(VCSFetchStrategy):
required_attributes = ['svn'] required_attributes = ['svn']
def __init__(self, **kwargs): def __init__(self, **kwargs):
# Discards the keywords in kwargs that may conflict with the next call to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(SvnFetchStrategy, self).__init__( super(SvnFetchStrategy, self).__init__(
'svn', 'revision', **kwargs) 'svn', 'revision', **forwarded_args)
self._svn = None self._svn = None
if self.revision is not None: if self.revision is not None:
self.revision = str(self.revision) self.revision = str(self.revision)
@property @property
def svn(self): def svn(self):
if not self._svn: if not self._svn:
self._svn = which('svn', required=True) self._svn = which('svn', required=True)
return self._svn return self._svn
@_needs_stage @_needs_stage
def fetch(self): def fetch(self):
self.stage.chdir() self.stage.chdir()
@ -515,7 +518,6 @@ def fetch(self):
self.svn(*args) self.svn(*args)
self.stage.chdir_to_source() self.stage.chdir_to_source()
def _remove_untracked_files(self): def _remove_untracked_files(self):
"""Removes untracked files in an svn repository.""" """Removes untracked files in an svn repository."""
status = self.svn('status', '--no-ignore', output=str) status = self.svn('status', '--no-ignore', output=str)
@ -529,23 +531,19 @@ def _remove_untracked_files(self):
elif os.path.isdir(path): elif os.path.isdir(path):
shutil.rmtree(path, ignore_errors=True) shutil.rmtree(path, ignore_errors=True)
def archive(self, destination): def archive(self, destination):
super(SvnFetchStrategy, self).archive(destination, exclude='.svn') super(SvnFetchStrategy, self).archive(destination, exclude='.svn')
@_needs_stage @_needs_stage
def reset(self): def reset(self):
self.stage.chdir_to_source() self.stage.chdir_to_source()
self._remove_untracked_files() self._remove_untracked_files()
self.svn('revert', '.', '-R') self.svn('revert', '.', '-R')
def __str__(self): def __str__(self):
return "[svn] %s" % self.url return "[svn] %s" % self.url
class HgFetchStrategy(VCSFetchStrategy): class HgFetchStrategy(VCSFetchStrategy):
"""Fetch strategy that gets source code from a Mercurial repository. """Fetch strategy that gets source code from a Mercurial repository.
Use like this in a package: Use like this in a package:
@ -567,10 +565,13 @@ class HgFetchStrategy(VCSFetchStrategy):
required_attributes = ['hg'] required_attributes = ['hg']
def __init__(self, **kwargs): def __init__(self, **kwargs):
super(HgFetchStrategy, self).__init__( # Discards the keywords in kwargs that may conflict with the next call to __init__
'hg', 'revision', **kwargs) forwarded_args = copy.copy(kwargs)
self._hg = None forwarded_args.pop('name', None)
super(HgFetchStrategy, self).__init__(
'hg', 'revision', **forwarded_args)
self._hg = None
@property @property
def hg(self): def hg(self):
@ -597,11 +598,9 @@ def fetch(self):
self.hg(*args) self.hg(*args)
def archive(self, destination): def archive(self, destination):
super(HgFetchStrategy, self).archive(destination, exclude='.hg') super(HgFetchStrategy, self).archive(destination, exclude='.hg')
@_needs_stage @_needs_stage
def reset(self): def reset(self):
self.stage.chdir() self.stage.chdir()
@ -619,7 +618,6 @@ def reset(self):
shutil.move(scrubbed, source_path) shutil.move(scrubbed, source_path)
self.stage.chdir_to_source() self.stage.chdir_to_source()
def __str__(self): def __str__(self):
return "[hg] %s" % self.url return "[hg] %s" % self.url
@ -693,9 +691,10 @@ def __init__(self, msg, long_msg=None):
class FailedDownloadError(FetchError): class FailedDownloadError(FetchError):
"""Raised wen a download fails.""" """Raised wen a download fails."""
def __init__(self, url, msg=""): def __init__(self, url, msg=""):
super(FailedDownloadError, self).__init__( super(FailedDownloadError, self).__init__(
"Failed to fetch file from URL: %s" % url, msg) "Failed to fetch file from URL: %s" % url, msg)
self.url = url self.url = url
@ -718,12 +717,14 @@ def __init__(self, pkg, version):
class ChecksumError(FetchError): class ChecksumError(FetchError):
"""Raised when archive fails to checksum.""" """Raised when archive fails to checksum."""
def __init__(self, message, long_msg=None): def __init__(self, message, long_msg=None):
super(ChecksumError, self).__init__(message, long_msg) super(ChecksumError, self).__init__(message, long_msg)
class NoStageError(FetchError): class NoStageError(FetchError):
"""Raised when fetch operations are called before set_stage().""" """Raised when fetch operations are called before set_stage()."""
def __init__(self, method): def __init__(self, method):
super(NoStageError, self).__init__( super(NoStageError, self).__init__(
"Must call FetchStrategy.set_stage() before calling %s" % method.__name__) "Must call FetchStrategy.set_stage() before calling %s" % method.__name__)

82
lib/spack/spack/mirror.py
View File

@ -45,12 +45,11 @@
from spack.util.compression import extension, allowed_archive from spack.util.compression import extension, allowed_archive
def mirror_archive_filename(spec): def mirror_archive_filename(spec, fetcher):
"""Get the name of the spec's archive in the mirror.""" """Get the name of the spec's archive in the mirror."""
if not spec.version.concrete: if not spec.version.concrete:
raise ValueError("mirror.path requires spec with concrete version.") raise ValueError("mirror.path requires spec with concrete version.")
fetcher = spec.package.fetcher
if isinstance(fetcher, fs.URLFetchStrategy): if isinstance(fetcher, fs.URLFetchStrategy):
# If we fetch this version with a URLFetchStrategy, use URL's archive type # If we fetch this version with a URLFetchStrategy, use URL's archive type
ext = url.downloaded_file_extension(fetcher.url) ext = url.downloaded_file_extension(fetcher.url)
@ -61,9 +60,9 @@ def mirror_archive_filename(spec):
return "%s-%s.%s" % (spec.package.name, spec.version, ext) return "%s-%s.%s" % (spec.package.name, spec.version, ext)
def mirror_archive_path(spec): def mirror_archive_path(spec, fetcher):
"""Get the relative path to the spec's archive within a mirror.""" """Get the relative path to the spec's archive within a mirror."""
return join_path(spec.name, mirror_archive_filename(spec)) return join_path(spec.name, mirror_archive_filename(spec, fetcher))
def get_matching_versions(specs, **kwargs): def get_matching_versions(specs, **kwargs):
@ -167,72 +166,47 @@ def create(path, specs, **kwargs):
everything_already_exists = True everything_already_exists = True
for spec in version_specs: for spec in version_specs:
pkg = spec.package pkg = spec.package
tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@")))
stage = None
try: try:
# create a subdirectory for the current package@version for ii, stage in enumerate(pkg.stage):
archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec))) fetcher = stage.fetcher
subdir = os.path.dirname(archive_path) if ii == 0:
try: # create a subdirectory for the current package@version
archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher)))
name = spec.format("$_$@")
else:
resource = stage.resource
archive_path = join_path(subdir, suggest_archive_basename(resource))
name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@"))
subdir = os.path.dirname(archive_path)
mkdirp(subdir) mkdirp(subdir)
except OSError as e:
raise MirrorError(
"Cannot create directory '%s':" % subdir, str(e))
if os.path.exists(archive_path): if os.path.exists(archive_path):
tty.msg("Already added %s" % spec.format("$_$@")) tty.msg("{name} : already added".format(name=name))
else: else:
everything_already_exists = False everything_already_exists = False
# Set up a stage and a fetcher for the download fetcher.fetch()
unique_fetch_name = spec.format("$_$@") if not kwargs.get('no_checksum', False):
fetcher = fs.for_package_version(pkg, pkg.version) fetcher.check()
stage = Stage(fetcher, name=unique_fetch_name) tty.msg("{name} : checksum passed".format(name=name))
fetcher.set_stage(stage)
# Do the fetch and checksum if necessary # Fetchers have to know how to archive their files. Use
fetcher.fetch() # that to move/copy/create an archive in the mirror.
if not kwargs.get('no_checksum', False): fetcher.archive(archive_path)
fetcher.check() tty.msg("{name} : added".format(name=name))
tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
# Fetchers have to know how to archive their files. Use
# that to move/copy/create an archive in the mirror.
fetcher.archive(archive_path)
tty.msg("Added %s." % spec.format("$_$@"))
# Fetch resources if they are associated with the spec
resources = pkg._get_resources()
for resource in resources:
resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
if os.path.exists(resource_archive_path):
tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
continue
everything_already_exists = False
resource_stage_folder = pkg._resource_stage(resource)
resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
resource.fetcher.set_stage(resource_stage)
resource.fetcher.fetch()
if not kwargs.get('no_checksum', False):
resource.fetcher.check()
tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
resource.fetcher.archive(resource_archive_path)
tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
if everything_already_exists: if everything_already_exists:
present.append(spec) present.append(spec)
else: else:
mirrored.append(spec) mirrored.append(spec)
except Exception, e: except Exception, e:
if spack.debug: if spack.debug:
sys.excepthook(*sys.exc_info()) sys.excepthook(*sys.exc_info())
else: else:
tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message)
error.append(spec) error.append(spec)
finally: finally:
if stage: pkg.stage.destroy()
stage.destroy()
return (present, mirrored, error) return (present, mirrored, error)

128
lib/spack/spack/package.py
View File

@ -63,7 +63,7 @@
import spack.util.web import spack.util.web
import spack.fetch_strategy as fs import spack.fetch_strategy as fs
from spack.version import * from spack.version import *
from spack.stage import Stage from spack.stage import Stage, ResourceStage, StageComposite
from spack.util.compression import allowed_archive, extension from spack.util.compression import allowed_archive, extension
from spack.util.executable import ProcessError from spack.util.executable import ProcessError
@ -433,23 +433,46 @@ def url_for_version(self, version):
return spack.url.substitute_version(self.nearest_url(version), return spack.url.substitute_version(self.nearest_url(version),
self.url_version(version)) self.url_version(version))
def _make_resource_stage(self, root_stage, fetcher, resource):
resource_stage_folder = self._resource_stage(resource)
resource_mirror = join_path(self.name, os.path.basename(fetcher.url))
stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource,
name=resource_stage_folder, mirror_path=resource_mirror)
return stage
def _make_root_stage(self, fetcher):
# Construct a mirror path (TODO: get this out of package.py)
mp = spack.mirror.mirror_archive_path(self.spec, fetcher)
# Construct a path where the stage should build..
s = self.spec
stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
# Build the composite stage
stage = Stage(fetcher, mirror_path=mp, name=stage_name)
return stage
def _make_stage(self):
# Construct a composite stage on top of the composite FetchStrategy
composite_fetcher = self.fetcher
composite_stage = StageComposite()
resources = self._get_resources()
for ii, fetcher in enumerate(composite_fetcher):
if ii == 0:
# Construct root stage first
stage = self._make_root_stage(fetcher)
else:
# Construct resource stage
resource = resources[ii - 1] # ii == 0 is root!
stage = self._make_resource_stage(composite_stage[0], fetcher, resource)
# Append the item to the composite
composite_stage.append(stage)
return composite_stage
@property @property
def stage(self): def stage(self):
if not self.spec.concrete: if not self.spec.concrete:
raise ValueError("Can only get a stage for a concrete package.") raise ValueError("Can only get a stage for a concrete package.")
if self._stage is None: if self._stage is None:
# Construct a mirror path (TODO: get this out of package.py) self._stage = self._make_stage()
mp = spack.mirror.mirror_archive_path(self.spec)
# Construct a path where the stage should build..
s = self.spec
stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
# Build the stage
self._stage = Stage(self.fetcher, mirror_path=mp, name=stage_name)
return self._stage return self._stage
@ -459,17 +482,25 @@ def stage(self, stage):
self._stage = stage self._stage = stage
def _make_fetcher(self):
# Construct a composite fetcher that always contains at least one element (the root package). In case there
# are resources associated with the package, append their fetcher to the composite.
root_fetcher = fs.for_package_version(self, self.version)
fetcher = fs.FetchStrategyComposite() # Composite fetcher
fetcher.append(root_fetcher) # Root fetcher is always present
resources = self._get_resources()
for resource in resources:
fetcher.append(resource.fetcher)
return fetcher
@property @property
def fetcher(self): def fetcher(self):
if not self.spec.versions.concrete: if not self.spec.versions.concrete:
raise ValueError( raise ValueError("Can only get a fetcher for a package with concrete versions.")
"Can only get a fetcher for a package with concrete versions.")
if not self._fetcher: if not self._fetcher:
self._fetcher = fs.for_package_version(self, self.version) self._fetcher = self._make_fetcher()
return self._fetcher return self._fetcher
@fetcher.setter @fetcher.setter
def fetcher(self, f): def fetcher(self, f):
self._fetcher = f self._fetcher = f
@ -632,7 +663,7 @@ def remove_prefix(self):
def do_fetch(self, mirror_only=False): def do_fetch(self, mirror_only=False):
"""Creates a stage directory and downloads the taball for this package. """Creates a stage directory and downloads the tarball for this package.
Working directory will be set to the stage directory. Working directory will be set to the stage directory.
""" """
if not self.spec.concrete: if not self.spec.concrete:
@ -658,20 +689,6 @@ def do_fetch(self, mirror_only=False):
self.stage.fetch(mirror_only) self.stage.fetch(mirror_only)
##########
# Fetch resources
resources = self._get_resources()
for resource in resources:
resource_stage_folder = self._resource_stage(resource)
# FIXME : works only for URLFetchStrategy
resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
resource.fetcher.set_stage(resource_stage)
# Delegate to stage object to trigger mirror logic
resource_stage.fetch()
resource_stage.check()
##########
self._fetch_time = time.time() - start_time self._fetch_time = time.time() - start_time
if spack.do_checksum and self.version in self.versions: if spack.do_checksum and self.version in self.versions:
@ -684,52 +701,10 @@ def do_stage(self, mirror_only=False):
if not self.spec.concrete: if not self.spec.concrete:
raise ValueError("Can only stage concrete packages.") raise ValueError("Can only stage concrete packages.")
def _expand_archive(stage, name=self.name):
archive_dir = stage.source_path
if not archive_dir:
stage.expand_archive()
tty.msg("Created stage in %s." % stage.path)
else:
tty.msg("Already staged %s in %s." % (name, stage.path))
self.do_fetch(mirror_only) self.do_fetch(mirror_only)
_expand_archive(self.stage) self.stage.expand_archive()
##########
# Stage resources in appropriate path
resources = self._get_resources()
# TODO: this is to allow nested resources, a better solution would be
# good
for resource in sorted(resources, key=lambda res: len(res.destination)):
stage = resource.fetcher.stage
_expand_archive(stage, resource.name)
# Turn placement into a dict with relative paths
placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement
if not isinstance(placement, dict):
placement = {'': placement}
# Make the paths in the dictionary absolute and link
for key, value in placement.iteritems():
target_path = join_path(self.stage.source_path, resource.destination)
link_path = join_path(target_path, value)
source_path = join_path(stage.source_path, key)
try:
os.makedirs(target_path)
except OSError as err:
if err.errno == errno.EEXIST and os.path.isdir(target_path):
pass
else: raise
# NOTE: a reasonable fix for the TODO above might be to have
# these expand in place, but expand_archive does not offer
# this
if not os.path.exists(link_path):
shutil.move(source_path, link_path)
##########
self.stage.chdir_to_source() self.stage.chdir_to_source()
def do_patch(self): def do_patch(self):
"""Calls do_stage(), then applied patches to the expanded tarball if they """Calls do_stage(), then applied patches to the expanded tarball if they
haven't been applied already.""" haven't been applied already."""
@ -828,6 +803,9 @@ def _get_resources(self):
for when_spec, resource_list in self.resources.items(): for when_spec, resource_list in self.resources.items():
if when_spec in self.spec: if when_spec in self.spec:
resources.extend(resource_list) resources.extend(resource_list)
# Sorts the resources by the length of the string representing their destination. Since any nested resource
# must contain another resource's name in its path, it seems that should work
resources = sorted(resources, key=lambda res: len(res.destination))
return resources return resources
def _resource_stage(self, resource): def _resource_stage(self, resource):

86
lib/spack/spack/stage.py
View File

@ -23,7 +23,7 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
############################################################################## ##############################################################################
import os import os
import re import errno
import shutil import shutil
import tempfile import tempfile
from urlparse import urljoin from urlparse import urljoin
@ -31,17 +31,18 @@
import llnl.util.tty as tty import llnl.util.tty as tty
from llnl.util.filesystem import * from llnl.util.filesystem import *
import spack.util.pattern as pattern
import spack import spack
import spack.config import spack.config
import spack.fetch_strategy as fs import spack.fetch_strategy as fs
import spack.error import spack.error
STAGE_PREFIX = 'spack-stage-' STAGE_PREFIX = 'spack-stage-'
class Stage(object): class Stage(object):
"""A Stage object manaages a directory where some source code is """A Stage object manages a directory where some source code is
downloaded and built before being installed. It handles downloaded and built before being installed. It handles
fetching the source code, either as an archive to be expanded fetching the source code, either as an archive to be expanded
or by checking it out of a repository. A stage's lifecycle or by checking it out of a repository. A stage's lifecycle
@ -93,7 +94,7 @@ def __init__(self, url_or_fetch_strategy, **kwargs):
raise ValueError("Can't construct Stage without url or fetch strategy") raise ValueError("Can't construct Stage without url or fetch strategy")
self.fetcher.set_stage(self) self.fetcher.set_stage(self)
self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. self.default_fetcher = self.fetcher # self.fetcher can change with mirrors.
self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. self.skip_checksum_for_mirror = True # used for mirrored archives of repositories.
self.name = kwargs.get('name') self.name = kwargs.get('name')
self.mirror_path = kwargs.get('mirror_path') self.mirror_path = kwargs.get('mirror_path')
@ -102,7 +103,6 @@ def __init__(self, url_or_fetch_strategy, **kwargs):
self.path = None self.path = None
self._setup() self._setup()
def _cleanup_dead_links(self): def _cleanup_dead_links(self):
"""Remove any dead links in the stage directory.""" """Remove any dead links in the stage directory."""
for file in os.listdir(spack.stage_path): for file in os.listdir(spack.stage_path):
@ -112,7 +112,6 @@ def _cleanup_dead_links(self):
if not os.path.exists(path): if not os.path.exists(path):
os.unlink(path) os.unlink(path)
def _need_to_create_path(self): def _need_to_create_path(self):
"""Makes sure nothing weird has happened since the last time we """Makes sure nothing weird has happened since the last time we
looked at path. Returns True if path already exists and is ok. looked at path. Returns True if path already exists and is ok.
@ -130,7 +129,7 @@ def _need_to_create_path(self):
# Path looks ok, but need to check the target of the link. # Path looks ok, but need to check the target of the link.
if os.path.islink(self.path): if os.path.islink(self.path):
real_path = os.path.realpath(self.path) real_path = os.path.realpath(self.path)
real_tmp = os.path.realpath(self.tmp_root) real_tmp = os.path.realpath(self.tmp_root)
if spack.use_tmp_stage: if spack.use_tmp_stage:
# If we're using a tmp dir, it's a link, and it points at the right spot, # If we're using a tmp dir, it's a link, and it points at the right spot,
@ -149,7 +148,6 @@ def _need_to_create_path(self):
return False return False
def _setup(self): def _setup(self):
"""Creates the stage directory. """Creates the stage directory.
If spack.use_tmp_stage is False, the stage directory is created If spack.use_tmp_stage is False, the stage directory is created
@ -198,7 +196,6 @@ def _setup(self):
# Make sure we can actually do something with the stage we made. # Make sure we can actually do something with the stage we made.
ensure_access(self.path) ensure_access(self.path)
@property @property
def archive_file(self): def archive_file(self):
"""Path to the source archive within this stage directory.""" """Path to the source archive within this stage directory."""
@ -215,7 +212,6 @@ def archive_file(self):
else: else:
return None return None
@property @property
def source_path(self): def source_path(self):
"""Returns the path to the expanded/checked out source code """Returns the path to the expanded/checked out source code
@ -230,7 +226,6 @@ def source_path(self):
return p return p
return None return None
def chdir(self): def chdir(self):
"""Changes directory to the stage path. Or dies if it is not set up.""" """Changes directory to the stage path. Or dies if it is not set up."""
if os.path.isdir(self.path): if os.path.isdir(self.path):
@ -238,7 +233,6 @@ def chdir(self):
else: else:
tty.die("Setup failed: no such directory: " + self.path) tty.die("Setup failed: no such directory: " + self.path)
def fetch(self, mirror_only=False): def fetch(self, mirror_only=False):
"""Downloads an archive or checks out code from a repository.""" """Downloads an archive or checks out code from a repository."""
self.chdir() self.chdir()
@ -291,7 +285,6 @@ def fetch(self, mirror_only=False):
self.fetcher = self.default_fetcher self.fetcher = self.default_fetcher
raise fs.FetchError(errMessage, None) raise fs.FetchError(errMessage, None)
def check(self): def check(self):
"""Check the downloaded archive against a checksum digest. """Check the downloaded archive against a checksum digest.
No-op if this stage checks code out of a repository.""" No-op if this stage checks code out of a repository."""
@ -305,14 +298,17 @@ def check(self):
else: else:
self.fetcher.check() self.fetcher.check()
def expand_archive(self): def expand_archive(self):
"""Changes to the stage directory and attempt to expand the downloaded """Changes to the stage directory and attempt to expand the downloaded
archive. Fail if the stage is not set up or if the archive is not yet archive. Fail if the stage is not set up or if the archive is not yet
downloaded. downloaded.
""" """
self.fetcher.expand() archive_dir = self.source_path
if not archive_dir:
self.fetcher.expand()
tty.msg("Created stage in %s." % self.path)
else:
tty.msg("Already staged %s in %s." % (self.name, self.path))
def chdir_to_source(self): def chdir_to_source(self):
"""Changes directory to the expanded archive directory. """Changes directory to the expanded archive directory.
@ -326,14 +322,12 @@ def chdir_to_source(self):
if not os.listdir(path): if not os.listdir(path):
tty.die("Archive was empty for %s" % self.name) tty.die("Archive was empty for %s" % self.name)
def restage(self): def restage(self):
"""Removes the expanded archive path if it exists, then re-expands """Removes the expanded archive path if it exists, then re-expands
the archive. the archive.
""" """
self.fetcher.reset() self.fetcher.reset()
def destroy(self): def destroy(self):
"""Remove this stage directory.""" """Remove this stage directory."""
remove_linked_tree(self.path) remove_linked_tree(self.path)
@ -345,8 +339,63 @@ def destroy(self):
os.chdir(os.path.dirname(self.path)) os.chdir(os.path.dirname(self.path))
class ResourceStage(Stage):
def __init__(self, url_or_fetch_strategy, root, resource, **kwargs):
super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs)
self.root_stage = root
self.resource = resource
def expand_archive(self):
super(ResourceStage, self).expand_archive()
root_stage = self.root_stage
resource = self.resource
placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement
if not isinstance(placement, dict):
placement = {'': placement}
# Make the paths in the dictionary absolute and link
for key, value in placement.iteritems():
target_path = join_path(root_stage.source_path, resource.destination)
destination_path = join_path(target_path, value)
source_path = join_path(self.source_path, key)
try:
os.makedirs(target_path)
except OSError as err:
if err.errno == errno.EEXIST and os.path.isdir(target_path):
pass
else:
raise
if not os.path.exists(destination_path):
# Create a symlink
tty.info('Moving resource stage\n\tsource : {stage}\n\tdestination : {destination}'.format(
stage=source_path, destination=destination_path
))
shutil.move(source_path, destination_path)
@pattern.composite(method_list=['fetch', 'check', 'expand_archive', 'restage', 'destroy'])
class StageComposite:
"""
Composite for Stage type objects. The first item in this composite is considered to be the root package, and
operations that return a value are forwarded to it.
"""
@property
def source_path(self):
return self[0].source_path
@property
def path(self):
return self[0].path
def chdir_to_source(self):
return self[0].chdir_to_source()
class DIYStage(object): class DIYStage(object):
"""Simple class that allows any directory to be a spack stage.""" """Simple class that allows any directory to be a spack stage."""
def __init__(self, path): def __init__(self, path):
self.archive_file = None self.archive_file = None
self.path = path self.path = path
@ -384,7 +433,6 @@ def _get_mirrors():
return [val for name, val in config.iteritems()] return [val for name, val in config.iteritems()]
def ensure_access(file=spack.stage_path): def ensure_access(file=spack.stage_path):
"""Ensure we can access a directory and die with an error if we can't.""" """Ensure we can access a directory and die with an error if we can't."""
if not can_access(file): if not can_access(file):

1
lib/spack/spack/test/__init__.py
View File

@ -48,6 +48,7 @@
'package_sanity', 'package_sanity',
'config', 'config',
'directory_layout', 'directory_layout',
'pattern',
'python_version', 'python_version',
'git_fetch', 'git_fetch',
'svn_fetch', 'svn_fetch',

7
lib/spack/spack/test/install.py
View File

@ -31,7 +31,7 @@
import spack import spack
from spack.stage import Stage from spack.stage import Stage
from spack.fetch_strategy import URLFetchStrategy from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite
from spack.directory_layout import YamlDirectoryLayout from spack.directory_layout import YamlDirectoryLayout
from spack.util.executable import which from spack.util.executable import which
from spack.test.mock_packages_test import * from spack.test.mock_packages_test import *
@ -79,7 +79,10 @@ def test_install_and_uninstall(self):
pkg = spack.repo.get(spec) pkg = spack.repo.get(spec)
# Fake the URL for the package so it downloads from a file. # Fake the URL for the package so it downloads from a file.
pkg.fetcher = URLFetchStrategy(self.repo.url)
fetcher = FetchStrategyComposite()
fetcher.append(URLFetchStrategy(self.repo.url))
pkg.fetcher = fetcher
try: try:
pkg.do_install() pkg.do_install()

1
lib/spack/spack/test/mirror.py
View File

@ -102,6 +102,7 @@ def check_mirror(self):
spec = Spec(name).concretized() spec = Spec(name).concretized()
pkg = spec.package pkg = spec.package
pkg._stage = None
saved_checksum_setting = spack.do_checksum saved_checksum_setting = spack.do_checksum
try: try:
# Stage the archive from the mirror and cd to it. # Stage the archive from the mirror and cd to it.

104
lib/spack/spack/test/pattern.py Normal file
View File

@ -0,0 +1,104 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import unittest
import spack.util.pattern as pattern
class CompositeTest(unittest.TestCase):
def setUp(self):
class Base:
counter = 0
def add(self):
raise NotImplemented('add not implemented')
def subtract(self):
raise NotImplemented('subtract not implemented')
class One(Base):
def add(self):
Base.counter += 1
def subtract(self):
Base.counter -= 1
class Two(Base):
def add(self):
Base.counter += 2
def subtract(self):
Base.counter -= 2
self.Base = Base
self.One = One
self.Two = Two
def test_composite_from_method_list(self):
@pattern.composite(method_list=['add', 'subtract'])
class CompositeFromMethodList:
pass
composite = CompositeFromMethodList()
composite.append(self.One())
composite.append(self.Two())
composite.add()
self.assertEqual(self.Base.counter, 3)
composite.pop()
composite.subtract()
self.assertEqual(self.Base.counter, 2)
def test_composite_from_interface(self):
@pattern.composite(interface=self.Base)
class CompositeFromInterface:
pass
composite = CompositeFromInterface()
composite.append(self.One())
composite.append(self.Two())
composite.add()
self.assertEqual(self.Base.counter, 3)
composite.pop()
composite.subtract()
self.assertEqual(self.Base.counter, 2)
def test_error_conditions(self):
def wrong_container():
@pattern.composite(interface=self.Base, container=2)
class CompositeFromInterface:
pass
def no_methods():
@pattern.composite()
class CompositeFromInterface:
pass
self.assertRaises(TypeError, wrong_container)
self.assertRaises(TypeError, no_methods)

116
lib/spack/spack/util/pattern.py Normal file
View File

@ -0,0 +1,116 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import inspect
import collections
import functools
def composite(interface=None, method_list=None, container=list):
"""
Returns a class decorator that patches a class adding all the methods it needs to be a composite for a given
interface.
:param interface: class exposing the interface to which the composite object must conform. Only non-private and
non-special methods will be taken into account
:param method_list: names of methods that should be part of the composite
:param container: container for the composite object (default = list). Must fulfill the MutableSequence contract.
The composite class will expose the container API to manage object composition
:return: class decorator
"""
# Check if container fulfills the MutableSequence contract and raise an exception if it doesn't
# The patched class returned by the decorator will inherit from the container class to expose the
# interface needed to manage objects composition
if not issubclass(container, collections.MutableSequence):
raise TypeError("Container must fulfill the MutableSequence contract")
# Check if at least one of the 'interface' or the 'method_list' arguments are defined
if interface is None and method_list is None:
raise TypeError("Either 'interface' or 'method_list' must be defined on a call to composite")
def cls_decorator(cls):
# Retrieve the base class of the composite. Inspect its methods and decide which ones will be overridden
def no_special_no_private(x):
return inspect.ismethod(x) and not x.__name__.startswith('_')
# Patch the behavior of each of the methods in the previous list. This is done associating an instance of the
# descriptor below to any method that needs to be patched.
class IterateOver(object):
"""
Decorator used to patch methods in a composite. It iterates over all the items in the instance containing the
associated attribute and calls for each of them an attribute with the same name
"""
def __init__(self, name, func=None):
self.name = name
self.func = func
def __get__(self, instance, owner):
def getter(*args, **kwargs):
for item in instance:
getattr(item, self.name)(*args, **kwargs)
# If we are using this descriptor to wrap a method from an interface, then we must conditionally
# use the `functools.wraps` decorator to set the appropriate fields.
if self.func is not None:
getter = functools.wraps(self.func)(getter)
return getter
dictionary_for_type_call = {}
# Construct a dictionary with the methods explicitly passed as name
if method_list is not None:
# python@2.7: method_list_dict = {name: IterateOver(name) for name in method_list}
method_list_dict = {}
for name in method_list:
method_list_dict[name] = IterateOver(name)
dictionary_for_type_call.update(method_list_dict)
# Construct a dictionary with the methods inspected from the interface
if interface is not None:
##########
# python@2.7: interface_methods = {name: method for name, method in inspect.getmembers(interface, predicate=no_special_no_private)}
interface_methods = {}
for name, method in inspect.getmembers(interface, predicate=no_special_no_private):
interface_methods[name] = method
##########
# python@2.7: interface_methods_dict = {name: IterateOver(name, method) for name, method in interface_methods.iteritems()}
interface_methods_dict = {}
for name, method in interface_methods.iteritems():
interface_methods_dict[name] = IterateOver(name, method)
##########
dictionary_for_type_call.update(interface_methods_dict)
# Get the methods that are defined in the scope of the composite class and override any previous definition
##########
# python@2.7: cls_method = {name: method for name, method in inspect.getmembers(cls, predicate=inspect.ismethod)}
cls_method = {}
for name, method in inspect.getmembers(cls, predicate=inspect.ismethod):
cls_method[name] = method
##########
dictionary_for_type_call.update(cls_method)
# Generate the new class on the fly and return it
# FIXME : inherit from interface if we start to use ABC classes?
wrapper_class = type(cls.__name__, (cls, container), dictionary_for_type_call)
return wrapper_class
return cls_decorator

19
var/spack/repos/builtin/packages/llvm/package.py
View File

@ -171,6 +171,25 @@ class Llvm(Package):
when='@%(version)s' % release, when='@%(version)s' % release,
placement=resources[name].get('placement', None)) placement=resources[name].get('placement', None))
# SVN - current develop
version('develop', svn='http://llvm.org/svn/llvm-project/llvm/trunk')
resource(name='clang', svn='http://llvm.org/svn/llvm-project/cfe/trunk',
destination='tools', when='@develop', placement='clang')
resource(name='compiler-rt', svn='http://llvm.org/svn/llvm-project/compiler-rt/trunk',
destination='projects', when='@develop', placement='compiler-rt')
resource(name='openmp', svn='http://llvm.org/svn/llvm-project/openmp/trunk',
destination='projects', when='@develop', placement='openmp')
resource(name='libcxx', svn='http://llvm.org/svn/llvm-project/libcxx/trunk',
destination='projects', when='@develop', placement='libcxx')
resource(name='libcxxabi', svn='http://llvm.org/svn/llvm-project/libcxxabi/trunk',
destination='projects', when='@develop', placement='libcxxabi')
resource(name='polly', svn='http://llvm.org/svn/llvm-project/polly/trunk',
destination='tools', when='@develop', placement='polly')
resource(name='lldb', svn='http://llvm.org/svn/llvm-project/lldb/trunk',
destination='tools', when='@develop', placement='lldb')
def install(self, spec, prefix): def install(self, spec, prefix):
env['CXXFLAGS'] = self.compiler.cxx11_flag env['CXXFLAGS'] = self.compiler.cxx11_flag
cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ] cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ]