zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #298 from epfl-scitas/refactoring/resource_directive

Browse Source 
refactoring proposal : composite for Stage and FetchStrategy
This commit is contained in:
Todd Gamblin 2016-02-18 23:13:12 -08:00
commit 3c1aa9a4ad
11 changed files with 461 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/spack/spack/directives.py
View File

@ -296,8 +296,8 @@ def resource(pkg, **kwargs):
raise RuntimeError(message)
when_spec = parse_anonymous_spec(when, pkg.name)
resources = pkg.resources.setdefault(when_spec, [])
fetcher = from_kwargs(**kwargs)
name = kwargs.get('name')
fetcher = from_kwargs(**kwargs)
resources.append(Resource(name, fetcher, destination, placement))

131
lib/spack/spack/fetch_strategy.py
View File

@ -44,6 +44,7 @@
import sys
import re
import shutil
import copy
from functools import wraps
import llnl.util.tty as tty
from llnl.util.filesystem import *
@ -55,53 +56,59 @@
from spack.version import Version, ver
from spack.util.compression import decompressor_for, extension
import spack.util.pattern as pattern
"""List of all fetch strategies, created by FetchStrategy metaclass."""
all_strategies = []
def _needs_stage(fun):
"""Many methods on fetch strategies require a stage to be set
using set_stage(). This decorator adds a check for self.stage."""
@wraps(fun)
def wrapper(self, *args, **kwargs):
if not self.stage:
raise NoStageError(fun)
return fun(self, *args, **kwargs)
return wrapper
class FetchStrategy(object):
"""Superclass of all fetch strategies."""
enabled = False # Non-abstract subclasses should be enabled.
enabled = False # Non-abstract subclasses should be enabled.
required_attributes = None # Attributes required in version() args.
class __metaclass__(type):
"""This metaclass registers all fetch strategies in a list."""
def __init__(cls, name, bases, dict):
type.__init__(cls, name, bases, dict)
if cls.enabled: all_strategies.append(cls)
def __init__(self):
# The stage is initialized late, so that fetch strategies can be constructed
# at package construction time. This is where things will be fetched.
self.stage = None
def set_stage(self, stage):
"""This is called by Stage before any of the fetching
methods are called on the stage."""
self.stage = stage
# Subclasses need to implement these methods
def fetch(self): pass # Return True on success, False on fail.
def check(self): pass # Do checksum.
def expand(self): pass # Expand archive.
def reset(self): pass # Revert to freshly downloaded state.
def archive(self, destination): pass # Used to create tarball for mirror.
def __str__(self): # Should be human readable URL.
def __str__(self): # Should be human readable URL.
return "FetchStrategy.__str___"
# This method is used to match fetch strategies to version()
@ -111,6 +118,15 @@ def matches(cls, args):
return any(k in args for k in cls.required_attributes)
@pattern.composite(interface=FetchStrategy)
class FetchStrategyComposite(object):
"""
Composite for a FetchStrategy object. Implements the GoF composite pattern.
"""
matches = FetchStrategy.matches
set_stage = FetchStrategy.set_stage
class URLFetchStrategy(FetchStrategy):
"""FetchStrategy that pulls source code from a URL for an archive,
checks the archive against a checksum,and decompresses the archive.
@ -142,15 +158,15 @@ def fetch(self):
tty.msg("Trying to fetch from %s" % self.url)
curl_args = ['-O', # save file to disk
'-f', # fail on >400 errors
'-D', '-', # print out HTML headers
'-L', self.url,]
curl_args = ['-O', # save file to disk
'-f', # fail on >400 errors
'-D', '-', # print out HTML headers
'-L', self.url, ]
if sys.stdout.isatty():
curl_args.append('-#') # status bar when using a tty
else:
curl_args.append('-sS') # just errors when not.
curl_args.append('-sS') # just errors when not.
# Run curl but grab the mime type from the http headers
headers = spack.curl(
@ -164,24 +180,23 @@ def fetch(self):
if spack.curl.returncode == 22:
# This is a 404. Curl will print the error.
raise FailedDownloadError(
self.url, "URL %s was not found!" % self.url)
self.url, "URL %s was not found!" % self.url)
elif spack.curl.returncode == 60:
# This is a certificate error. Suggest spack -k
raise FailedDownloadError(
self.url,
"Curl was unable to fetch due to invalid certificate. "
"This is either an attack, or your cluster's SSL configuration "
"is bad. If you believe your SSL configuration is bad, you "
"can try running spack -k, which will not check SSL certificates."
"Use this at your own risk.")
self.url,
"Curl was unable to fetch due to invalid certificate. "
"This is either an attack, or your cluster's SSL configuration "
"is bad. If you believe your SSL configuration is bad, you "
"can try running spack -k, which will not check SSL certificates."
"Use this at your own risk.")
else:
# This is some other curl error. Curl will print the
# error, but print a spack message too
raise FailedDownloadError(
self.url, "Curl failed with error %d" % spack.curl.returncode)
self.url, "Curl failed with error %d" % spack.curl.returncode)
# Check if we somehow got an HTML file rather than the archive we
# asked for. We only look at the last content type, to handle
@ -196,7 +211,6 @@ def fetch(self):
if not self.archive_file:
raise FailedDownloadError(self.url)
@property
def archive_file(self):
"""Path to the source archive within this stage directory."""
@ -209,7 +223,7 @@ def expand(self):
self.stage.chdir()
if not self.archive_file:
raise NoArchiveFileError("URLFetchStrategy couldn't find archive file",
"Failed on expand() for URL %s" % self.url)
"Failed on expand() for URL %s" % self.url)
decompress = decompressor_for(self.archive_file)
@ -241,7 +255,6 @@ def expand(self):
# Set the wd back to the stage when done.
self.stage.chdir()
def archive(self, destination):
"""Just moves this archive to the destination."""
if not self.archive_file:
@ -252,7 +265,6 @@ def archive(self, destination):
shutil.move(self.archive_file, destination)
@_needs_stage
def check(self):
"""Check the downloaded archive against a checksum digest.
@ -263,9 +275,8 @@ def check(self):
checker = crypto.Checker(self.digest)
if not checker.check(self.archive_file):
raise ChecksumError(
"%s checksum failed for %s." % (checker.hash_name, self.archive_file),
"Expected %s but got %s." % (self.digest, checker.sum))
"%s checksum failed for %s." % (checker.hash_name, self.archive_file),
"Expected %s but got %s." % (self.digest, checker.sum))
@_needs_stage
def reset(self):
@ -277,12 +288,10 @@ def reset(self):
shutil.rmtree(self.stage.source_path, ignore_errors=True)
self.expand()
def __repr__(self):
url = self.url if self.url else "no url"
return "URLFetchStrategy<%s>" % url
def __str__(self):
if self.url:
return self.url
@ -298,33 +307,30 @@ def __init__(self, name, *rev_types, **kwargs):
# Set a URL based on the type of fetch strategy.
self.url = kwargs.get(name, None)
if not self.url: raise ValueError(
"%s requires %s argument." % (self.__class__, name))
"%s requires %s argument." % (self.__class__, name))
# Ensure that there's only one of the rev_types
if sum(k in kwargs for k in rev_types) > 1:
raise FetchStrategyError(
"Supply only one of %s to fetch with %s." % (
comma_or(rev_types), name))
"Supply only one of %s to fetch with %s." % (
comma_or(rev_types), name))
# Set attributes for each rev type.
for rt in rev_types:
setattr(self, rt, kwargs.get(rt, None))
@_needs_stage
def check(self):
tty.msg("No checksum needed when fetching with %s." % self.name)
@_needs_stage
def expand(self):
tty.debug("Source fetched with %s is already expanded." % self.name)
@_needs_stage
def archive(self, destination, **kwargs):
assert(extension(destination) == 'tar.gz')
assert(self.stage.source_path.startswith(self.stage.path))
assert (extension(destination) == 'tar.gz')
assert (self.stage.source_path.startswith(self.stage.path))
tar = which('tar', required=True)
@ -338,16 +344,13 @@ def archive(self, destination, **kwargs):
self.stage.chdir()
tar('-czf', destination, os.path.basename(self.stage.source_path))
def __str__(self):
return "VCS: %s" % self.url
def __repr__(self):
return "%s<%s>" % (self.__class__, self.url)
class GitFetchStrategy(VCSFetchStrategy):
"""Fetch strategy that gets source code from a git repository.
Use like this in a package:
@ -368,24 +371,25 @@ class GitFetchStrategy(VCSFetchStrategy):
required_attributes = ('git',)
def __init__(self, **kwargs):
super(GitFetchStrategy, self).__init__(
'git', 'tag', 'branch', 'commit', **kwargs)
self._git = None
# Discards the keywords in kwargs that may conflict with the next call to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(GitFetchStrategy, self).__init__(
'git', 'tag', 'branch', 'commit', **forwarded_args)
self._git = None
@property
def git_version(self):
vstring = self.git('--version', output=str).lstrip('git version ')
return Version(vstring)
@property
def git(self):
if not self._git:
self._git = which('git', required=True)
return self._git
@_needs_stage
def fetch(self):
self.stage.chdir()
@ -418,7 +422,7 @@ def fetch(self):
if self.branch:
args.extend(['--branch', self.branch])
elif self.tag and self.git_version >= ver('1.8.5.2'):
args.extend(['--branch', self.tag])
args.extend(['--branch', self.tag])
# Try to be efficient if we're using a new enough git.
# This checks out only one branch's history
@ -429,7 +433,7 @@ def fetch(self):
# Yet more efficiency, only download a 1-commit deep tree
if self.git_version >= ver('1.7.1'):
try:
self.git(*(args + ['--depth','1', self.url]))
self.git(*(args + ['--depth', '1', self.url]))
cloned = True
except spack.error.SpackError:
# This will fail with the dumb HTTP transport
@ -452,18 +456,15 @@ def fetch(self):
self.git('pull', '--tags', ignore_errors=1)
self.git('checkout', self.tag)
def archive(self, destination):
super(GitFetchStrategy, self).archive(destination, exclude='.git')
@_needs_stage
def reset(self):
self.stage.chdir_to_source()
self.git('checkout', '.')
self.git('clean', '-f')
def __str__(self):
return "[git] %s" % self.url
@ -483,20 +484,22 @@ class SvnFetchStrategy(VCSFetchStrategy):
required_attributes = ['svn']
def __init__(self, **kwargs):
# Discards the keywords in kwargs that may conflict with the next call to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(SvnFetchStrategy, self).__init__(
'svn', 'revision', **kwargs)
'svn', 'revision', **forwarded_args)
self._svn = None
if self.revision is not None:
self.revision = str(self.revision)
@property
def svn(self):
if not self._svn:
self._svn = which('svn', required=True)
return self._svn
@_needs_stage
def fetch(self):
self.stage.chdir()
@ -515,7 +518,6 @@ def fetch(self):
self.svn(*args)
self.stage.chdir_to_source()
def _remove_untracked_files(self):
"""Removes untracked files in an svn repository."""
status = self.svn('status', '--no-ignore', output=str)
@ -529,23 +531,19 @@ def _remove_untracked_files(self):
elif os.path.isdir(path):
shutil.rmtree(path, ignore_errors=True)
def archive(self, destination):
super(SvnFetchStrategy, self).archive(destination, exclude='.svn')
@_needs_stage
def reset(self):
self.stage.chdir_to_source()
self._remove_untracked_files()
self.svn('revert', '.', '-R')
def __str__(self):
return "[svn] %s" % self.url
class HgFetchStrategy(VCSFetchStrategy):
"""Fetch strategy that gets source code from a Mercurial repository.
Use like this in a package:
@ -567,10 +565,13 @@ class HgFetchStrategy(VCSFetchStrategy):
required_attributes = ['hg']
def __init__(self, **kwargs):
super(HgFetchStrategy, self).__init__(
'hg', 'revision', **kwargs)
self._hg = None
# Discards the keywords in kwargs that may conflict with the next call to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop('name', None)
super(HgFetchStrategy, self).__init__(
'hg', 'revision', **forwarded_args)
self._hg = None
@property
def hg(self):
@ -597,11 +598,9 @@ def fetch(self):
self.hg(*args)
def archive(self, destination):
super(HgFetchStrategy, self).archive(destination, exclude='.hg')
@_needs_stage
def reset(self):
self.stage.chdir()
@ -619,7 +618,6 @@ def reset(self):
shutil.move(scrubbed, source_path)
self.stage.chdir_to_source()
def __str__(self):
return "[hg] %s" % self.url
@ -693,9 +691,10 @@ def __init__(self, msg, long_msg=None):
class FailedDownloadError(FetchError):
"""Raised wen a download fails."""
def __init__(self, url, msg=""):
super(FailedDownloadError, self).__init__(
"Failed to fetch file from URL: %s" % url, msg)
"Failed to fetch file from URL: %s" % url, msg)
self.url = url
@ -718,12 +717,14 @@ def __init__(self, pkg, version):
class ChecksumError(FetchError):
"""Raised when archive fails to checksum."""
def __init__(self, message, long_msg=None):
super(ChecksumError, self).__init__(message, long_msg)
class NoStageError(FetchError):
"""Raised when fetch operations are called before set_stage()."""
def __init__(self, method):
super(NoStageError, self).__init__(
"Must call FetchStrategy.set_stage() before calling %s" % method.__name__)
"Must call FetchStrategy.set_stage() before calling %s" % method.__name__)

82
lib/spack/spack/mirror.py
View File

@ -45,12 +45,11 @@
from spack.util.compression import extension, allowed_archive
def mirror_archive_filename(spec):
def mirror_archive_filename(spec, fetcher):
"""Get the name of the spec's archive in the mirror."""
if not spec.version.concrete:
raise ValueError("mirror.path requires spec with concrete version.")
fetcher = spec.package.fetcher
if isinstance(fetcher, fs.URLFetchStrategy):
# If we fetch this version with a URLFetchStrategy, use URL's archive type
ext = url.downloaded_file_extension(fetcher.url)
@ -61,9 +60,9 @@ def mirror_archive_filename(spec):
return "%s-%s.%s" % (spec.package.name, spec.version, ext)
def mirror_archive_path(spec):
def mirror_archive_path(spec, fetcher):
"""Get the relative path to the spec's archive within a mirror."""
return join_path(spec.name, mirror_archive_filename(spec))
return join_path(spec.name, mirror_archive_filename(spec, fetcher))
def get_matching_versions(specs, **kwargs):
@ -167,72 +166,47 @@ def create(path, specs, **kwargs):
everything_already_exists = True
for spec in version_specs:
pkg = spec.package
stage = None
tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@")))
try:
# create a subdirectory for the current package@version
archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec)))
subdir = os.path.dirname(archive_path)
try:
for ii, stage in enumerate(pkg.stage):
fetcher = stage.fetcher
if ii == 0:
# create a subdirectory for the current package@version
archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher)))
name = spec.format("$_$@")
else:
resource = stage.resource
archive_path = join_path(subdir, suggest_archive_basename(resource))
name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@"))
subdir = os.path.dirname(archive_path)
mkdirp(subdir)
except OSError as e:
raise MirrorError(
"Cannot create directory '%s':" % subdir, str(e))
if os.path.exists(archive_path):
tty.msg("Already added %s" % spec.format("$_$@"))
else:
everything_already_exists = False
# Set up a stage and a fetcher for the download
unique_fetch_name = spec.format("$_$@")
fetcher = fs.for_package_version(pkg, pkg.version)
stage = Stage(fetcher, name=unique_fetch_name)
fetcher.set_stage(stage)
if os.path.exists(archive_path):
tty.msg("{name} : already added".format(name=name))
else:
everything_already_exists = False
fetcher.fetch()
if not kwargs.get('no_checksum', False):
fetcher.check()
tty.msg("{name} : checksum passed".format(name=name))
# Do the fetch and checksum if necessary
fetcher.fetch()
if not kwargs.get('no_checksum', False):
fetcher.check()
tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
# Fetchers have to know how to archive their files. Use
# that to move/copy/create an archive in the mirror.
fetcher.archive(archive_path)
tty.msg("Added %s." % spec.format("$_$@"))
# Fetch resources if they are associated with the spec
resources = pkg._get_resources()
for resource in resources:
resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
if os.path.exists(resource_archive_path):
tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
continue
everything_already_exists = False
resource_stage_folder = pkg._resource_stage(resource)
resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
resource.fetcher.set_stage(resource_stage)
resource.fetcher.fetch()
if not kwargs.get('no_checksum', False):
resource.fetcher.check()
tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
resource.fetcher.archive(resource_archive_path)
tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
# Fetchers have to know how to archive their files. Use
# that to move/copy/create an archive in the mirror.
fetcher.archive(archive_path)
tty.msg("{name} : added".format(name=name))
if everything_already_exists:
present.append(spec)
else:
mirrored.append(spec)
except Exception, e:
if spack.debug:
sys.excepthook(*sys.exc_info())
else:
tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message)
error.append(spec)
finally:
if stage:
stage.destroy()
pkg.stage.destroy()
return (present, mirrored, error)

128
lib/spack/spack/package.py
View File

@ -63,7 +63,7 @@
import spack.util.web
import spack.fetch_strategy as fs
from spack.version import *
from spack.stage import Stage
from spack.stage import Stage, ResourceStage, StageComposite
from spack.util.compression import allowed_archive, extension
from spack.util.executable import ProcessError
@ -433,23 +433,46 @@ def url_for_version(self, version):
return spack.url.substitute_version(self.nearest_url(version),
self.url_version(version))
def _make_resource_stage(self, root_stage, fetcher, resource):
resource_stage_folder = self._resource_stage(resource)
resource_mirror = join_path(self.name, os.path.basename(fetcher.url))
stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource,
name=resource_stage_folder, mirror_path=resource_mirror)
return stage
def _make_root_stage(self, fetcher):
# Construct a mirror path (TODO: get this out of package.py)
mp = spack.mirror.mirror_archive_path(self.spec, fetcher)
# Construct a path where the stage should build..
s = self.spec
stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
# Build the composite stage
stage = Stage(fetcher, mirror_path=mp, name=stage_name)
return stage
def _make_stage(self):
# Construct a composite stage on top of the composite FetchStrategy
composite_fetcher = self.fetcher
composite_stage = StageComposite()
resources = self._get_resources()
for ii, fetcher in enumerate(composite_fetcher):
if ii == 0:
# Construct root stage first
stage = self._make_root_stage(fetcher)
else:
# Construct resource stage
resource = resources[ii - 1] # ii == 0 is root!
stage = self._make_resource_stage(composite_stage[0], fetcher, resource)
# Append the item to the composite
composite_stage.append(stage)
return composite_stage
@property
def stage(self):
if not self.spec.concrete:
raise ValueError("Can only get a stage for a concrete package.")
if self._stage is None:
# Construct a mirror path (TODO: get this out of package.py)
mp = spack.mirror.mirror_archive_path(self.spec)
# Construct a path where the stage should build..
s = self.spec
stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
# Build the stage
self._stage = Stage(self.fetcher, mirror_path=mp, name=stage_name)
self._stage = self._make_stage()
return self._stage
@ -459,17 +482,25 @@ def stage(self, stage):
self._stage = stage
def _make_fetcher(self):
# Construct a composite fetcher that always contains at least one element (the root package). In case there
# are resources associated with the package, append their fetcher to the composite.
root_fetcher = fs.for_package_version(self, self.version)
fetcher = fs.FetchStrategyComposite() # Composite fetcher
fetcher.append(root_fetcher) # Root fetcher is always present
resources = self._get_resources()
for resource in resources:
fetcher.append(resource.fetcher)
return fetcher
@property
def fetcher(self):
if not self.spec.versions.concrete:
raise ValueError(
"Can only get a fetcher for a package with concrete versions.")
raise ValueError("Can only get a fetcher for a package with concrete versions.")
if not self._fetcher:
self._fetcher = fs.for_package_version(self, self.version)
self._fetcher = self._make_fetcher()
return self._fetcher
@fetcher.setter
def fetcher(self, f):
self._fetcher = f
@ -632,7 +663,7 @@ def remove_prefix(self):
def do_fetch(self, mirror_only=False):
"""Creates a stage directory and downloads the taball for this package.
"""Creates a stage directory and downloads the tarball for this package.
Working directory will be set to the stage directory.
"""
if not self.spec.concrete:
@ -658,20 +689,6 @@ def do_fetch(self, mirror_only=False):
self.stage.fetch(mirror_only)
##########
# Fetch resources
resources = self._get_resources()
for resource in resources:
resource_stage_folder = self._resource_stage(resource)
# FIXME : works only for URLFetchStrategy
resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
resource.fetcher.set_stage(resource_stage)
# Delegate to stage object to trigger mirror logic
resource_stage.fetch()
resource_stage.check()
##########
self._fetch_time = time.time() - start_time
if spack.do_checksum and self.version in self.versions:
@ -684,52 +701,10 @@ def do_stage(self, mirror_only=False):
if not self.spec.concrete:
raise ValueError("Can only stage concrete packages.")
def _expand_archive(stage, name=self.name):
archive_dir = stage.source_path
if not archive_dir:
stage.expand_archive()
tty.msg("Created stage in %s." % stage.path)
else:
tty.msg("Already staged %s in %s." % (name, stage.path))
self.do_fetch(mirror_only)
_expand_archive(self.stage)
##########
# Stage resources in appropriate path
resources = self._get_resources()
# TODO: this is to allow nested resources, a better solution would be
# good
for resource in sorted(resources, key=lambda res: len(res.destination)):
stage = resource.fetcher.stage
_expand_archive(stage, resource.name)
# Turn placement into a dict with relative paths
placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement
if not isinstance(placement, dict):
placement = {'': placement}
# Make the paths in the dictionary absolute and link
for key, value in placement.iteritems():
target_path = join_path(self.stage.source_path, resource.destination)
link_path = join_path(target_path, value)
source_path = join_path(stage.source_path, key)
try:
os.makedirs(target_path)
except OSError as err:
if err.errno == errno.EEXIST and os.path.isdir(target_path):
pass
else: raise
# NOTE: a reasonable fix for the TODO above might be to have
# these expand in place, but expand_archive does not offer
# this
if not os.path.exists(link_path):
shutil.move(source_path, link_path)
##########
self.stage.expand_archive()
self.stage.chdir_to_source()
def do_patch(self):
"""Calls do_stage(), then applied patches to the expanded tarball if they
haven't been applied already."""
@ -828,6 +803,9 @@ def _get_resources(self):
for when_spec, resource_list in self.resources.items():
if when_spec in self.spec:
resources.extend(resource_list)
# Sorts the resources by the length of the string representing their destination. Since any nested resource
# must contain another resource's name in its path, it seems that should work
resources = sorted(resources, key=lambda res: len(res.destination))
return resources
def _resource_stage(self, resource):

86
lib/spack/spack/stage.py
View File

@ -23,7 +23,7 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import os
import re
import errno
import shutil
import tempfile
from urlparse import urljoin
@ -31,17 +31,18 @@
import llnl.util.tty as tty
from llnl.util.filesystem import *
import spack.util.pattern as pattern
import spack
import spack.config
import spack.fetch_strategy as fs
import spack.error
STAGE_PREFIX = 'spack-stage-'
class Stage(object):
"""A Stage object manaages a directory where some source code is
"""A Stage object manages a directory where some source code is
downloaded and built before being installed. It handles
fetching the source code, either as an archive to be expanded
or by checking it out of a repository. A stage's lifecycle
@ -93,7 +94,7 @@ def __init__(self, url_or_fetch_strategy, **kwargs):
raise ValueError("Can't construct Stage without url or fetch strategy")
self.fetcher.set_stage(self)
self.default_fetcher = self.fetcher # self.fetcher can change with mirrors.
self.skip_checksum_for_mirror = True # used for mirrored archives of repositories.
self.skip_checksum_for_mirror = True # used for mirrored archives of repositories.
self.name = kwargs.get('name')
self.mirror_path = kwargs.get('mirror_path')
@ -102,7 +103,6 @@ def __init__(self, url_or_fetch_strategy, **kwargs):
self.path = None
self._setup()
def _cleanup_dead_links(self):
"""Remove any dead links in the stage directory."""
for file in os.listdir(spack.stage_path):
@ -112,7 +112,6 @@ def _cleanup_dead_links(self):
if not os.path.exists(path):
os.unlink(path)
def _need_to_create_path(self):
"""Makes sure nothing weird has happened since the last time we
looked at path. Returns True if path already exists and is ok.
@ -130,7 +129,7 @@ def _need_to_create_path(self):
# Path looks ok, but need to check the target of the link.
if os.path.islink(self.path):
real_path = os.path.realpath(self.path)
real_tmp = os.path.realpath(self.tmp_root)
real_tmp = os.path.realpath(self.tmp_root)
if spack.use_tmp_stage:
# If we're using a tmp dir, it's a link, and it points at the right spot,
@ -149,7 +148,6 @@ def _need_to_create_path(self):
return False
def _setup(self):
"""Creates the stage directory.
If spack.use_tmp_stage is False, the stage directory is created
@ -198,7 +196,6 @@ def _setup(self):
# Make sure we can actually do something with the stage we made.
ensure_access(self.path)
@property
def archive_file(self):
"""Path to the source archive within this stage directory."""
@ -215,7 +212,6 @@ def archive_file(self):
else:
return None
@property
def source_path(self):
"""Returns the path to the expanded/checked out source code
@ -230,7 +226,6 @@ def source_path(self):
return p
return None
def chdir(self):
"""Changes directory to the stage path. Or dies if it is not set up."""
if os.path.isdir(self.path):
@ -238,7 +233,6 @@ def chdir(self):
else:
tty.die("Setup failed: no such directory: " + self.path)
def fetch(self, mirror_only=False):
"""Downloads an archive or checks out code from a repository."""
self.chdir()
@ -291,7 +285,6 @@ def fetch(self, mirror_only=False):
self.fetcher = self.default_fetcher
raise fs.FetchError(errMessage, None)
def check(self):
"""Check the downloaded archive against a checksum digest.
No-op if this stage checks code out of a repository."""
@ -305,14 +298,17 @@ def check(self):
else:
self.fetcher.check()
def expand_archive(self):
"""Changes to the stage directory and attempt to expand the downloaded
archive. Fail if the stage is not set up or if the archive is not yet
downloaded.
"""
self.fetcher.expand()
archive_dir = self.source_path
if not archive_dir:
self.fetcher.expand()
tty.msg("Created stage in %s." % self.path)
else:
tty.msg("Already staged %s in %s." % (self.name, self.path))
def chdir_to_source(self):
"""Changes directory to the expanded archive directory.
@ -326,14 +322,12 @@ def chdir_to_source(self):
if not os.listdir(path):
tty.die("Archive was empty for %s" % self.name)
def restage(self):
"""Removes the expanded archive path if it exists, then re-expands
the archive.
"""
self.fetcher.reset()
def destroy(self):
"""Remove this stage directory."""
remove_linked_tree(self.path)
@ -345,8 +339,63 @@ def destroy(self):
os.chdir(os.path.dirname(self.path))
class ResourceStage(Stage):
def __init__(self, url_or_fetch_strategy, root, resource, **kwargs):
super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs)
self.root_stage = root
self.resource = resource
def expand_archive(self):
super(ResourceStage, self).expand_archive()
root_stage = self.root_stage
resource = self.resource
placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement
if not isinstance(placement, dict):
placement = {'': placement}
# Make the paths in the dictionary absolute and link
for key, value in placement.iteritems():
target_path = join_path(root_stage.source_path, resource.destination)
destination_path = join_path(target_path, value)
source_path = join_path(self.source_path, key)
try:
os.makedirs(target_path)
except OSError as err:
if err.errno == errno.EEXIST and os.path.isdir(target_path):
pass
else:
raise
if not os.path.exists(destination_path):
# Create a symlink
tty.info('Moving resource stage\n\tsource : {stage}\n\tdestination : {destination}'.format(
stage=source_path, destination=destination_path
))
shutil.move(source_path, destination_path)
@pattern.composite(method_list=['fetch', 'check', 'expand_archive', 'restage', 'destroy'])
class StageComposite:
"""
Composite for Stage type objects. The first item in this composite is considered to be the root package, and
operations that return a value are forwarded to it.
"""
@property
def source_path(self):
return self[0].source_path
@property
def path(self):
return self[0].path
def chdir_to_source(self):
return self[0].chdir_to_source()
class DIYStage(object):
"""Simple class that allows any directory to be a spack stage."""
def __init__(self, path):
self.archive_file = None
self.path = path
@ -384,7 +433,6 @@ def _get_mirrors():
return [val for name, val in config.iteritems()]
def ensure_access(file=spack.stage_path):
"""Ensure we can access a directory and die with an error if we can't."""
if not can_access(file):

1
lib/spack/spack/test/__init__.py
View File

@ -48,6 +48,7 @@
'package_sanity',
'config',
'directory_layout',
'pattern',
'python_version',
'git_fetch',
'svn_fetch',

7
lib/spack/spack/test/install.py
View File

@ -31,7 +31,7 @@
import spack
from spack.stage import Stage
from spack.fetch_strategy import URLFetchStrategy
from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite
from spack.directory_layout import YamlDirectoryLayout
from spack.util.executable import which
from spack.test.mock_packages_test import *
@ -79,7 +79,10 @@ def test_install_and_uninstall(self):
pkg = spack.repo.get(spec)
# Fake the URL for the package so it downloads from a file.
pkg.fetcher = URLFetchStrategy(self.repo.url)
fetcher = FetchStrategyComposite()
fetcher.append(URLFetchStrategy(self.repo.url))
pkg.fetcher = fetcher
try:
pkg.do_install()

1
lib/spack/spack/test/mirror.py
View File

@ -102,6 +102,7 @@ def check_mirror(self):
spec = Spec(name).concretized()
pkg = spec.package
pkg._stage = None
saved_checksum_setting = spack.do_checksum
try:
# Stage the archive from the mirror and cd to it.

104
lib/spack/spack/test/pattern.py Normal file
View File

@ -0,0 +1,104 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import unittest
import spack.util.pattern as pattern
class CompositeTest(unittest.TestCase):
def setUp(self):
class Base:
counter = 0
def add(self):
raise NotImplemented('add not implemented')
def subtract(self):
raise NotImplemented('subtract not implemented')
class One(Base):
def add(self):
Base.counter += 1
def subtract(self):
Base.counter -= 1
class Two(Base):
def add(self):
Base.counter += 2
def subtract(self):
Base.counter -= 2
self.Base = Base
self.One = One
self.Two = Two
def test_composite_from_method_list(self):
@pattern.composite(method_list=['add', 'subtract'])
class CompositeFromMethodList:
pass
composite = CompositeFromMethodList()
composite.append(self.One())
composite.append(self.Two())
composite.add()
self.assertEqual(self.Base.counter, 3)
composite.pop()
composite.subtract()
self.assertEqual(self.Base.counter, 2)
def test_composite_from_interface(self):
@pattern.composite(interface=self.Base)
class CompositeFromInterface:
pass
composite = CompositeFromInterface()
composite.append(self.One())
composite.append(self.Two())
composite.add()
self.assertEqual(self.Base.counter, 3)
composite.pop()
composite.subtract()
self.assertEqual(self.Base.counter, 2)
def test_error_conditions(self):
def wrong_container():
@pattern.composite(interface=self.Base, container=2)
class CompositeFromInterface:
pass
def no_methods():
@pattern.composite()
class CompositeFromInterface:
pass
self.assertRaises(TypeError, wrong_container)
self.assertRaises(TypeError, no_methods)

116
lib/spack/spack/util/pattern.py Normal file
View File

@ -0,0 +1,116 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import inspect
import collections
import functools
def composite(interface=None, method_list=None, container=list):
"""
Returns a class decorator that patches a class adding all the methods it needs to be a composite for a given
interface.
:param interface: class exposing the interface to which the composite object must conform. Only non-private and
non-special methods will be taken into account
:param method_list: names of methods that should be part of the composite
:param container: container for the composite object (default = list). Must fulfill the MutableSequence contract.
The composite class will expose the container API to manage object composition
:return: class decorator
"""
# Check if container fulfills the MutableSequence contract and raise an exception if it doesn't
# The patched class returned by the decorator will inherit from the container class to expose the
# interface needed to manage objects composition
if not issubclass(container, collections.MutableSequence):
raise TypeError("Container must fulfill the MutableSequence contract")
# Check if at least one of the 'interface' or the 'method_list' arguments are defined
if interface is None and method_list is None:
raise TypeError("Either 'interface' or 'method_list' must be defined on a call to composite")
def cls_decorator(cls):
# Retrieve the base class of the composite. Inspect its methods and decide which ones will be overridden
def no_special_no_private(x):
return inspect.ismethod(x) and not x.__name__.startswith('_')
# Patch the behavior of each of the methods in the previous list. This is done associating an instance of the
# descriptor below to any method that needs to be patched.
class IterateOver(object):
"""
Decorator used to patch methods in a composite. It iterates over all the items in the instance containing the
associated attribute and calls for each of them an attribute with the same name
"""
def __init__(self, name, func=None):
self.name = name
self.func = func
def __get__(self, instance, owner):
def getter(*args, **kwargs):
for item in instance:
getattr(item, self.name)(*args, **kwargs)
# If we are using this descriptor to wrap a method from an interface, then we must conditionally
# use the `functools.wraps` decorator to set the appropriate fields.
if self.func is not None:
getter = functools.wraps(self.func)(getter)
return getter
dictionary_for_type_call = {}
# Construct a dictionary with the methods explicitly passed as name
if method_list is not None:
# python@2.7: method_list_dict = {name: IterateOver(name) for name in method_list}
method_list_dict = {}
for name in method_list:
method_list_dict[name] = IterateOver(name)
dictionary_for_type_call.update(method_list_dict)
# Construct a dictionary with the methods inspected from the interface
if interface is not None:
##########
# python@2.7: interface_methods = {name: method for name, method in inspect.getmembers(interface, predicate=no_special_no_private)}
interface_methods = {}
for name, method in inspect.getmembers(interface, predicate=no_special_no_private):
interface_methods[name] = method
##########
# python@2.7: interface_methods_dict = {name: IterateOver(name, method) for name, method in interface_methods.iteritems()}
interface_methods_dict = {}
for name, method in interface_methods.iteritems():
interface_methods_dict[name] = IterateOver(name, method)
##########
dictionary_for_type_call.update(interface_methods_dict)
# Get the methods that are defined in the scope of the composite class and override any previous definition
##########
# python@2.7: cls_method = {name: method for name, method in inspect.getmembers(cls, predicate=inspect.ismethod)}
cls_method = {}
for name, method in inspect.getmembers(cls, predicate=inspect.ismethod):
cls_method[name] = method
##########
dictionary_for_type_call.update(cls_method)
# Generate the new class on the fly and return it
# FIXME : inherit from interface if we start to use ABC classes?
wrapper_class = type(cls.__name__, (cls, container), dictionary_for_type_call)
return wrapper_class
return cls_decorator

19
var/spack/repos/builtin/packages/llvm/package.py
View File

@ -171,6 +171,25 @@ class Llvm(Package):
when='@%(version)s' % release,
placement=resources[name].get('placement', None))
# SVN - current develop
version('develop', svn='http://llvm.org/svn/llvm-project/llvm/trunk')
resource(name='clang', svn='http://llvm.org/svn/llvm-project/cfe/trunk',
destination='tools', when='@develop', placement='clang')
resource(name='compiler-rt', svn='http://llvm.org/svn/llvm-project/compiler-rt/trunk',
destination='projects', when='@develop', placement='compiler-rt')
resource(name='openmp', svn='http://llvm.org/svn/llvm-project/openmp/trunk',
destination='projects', when='@develop', placement='openmp')
resource(name='libcxx', svn='http://llvm.org/svn/llvm-project/libcxx/trunk',
destination='projects', when='@develop', placement='libcxx')
resource(name='libcxxabi', svn='http://llvm.org/svn/llvm-project/libcxxabi/trunk',
destination='projects', when='@develop', placement='libcxxabi')
resource(name='polly', svn='http://llvm.org/svn/llvm-project/polly/trunk',
destination='tools', when='@develop', placement='polly')
resource(name='lldb', svn='http://llvm.org/svn/llvm-project/lldb/trunk',
destination='tools', when='@develop', placement='lldb')
def install(self, spec, prefix):
env['CXXFLAGS'] = self.compiler.cxx11_flag
cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ]