Package : factored out code in do_stage and do_fetch, changed mirror command accordingly
This commit is contained in:
alalazo
parent
dcddb19e5b
commit
f499f71f64
@ -55,53 +55,59 @@
|
||||
from spack.version import Version, ver
|
||||
from spack.util.compression import decompressor_for, extension
|
||||
|
||||
import spack.util.pattern as pattern
|
||||
|
||||
"""List of all fetch strategies, created by FetchStrategy metaclass."""
|
||||
all_strategies = []
|
||||
|
||||
|
||||
def _needs_stage(fun):
|
||||
"""Many methods on fetch strategies require a stage to be set
|
||||
using set_stage(). This decorator adds a check for self.stage."""
|
||||
|
||||
@wraps(fun)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
if not self.stage:
|
||||
raise NoStageError(fun)
|
||||
return fun(self, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class FetchStrategy(object):
|
||||
"""Superclass of all fetch strategies."""
|
||||
enabled = False # Non-abstract subclasses should be enabled.
|
||||
enabled = False # Non-abstract subclasses should be enabled.
|
||||
required_attributes = None # Attributes required in version() args.
|
||||
|
||||
class __metaclass__(type):
|
||||
"""This metaclass registers all fetch strategies in a list."""
|
||||
|
||||
def __init__(cls, name, bases, dict):
|
||||
type.__init__(cls, name, bases, dict)
|
||||
if cls.enabled: all_strategies.append(cls)
|
||||
|
||||
|
||||
def __init__(self):
|
||||
# The stage is initialized late, so that fetch strategies can be constructed
|
||||
# at package construction time. This is where things will be fetched.
|
||||
self.stage = None
|
||||
|
||||
|
||||
def set_stage(self, stage):
|
||||
"""This is called by Stage before any of the fetching
|
||||
methods are called on the stage."""
|
||||
self.stage = stage
|
||||
|
||||
|
||||
# Subclasses need to implement these methods
|
||||
def fetch(self): pass # Return True on success, False on fail.
|
||||
|
||||
def check(self): pass # Do checksum.
|
||||
|
||||
def expand(self): pass # Expand archive.
|
||||
|
||||
def reset(self): pass # Revert to freshly downloaded state.
|
||||
|
||||
def archive(self, destination): pass # Used to create tarball for mirror.
|
||||
|
||||
def __str__(self): # Should be human readable URL.
|
||||
def __str__(self): # Should be human readable URL.
|
||||
return "FetchStrategy.__str___"
|
||||
|
||||
# This method is used to match fetch strategies to version()
|
||||
@ -111,6 +117,15 @@ def matches(cls, args):
|
||||
return any(k in args for k in cls.required_attributes)
|
||||
|
||||
|
||||
@pattern.composite(interface=FetchStrategy)
|
||||
class FetchStrategyComposite(object):
|
||||
"""
|
||||
Composite for a FetchStrategy object. Implements the GoF composite pattern.
|
||||
"""
|
||||
matches = FetchStrategy.matches
|
||||
set_stage = FetchStrategy.set_stage
|
||||
|
||||
|
||||
class URLFetchStrategy(FetchStrategy):
|
||||
"""FetchStrategy that pulls source code from a URL for an archive,
|
||||
checks the archive against a checksum,and decompresses the archive.
|
||||
@ -142,19 +157,19 @@ def fetch(self):
|
||||
|
||||
tty.msg("Trying to fetch from %s" % self.url)
|
||||
|
||||
curl_args = ['-O', # save file to disk
|
||||
'-f', # fail on >400 errors
|
||||
'-D', '-', # print out HTML headers
|
||||
'-L', self.url,]
|
||||
curl_args = ['-O', # save file to disk
|
||||
'-f', # fail on >400 errors
|
||||
'-D', '-', # print out HTML headers
|
||||
'-L', self.url, ]
|
||||
|
||||
if sys.stdout.isatty():
|
||||
curl_args.append('-#') # status bar when using a tty
|
||||
else:
|
||||
curl_args.append('-sS') # just errors when not.
|
||||
curl_args.append('-sS') # just errors when not.
|
||||
|
||||
# Run curl but grab the mime type from the http headers
|
||||
headers = spack.curl(
|
||||
*curl_args, return_output=True, fail_on_error=False)
|
||||
*curl_args, return_output=True, fail_on_error=False)
|
||||
|
||||
if spack.curl.returncode != 0:
|
||||
# clean up archive on failure.
|
||||
@ -164,24 +179,23 @@ def fetch(self):
|
||||
if spack.curl.returncode == 22:
|
||||
# This is a 404. Curl will print the error.
|
||||
raise FailedDownloadError(
|
||||
self.url, "URL %s was not found!" % self.url)
|
||||
self.url, "URL %s was not found!" % self.url)
|
||||
|
||||
elif spack.curl.returncode == 60:
|
||||
# This is a certificate error. Suggest spack -k
|
||||
raise FailedDownloadError(
|
||||
self.url,
|
||||
"Curl was unable to fetch due to invalid certificate. "
|
||||
"This is either an attack, or your cluster's SSL configuration "
|
||||
"is bad. If you believe your SSL configuration is bad, you "
|
||||
"can try running spack -k, which will not check SSL certificates."
|
||||
"Use this at your own risk.")
|
||||
self.url,
|
||||
"Curl was unable to fetch due to invalid certificate. "
|
||||
"This is either an attack, or your cluster's SSL configuration "
|
||||
"is bad. If you believe your SSL configuration is bad, you "
|
||||
"can try running spack -k, which will not check SSL certificates."
|
||||
"Use this at your own risk.")
|
||||
|
||||
else:
|
||||
# This is some other curl error. Curl will print the
|
||||
# error, but print a spack message too
|
||||
raise FailedDownloadError(
|
||||
self.url, "Curl failed with error %d" % spack.curl.returncode)
|
||||
|
||||
self.url, "Curl failed with error %d" % spack.curl.returncode)
|
||||
|
||||
# Check if we somehow got an HTML file rather than the archive we
|
||||
# asked for. We only look at the last content type, to handle
|
||||
@ -196,7 +210,6 @@ def fetch(self):
|
||||
if not self.archive_file:
|
||||
raise FailedDownloadError(self.url)
|
||||
|
||||
|
||||
@property
|
||||
def archive_file(self):
|
||||
"""Path to the source archive within this stage directory."""
|
||||
@ -209,7 +222,7 @@ def expand(self):
|
||||
self.stage.chdir()
|
||||
if not self.archive_file:
|
||||
raise NoArchiveFileError("URLFetchStrategy couldn't find archive file",
|
||||
"Failed on expand() for URL %s" % self.url)
|
||||
"Failed on expand() for URL %s" % self.url)
|
||||
|
||||
decompress = decompressor_for(self.archive_file)
|
||||
|
||||
@ -241,7 +254,6 @@ def expand(self):
|
||||
# Set the wd back to the stage when done.
|
||||
self.stage.chdir()
|
||||
|
||||
|
||||
def archive(self, destination):
|
||||
"""Just moves this archive to the destination."""
|
||||
if not self.archive_file:
|
||||
@ -252,7 +264,6 @@ def archive(self, destination):
|
||||
|
||||
shutil.move(self.archive_file, destination)
|
||||
|
||||
|
||||
@_needs_stage
|
||||
def check(self):
|
||||
"""Check the downloaded archive against a checksum digest.
|
||||
@ -263,9 +274,8 @@ def check(self):
|
||||
checker = crypto.Checker(self.digest)
|
||||
if not checker.check(self.archive_file):
|
||||
raise ChecksumError(
|
||||
"%s checksum failed for %s." % (checker.hash_name, self.archive_file),
|
||||
"Expected %s but got %s." % (self.digest, checker.sum))
|
||||
|
||||
"%s checksum failed for %s." % (checker.hash_name, self.archive_file),
|
||||
"Expected %s but got %s." % (self.digest, checker.sum))
|
||||
|
||||
@_needs_stage
|
||||
def reset(self):
|
||||
@ -277,12 +287,10 @@ def reset(self):
|
||||
shutil.rmtree(self.stage.source_path, ignore_errors=True)
|
||||
self.expand()
|
||||
|
||||
|
||||
def __repr__(self):
|
||||
url = self.url if self.url else "no url"
|
||||
return "URLFetchStrategy<%s>" % url
|
||||
|
||||
|
||||
def __str__(self):
|
||||
if self.url:
|
||||
return self.url
|
||||
@ -298,33 +306,30 @@ def __init__(self, name, *rev_types, **kwargs):
|
||||
# Set a URL based on the type of fetch strategy.
|
||||
self.url = kwargs.get(name, None)
|
||||
if not self.url: raise ValueError(
|
||||
"%s requires %s argument." % (self.__class__, name))
|
||||
"%s requires %s argument." % (self.__class__, name))
|
||||
|
||||
# Ensure that there's only one of the rev_types
|
||||
if sum(k in kwargs for k in rev_types) > 1:
|
||||
raise FetchStrategyError(
|
||||
"Supply only one of %s to fetch with %s." % (
|
||||
comma_or(rev_types), name))
|
||||
"Supply only one of %s to fetch with %s." % (
|
||||
comma_or(rev_types), name))
|
||||
|
||||
# Set attributes for each rev type.
|
||||
for rt in rev_types:
|
||||
setattr(self, rt, kwargs.get(rt, None))
|
||||
|
||||
|
||||
@_needs_stage
|
||||
def check(self):
|
||||
tty.msg("No checksum needed when fetching with %s." % self.name)
|
||||
|
||||
|
||||
@_needs_stage
|
||||
def expand(self):
|
||||
tty.debug("Source fetched with %s is already expanded." % self.name)
|
||||
|
||||
|
||||
@_needs_stage
|
||||
def archive(self, destination, **kwargs):
|
||||
assert(extension(destination) == 'tar.gz')
|
||||
assert(self.stage.source_path.startswith(self.stage.path))
|
||||
assert (extension(destination) == 'tar.gz')
|
||||
assert (self.stage.source_path.startswith(self.stage.path))
|
||||
|
||||
tar = which('tar', required=True)
|
||||
|
||||
@ -338,16 +343,13 @@ def archive(self, destination, **kwargs):
|
||||
self.stage.chdir()
|
||||
tar('-czf', destination, os.path.basename(self.stage.source_path))
|
||||
|
||||
|
||||
def __str__(self):
|
||||
return "VCS: %s" % self.url
|
||||
|
||||
|
||||
def __repr__(self):
|
||||
return "%s<%s>" % (self.__class__, self.url)
|
||||
|
||||
|
||||
|
||||
class GitFetchStrategy(VCSFetchStrategy):
|
||||
"""Fetch strategy that gets source code from a git repository.
|
||||
Use like this in a package:
|
||||
@ -369,23 +371,20 @@ class GitFetchStrategy(VCSFetchStrategy):
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super(GitFetchStrategy, self).__init__(
|
||||
'git', 'tag', 'branch', 'commit', **kwargs)
|
||||
'git', 'tag', 'branch', 'commit', **kwargs)
|
||||
self._git = None
|
||||
|
||||
|
||||
@property
|
||||
def git_version(self):
|
||||
vstring = self.git('--version', return_output=True).lstrip('git version ')
|
||||
return Version(vstring)
|
||||
|
||||
|
||||
@property
|
||||
def git(self):
|
||||
if not self._git:
|
||||
self._git = which('git', required=True)
|
||||
return self._git
|
||||
|
||||
|
||||
@_needs_stage
|
||||
def fetch(self):
|
||||
self.stage.chdir()
|
||||
@ -418,7 +417,7 @@ def fetch(self):
|
||||
if self.branch:
|
||||
args.extend(['--branch', self.branch])
|
||||
elif self.tag and self.git_version >= ver('1.8.5.2'):
|
||||
args.extend(['--branch', self.tag])
|
||||
args.extend(['--branch', self.tag])
|
||||
|
||||
# Try to be efficient if we're using a new enough git.
|
||||
# This checks out only one branch's history
|
||||
@ -429,7 +428,7 @@ def fetch(self):
|
||||
# Yet more efficiency, only download a 1-commit deep tree
|
||||
if self.git_version >= ver('1.7.1'):
|
||||
try:
|
||||
self.git(*(args + ['--depth','1', self.url]))
|
||||
self.git(*(args + ['--depth', '1', self.url]))
|
||||
cloned = True
|
||||
except spack.error.SpackError:
|
||||
# This will fail with the dumb HTTP transport
|
||||
@ -452,18 +451,15 @@ def fetch(self):
|
||||
self.git('pull', '--tags', ignore_errors=1)
|
||||
self.git('checkout', self.tag)
|
||||
|
||||
|
||||
def archive(self, destination):
|
||||
super(GitFetchStrategy, self).archive(destination, exclude='.git')
|
||||
|
||||
|
||||
@_needs_stage
|
||||
def reset(self):
|
||||
self.stage.chdir_to_source()
|
||||
self.git('checkout', '.')
|
||||
self.git('clean', '-f')
|
||||
|
||||
|
||||
def __str__(self):
|
||||
return "[git] %s" % self.url
|
||||
|
||||
@ -484,19 +480,17 @@ class SvnFetchStrategy(VCSFetchStrategy):
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super(SvnFetchStrategy, self).__init__(
|
||||
'svn', 'revision', **kwargs)
|
||||
'svn', 'revision', **kwargs)
|
||||
self._svn = None
|
||||
if self.revision is not None:
|
||||
self.revision = str(self.revision)
|
||||
|
||||
|
||||
@property
|
||||
def svn(self):
|
||||
if not self._svn:
|
||||
self._svn = which('svn', required=True)
|
||||
return self._svn
|
||||
|
||||
|
||||
@_needs_stage
|
||||
def fetch(self):
|
||||
self.stage.chdir()
|
||||
@ -515,7 +509,6 @@ def fetch(self):
|
||||
self.svn(*args)
|
||||
self.stage.chdir_to_source()
|
||||
|
||||
|
||||
def _remove_untracked_files(self):
|
||||
"""Removes untracked files in an svn repository."""
|
||||
status = self.svn('status', '--no-ignore', return_output=True)
|
||||
@ -529,23 +522,19 @@ def _remove_untracked_files(self):
|
||||
elif os.path.isdir(path):
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
|
||||
|
||||
def archive(self, destination):
|
||||
super(SvnFetchStrategy, self).archive(destination, exclude='.svn')
|
||||
|
||||
|
||||
@_needs_stage
|
||||
def reset(self):
|
||||
self.stage.chdir_to_source()
|
||||
self._remove_untracked_files()
|
||||
self.svn('revert', '.', '-R')
|
||||
|
||||
|
||||
def __str__(self):
|
||||
return "[svn] %s" % self.url
|
||||
|
||||
|
||||
|
||||
class HgFetchStrategy(VCSFetchStrategy):
|
||||
"""Fetch strategy that gets source code from a Mercurial repository.
|
||||
Use like this in a package:
|
||||
@ -568,10 +557,9 @@ class HgFetchStrategy(VCSFetchStrategy):
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super(HgFetchStrategy, self).__init__(
|
||||
'hg', 'revision', **kwargs)
|
||||
'hg', 'revision', **kwargs)
|
||||
self._hg = None
|
||||
|
||||
|
||||
@property
|
||||
def hg(self):
|
||||
if not self._hg:
|
||||
@ -597,11 +585,9 @@ def fetch(self):
|
||||
|
||||
self.hg(*args)
|
||||
|
||||
|
||||
def archive(self, destination):
|
||||
super(HgFetchStrategy, self).archive(destination, exclude='.hg')
|
||||
|
||||
|
||||
@_needs_stage
|
||||
def reset(self):
|
||||
self.stage.chdir()
|
||||
@ -619,7 +605,6 @@ def reset(self):
|
||||
shutil.move(scrubbed, source_path)
|
||||
self.stage.chdir_to_source()
|
||||
|
||||
|
||||
def __str__(self):
|
||||
return "[hg] %s" % self.url
|
||||
|
||||
@ -693,9 +678,10 @@ def __init__(self, msg, long_msg):
|
||||
|
||||
class FailedDownloadError(FetchError):
|
||||
"""Raised wen a download fails."""
|
||||
|
||||
def __init__(self, url, msg=""):
|
||||
super(FailedDownloadError, self).__init__(
|
||||
"Failed to fetch file from URL: %s" % url, msg)
|
||||
"Failed to fetch file from URL: %s" % url, msg)
|
||||
self.url = url
|
||||
|
||||
|
||||
@ -718,12 +704,14 @@ def __init__(self, pkg, version):
|
||||
|
||||
class ChecksumError(FetchError):
|
||||
"""Raised when archive fails to checksum."""
|
||||
|
||||
def __init__(self, message, long_msg=None):
|
||||
super(ChecksumError, self).__init__(message, long_msg)
|
||||
|
||||
|
||||
class NoStageError(FetchError):
|
||||
"""Raised when fetch operations are called before set_stage()."""
|
||||
|
||||
def __init__(self, method):
|
||||
super(NoStageError, self).__init__(
|
||||
"Must call FetchStrategy.set_stage() before calling %s" % method.__name__)
|
||||
"Must call FetchStrategy.set_stage() before calling %s" % method.__name__)
|
||||
|
||||
@ -45,12 +45,11 @@
|
||||
from spack.util.compression import extension, allowed_archive
|
||||
|
||||
|
||||
def mirror_archive_filename(spec):
|
||||
def mirror_archive_filename(spec, fetcher):
|
||||
"""Get the name of the spec's archive in the mirror."""
|
||||
if not spec.version.concrete:
|
||||
raise ValueError("mirror.path requires spec with concrete version.")
|
||||
|
||||
fetcher = spec.package.fetcher
|
||||
if isinstance(fetcher, fs.URLFetchStrategy):
|
||||
# If we fetch this version with a URLFetchStrategy, use URL's archive type
|
||||
ext = url.downloaded_file_extension(fetcher.url)
|
||||
@ -61,9 +60,9 @@ def mirror_archive_filename(spec):
|
||||
return "%s-%s.%s" % (spec.package.name, spec.version, ext)
|
||||
|
||||
|
||||
def mirror_archive_path(spec):
|
||||
def mirror_archive_path(spec, fetcher):
|
||||
"""Get the relative path to the spec's archive within a mirror."""
|
||||
return join_path(spec.name, mirror_archive_filename(spec))
|
||||
return join_path(spec.name, mirror_archive_filename(spec, fetcher))
|
||||
|
||||
|
||||
def get_matching_versions(specs, **kwargs):
|
||||
@ -158,68 +157,47 @@ def create(path, specs, **kwargs):
|
||||
everything_already_exists = True
|
||||
for spec in version_specs:
|
||||
pkg = spec.package
|
||||
|
||||
stage = None
|
||||
tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@")))
|
||||
try:
|
||||
# create a subdirectory for the current package@version
|
||||
archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec)))
|
||||
subdir = os.path.dirname(archive_path)
|
||||
mkdirp(subdir)
|
||||
for ii, stage in enumerate(pkg.stage):
|
||||
fetcher = stage.fetcher
|
||||
if ii == 0:
|
||||
# create a subdirectory for the current package@version
|
||||
archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher)))
|
||||
name = spec.format("$_$@")
|
||||
else:
|
||||
resource = stage.resource
|
||||
archive_path = join_path(subdir, suggest_archive_basename(resource))
|
||||
name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@"))
|
||||
subdir = os.path.dirname(archive_path)
|
||||
mkdirp(subdir)
|
||||
|
||||
if os.path.exists(archive_path):
|
||||
tty.msg("Already added %s" % spec.format("$_$@"))
|
||||
else:
|
||||
everything_already_exists = False
|
||||
# Set up a stage and a fetcher for the download
|
||||
unique_fetch_name = spec.format("$_$@")
|
||||
fetcher = fs.for_package_version(pkg, pkg.version)
|
||||
stage = Stage(fetcher, name=unique_fetch_name)
|
||||
fetcher.set_stage(stage)
|
||||
if os.path.exists(archive_path):
|
||||
tty.msg("{name} : already added".format(name=name))
|
||||
else:
|
||||
everything_already_exists = False
|
||||
fetcher.fetch()
|
||||
if not kwargs.get('no_checksum', False):
|
||||
fetcher.check()
|
||||
tty.msg("{name} : checksum passed".format(name=name))
|
||||
|
||||
# Do the fetch and checksum if necessary
|
||||
fetcher.fetch()
|
||||
if not kwargs.get('no_checksum', False):
|
||||
fetcher.check()
|
||||
tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
|
||||
|
||||
# Fetchers have to know how to archive their files. Use
|
||||
# that to move/copy/create an archive in the mirror.
|
||||
fetcher.archive(archive_path)
|
||||
tty.msg("Added %s." % spec.format("$_$@"))
|
||||
|
||||
# Fetch resources if they are associated with the spec
|
||||
resources = pkg._get_resources()
|
||||
for resource in resources:
|
||||
resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
|
||||
if os.path.exists(resource_archive_path):
|
||||
tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
|
||||
continue
|
||||
everything_already_exists = False
|
||||
resource_stage_folder = pkg._resource_stage(resource)
|
||||
resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
|
||||
resource.fetcher.set_stage(resource_stage)
|
||||
resource.fetcher.fetch()
|
||||
if not kwargs.get('no_checksum', False):
|
||||
resource.fetcher.check()
|
||||
tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
|
||||
resource.fetcher.archive(resource_archive_path)
|
||||
tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
|
||||
# Fetchers have to know how to archive their files. Use
|
||||
# that to move/copy/create an archive in the mirror.
|
||||
fetcher.archive(archive_path)
|
||||
tty.msg("{name} : added".format(name=name))
|
||||
|
||||
if everything_already_exists:
|
||||
present.append(spec)
|
||||
else:
|
||||
mirrored.append(spec)
|
||||
|
||||
except Exception, e:
|
||||
if spack.debug:
|
||||
sys.excepthook(*sys.exc_info())
|
||||
else:
|
||||
tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message)
|
||||
error.append(spec)
|
||||
|
||||
finally:
|
||||
if stage:
|
||||
stage.destroy()
|
||||
pkg.stage.destroy()
|
||||
|
||||
return (present, mirrored, error)
|
||||
|
||||
|
||||
@ -61,7 +61,7 @@
|
||||
import spack.util.web
|
||||
import spack.fetch_strategy as fs
|
||||
from spack.version import *
|
||||
from spack.stage import Stage
|
||||
from spack.stage import Stage, ResourceStage, StageComposite
|
||||
from spack.util.compression import allowed_archive, extension
|
||||
from spack.util.executable import ProcessError
|
||||
|
||||
@ -431,23 +431,47 @@ def url_for_version(self, version):
|
||||
return spack.url.substitute_version(self.nearest_url(version),
|
||||
self.url_version(version))
|
||||
|
||||
def _make_resource_stage(self, root_stage, fetcher, resource):
|
||||
resource_stage_folder = self._resource_stage(resource)
|
||||
# FIXME : works only for URLFetchStrategy
|
||||
resource_mirror = join_path(self.name, os.path.basename(fetcher.url))
|
||||
stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource,
|
||||
name=resource_stage_folder, mirror_path=resource_mirror)
|
||||
return stage
|
||||
|
||||
def _make_root_stage(self, fetcher):
|
||||
# Construct a mirror path (TODO: get this out of package.py)
|
||||
mp = spack.mirror.mirror_archive_path(self.spec, fetcher)
|
||||
# Construct a path where the stage should build..
|
||||
s = self.spec
|
||||
stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
|
||||
# Build the composite stage
|
||||
stage = Stage(fetcher, mirror_path=mp, name=stage_name)
|
||||
return stage
|
||||
|
||||
def _make_stage(self):
|
||||
# Construct a composite stage on top of the composite FetchStrategy
|
||||
composite_fetcher = self.fetcher
|
||||
composite_stage = StageComposite()
|
||||
resources = self._get_resources()
|
||||
for ii, fetcher in enumerate(composite_fetcher):
|
||||
if ii == 0:
|
||||
# Construct root stage first
|
||||
stage = self._make_root_stage(fetcher)
|
||||
else:
|
||||
# Construct resource stage
|
||||
resource = resources[ii - 1] # ii == 0 is root!
|
||||
stage = self._make_resource_stage(composite_stage[0], fetcher, resource)
|
||||
# Append the item to the composite
|
||||
composite_stage.append(stage)
|
||||
return composite_stage
|
||||
|
||||
@property
|
||||
def stage(self):
|
||||
if not self.spec.concrete:
|
||||
raise ValueError("Can only get a stage for a concrete package.")
|
||||
|
||||
if self._stage is None:
|
||||
# Construct a mirror path (TODO: get this out of package.py)
|
||||
mp = spack.mirror.mirror_archive_path(self.spec)
|
||||
|
||||
# Construct a path where the stage should build..
|
||||
s = self.spec
|
||||
stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
|
||||
|
||||
# Build the stage
|
||||
self._stage = Stage(self.fetcher, mirror_path=mp, name=stage_name)
|
||||
|
||||
self._stage = self._make_stage()
|
||||
return self._stage
|
||||
|
||||
|
||||
@ -457,17 +481,25 @@ def stage(self, stage):
|
||||
self._stage = stage
|
||||
|
||||
|
||||
def _make_fetcher(self):
|
||||
# Construct a composite fetcher that always contains at least one element (the root package). In case there
|
||||
# are resources associated with the package, append their fetcher to the composite.
|
||||
root_fetcher = fs.for_package_version(self, self.version)
|
||||
fetcher = fs.FetchStrategyComposite() # Composite fetcher
|
||||
fetcher.append(root_fetcher) # Root fetcher is always present
|
||||
resources = self._get_resources()
|
||||
for resource in resources:
|
||||
fetcher.append(resource.fetcher)
|
||||
return fetcher
|
||||
|
||||
@property
|
||||
def fetcher(self):
|
||||
if not self.spec.versions.concrete:
|
||||
raise ValueError(
|
||||
"Can only get a fetcher for a package with concrete versions.")
|
||||
|
||||
raise ValueError("Can only get a fetcher for a package with concrete versions.")
|
||||
if not self._fetcher:
|
||||
self._fetcher = fs.for_package_version(self, self.version)
|
||||
self._fetcher = self._make_fetcher()
|
||||
return self._fetcher
|
||||
|
||||
|
||||
@fetcher.setter
|
||||
def fetcher(self, f):
|
||||
self._fetcher = f
|
||||
@ -630,7 +662,7 @@ def remove_prefix(self):
|
||||
|
||||
|
||||
def do_fetch(self):
|
||||
"""Creates a stage directory and downloads the taball for this package.
|
||||
"""Creates a stage directory and downloads the tarball for this package.
|
||||
Working directory will be set to the stage directory.
|
||||
"""
|
||||
if not self.spec.concrete:
|
||||
@ -656,20 +688,6 @@ def do_fetch(self):
|
||||
|
||||
self.stage.fetch()
|
||||
|
||||
##########
|
||||
# Fetch resources
|
||||
resources = self._get_resources()
|
||||
for resource in resources:
|
||||
resource_stage_folder = self._resource_stage(resource)
|
||||
# FIXME : works only for URLFetchStrategy
|
||||
resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
|
||||
resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
|
||||
resource.fetcher.set_stage(resource_stage)
|
||||
# Delegate to stage object to trigger mirror logic
|
||||
resource_stage.fetch()
|
||||
resource_stage.check()
|
||||
##########
|
||||
|
||||
self._fetch_time = time.time() - start_time
|
||||
|
||||
if spack.do_checksum and self.version in self.versions:
|
||||
@ -681,39 +699,10 @@ def do_stage(self):
|
||||
if not self.spec.concrete:
|
||||
raise ValueError("Can only stage concrete packages.")
|
||||
|
||||
def _expand_archive(stage, name=self.name):
|
||||
archive_dir = stage.source_path
|
||||
if not archive_dir:
|
||||
stage.expand_archive()
|
||||
tty.msg("Created stage in %s." % stage.path)
|
||||
else:
|
||||
tty.msg("Already staged %s in %s." % (name, stage.path))
|
||||
|
||||
|
||||
self.do_fetch()
|
||||
_expand_archive(self.stage)
|
||||
|
||||
##########
|
||||
# Stage resources in appropriate path
|
||||
resources = self._get_resources()
|
||||
for resource in resources:
|
||||
stage = resource.fetcher.stage
|
||||
_expand_archive(stage, resource.name)
|
||||
# Turn placement into a dict with relative paths
|
||||
placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement
|
||||
if not isinstance(placement, dict):
|
||||
placement = {'': placement}
|
||||
# Make the paths in the dictionary absolute and link
|
||||
for key, value in placement.iteritems():
|
||||
link_path = join_path(self.stage.source_path, resource.destination, value)
|
||||
source_path = join_path(stage.source_path, key)
|
||||
if not os.path.exists(link_path):
|
||||
# Create a symlink
|
||||
os.symlink(source_path, link_path)
|
||||
##########
|
||||
self.stage.expand_archive()
|
||||
self.stage.chdir_to_source()
|
||||
|
||||
|
||||
def do_patch(self):
|
||||
"""Calls do_stage(), then applied patches to the expanded tarball if they
|
||||
haven't been applied already."""
|
||||
|
||||
@ -30,6 +30,8 @@
|
||||
import llnl.util.tty as tty
|
||||
from llnl.util.filesystem import *
|
||||
|
||||
import spack.util.pattern as pattern
|
||||
|
||||
import spack
|
||||
import spack.config
|
||||
import spack.fetch_strategy as fs
|
||||
@ -40,7 +42,7 @@
|
||||
|
||||
|
||||
class Stage(object):
|
||||
"""A Stage object manaages a directory where some source code is
|
||||
"""A Stage object manages a directory where some source code is
|
||||
downloaded and built before being installed. It handles
|
||||
fetching the source code, either as an archive to be expanded
|
||||
or by checking it out of a repository. A stage's lifecycle
|
||||
@ -276,7 +278,12 @@ def expand_archive(self):
|
||||
archive. Fail if the stage is not set up or if the archive is not yet
|
||||
downloaded.
|
||||
"""
|
||||
self.fetcher.expand()
|
||||
archive_dir = self.source_path
|
||||
if not archive_dir:
|
||||
self.fetcher.expand()
|
||||
tty.msg("Created stage in %s." % self.path)
|
||||
else:
|
||||
tty.msg("Already staged %s in %s." % (self.name, self.path))
|
||||
|
||||
|
||||
def chdir_to_source(self):
|
||||
@ -310,6 +317,46 @@ def destroy(self):
|
||||
os.chdir(os.path.dirname(self.path))
|
||||
|
||||
|
||||
class ResourceStage(Stage):
|
||||
def __init__(self, url_or_fetch_strategy, root, resource, **kwargs):
|
||||
super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs)
|
||||
self.root_stage = root
|
||||
self.resource = resource
|
||||
|
||||
def expand_archive(self):
|
||||
super(ResourceStage, self).expand_archive()
|
||||
root_stage = self.root_stage
|
||||
resource = self.resource
|
||||
placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement
|
||||
if not isinstance(placement, dict):
|
||||
placement = {'': placement}
|
||||
# Make the paths in the dictionary absolute and link
|
||||
for key, value in placement.iteritems():
|
||||
link_path = join_path(root_stage.source_path, resource.destination, value)
|
||||
source_path = join_path(self.source_path, key)
|
||||
if not os.path.exists(link_path):
|
||||
# Create a symlink
|
||||
os.symlink(source_path, link_path)
|
||||
|
||||
|
||||
@pattern.composite(method_list=['fetch', 'check', 'expand_archive', 'restage', 'destroy'])
|
||||
class StageComposite:
|
||||
"""
|
||||
Composite for Stage type objects. The first item in this composite is considered to be the root package, and
|
||||
operations that return a value are forwarded to it.
|
||||
"""
|
||||
@property
|
||||
def source_path(self):
|
||||
return self[0].source_path
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
return self[0].path
|
||||
|
||||
def chdir_to_source(self):
|
||||
return self[0].chdir_to_source()
|
||||
|
||||
|
||||
class DIYStage(object):
|
||||
"""Simple class that allows any directory to be a spack stage."""
|
||||
def __init__(self, path):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user