zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Callpath build works when a tag is fetched from git.

Browse Source
This commit is contained in:
Todd Gamblin
2014-09-16 16:56:16 -07:00
parent 0cc79e0564
commit c74cd63389
5 changed files with 199 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
lib/spack/spack/fetch_strategy.py
View File

@@ -47,12 +47,27 @@
import spack
import spack.error
import spack.util.crypto as crypto
from spack.util.executable import *
from spack.util.string import *
from spack.version import Version, ver
from spack.util.compression import decompressor_for
"""List of all fetch strategies, created by FetchStrategy metaclass."""
all_strategies = []
class FetchStrategy(object):
"""Superclass of all fetch strategies."""
enabled = False # Non-abstract subclasses should be enabled.
required_attributes = None # Attributes required in version() args.
class __metaclass__(type):
"""This metaclass registers all fetch strategies in a list."""
def __init__(cls, name, bases, dict):
type.__init__(cls, name, bases, dict)
if cls.enabled: all_strategies.append(cls)
def __init__(self):
# The stage is initialized late, so that fetch strategies can be constructed
# at package construction time. This is where things will be fetched.
@@ -76,12 +91,16 @@ def __str__(self):
# This method is used to match fetch strategies to version()
# arguments in packages.
@classmethod
def match(kwargs):
return any(k in kwargs for k in self.attributes)
def matches(cls, args):
return any(k in args for k in cls.required_attributes)
class URLFetchStrategy(FetchStrategy):
attributes = ('url', 'md5')
"""FetchStrategy that pulls source code from a URL for an archive,
checks the archive against a checksum,and decompresses the archive.
"""
enabled = True
required_attributes = ['url']
def __init__(self, url=None, digest=None, **kwargs):
super(URLFetchStrategy, self).__init__()
@@ -172,11 +191,11 @@ def check(self):
assert(self.stage)
if not self.digest:
raise NoDigestError("Attempt to check URLFetchStrategy with no digest.")
checker = crypto.Checker(digest)
checker = crypto.Checker(self.digest)
if not checker.check(self.archive_file):
raise ChecksumError(
"%s checksum failed for %s." % (checker.hash_name, self.archive_file),
"Expected %s but got %s." % (digest, checker.sum))
"Expected %s but got %s." % (self.digest, checker.sum))
def reset(self):
@@ -191,46 +210,73 @@ def reset(self):
def __str__(self):
if self.url:
return self.url
else:
return "URLFetchStrategy <no url>"
url = self.url if self.url else "no url"
return "URLFetchStrategy<%s>" % url
class VCSFetchStrategy(FetchStrategy):
def __init__(self, name):
def __init__(self, name, *rev_types, **kwargs):
super(VCSFetchStrategy, self).__init__()
self.name = name
# Set a URL based on the type of fetch strategy.
self.url = kwargs.get(name, None)
if not self.url: raise ValueError(
"%s requires %s argument." % (self.__class__, name))
# Ensure that there's only one of the rev_types
if sum((k in kwargs for k in rev_types)) > 1:
raise FetchStrategyError(
"Supply only one of %s to fetch with %s." % (
comma_or(rev_types), name))
# Set attributes for each rev type.
for rt in rev_types:
setattr(self, rt, getattr(kwargs, rt, None))
def check(self):
assert(self.stage)
tty.msg("No check needed when fetching with %s." % self.name)
tty.msg("No checksum needed when fetching with %s." % self.name)
def expand(self):
assert(self.stage)
tty.debug("Source fetched with %s is already expanded." % self.name)
def __str__(self):
return "%s<%s>" % (self.__class__, self.url)
class GitFetchStrategy(VCSFetchStrategy):
attributes = ('git', 'ref', 'tag', 'branch')
"""Fetch strategy that gets source code from a git repository.
Use like this in a package:
version('name', git='https://github.com/project/repo.git')
Optionally, you can provide a branch, or commit to check out, e.g.:
version('1.1', git='https://github.com/project/repo.git', tag='v1.1')
You can use these three optional attributes in addition to ``git``:
* ``branch``: Particular branch to build from (default is master)
* ``tag``: Particular tag to check out
* ``commit``: Particular commit hash in the repo
"""
enabled = True
required_attributes = ('git',)
def __init__(self, **kwargs):
super(GitFetchStrategy, self).__init__("git")
self.url = kwargs.get('git', None)
if not self.url:
raise ValueError("GitFetchStrategy requires git argument.")
if sum((k in kwargs for k in ('ref', 'tag', 'branch'))) > 1:
raise FetchStrategyError(
"Git requires exactly one ref, branch, or tag.")
super(GitFetchStrategy, self).__init__(
'git', 'tag', 'branch', 'commit', **kwargs)
self._git = None
self.ref = kwargs.get('ref', None)
self.branch = kwargs.get('branch', None)
# For git fetch branches and tags the same way.
if not self.branch:
self.branch = kwargs.get('tag', None)
self.branch = self.tag
@property
@@ -252,21 +298,20 @@ def fetch(self):
self.stage.chdir()
if self.stage.source_path:
tty.msg("Already fetched %s." % self.source_path)
tty.msg("Already fetched %s." % self.stage.source_path)
return
tty.msg("Trying to clone git repository: %s" % self.url)
if self.ref:
if self.commit:
# Need to do a regular clone and check out everything if
# they asked for a particular ref.
git('clone', self.url)
self.chdir_to_source()
git('checkout', self.ref)
# they asked for a particular commit.
self.git('clone', self.url)
self.stage.chdir_to_source()
self.git('checkout', self.commit)
else:
# Can be more efficient if not checking out a specific ref.
# Can be more efficient if not checking out a specific commit.
args = ['clone']
# If we want a particular branch ask for it.
@@ -279,26 +324,77 @@ def fetch(self):
args.append('--single-branch')
args.append(self.url)
git(*args)
self.chdir_to_source()
self.git(*args)
self.stage.chdir_to_source()
def reset(self):
assert(self.stage)
git = which('git', required=True)
self.stage.chdir_to_source()
git('checkout', '.')
git('clean', '-f')
self.git('checkout', '.')
self.git('clean', '-f')
def __str__(self):
return self.url
class SvnFetchStrategy(VCSFetchStrategy):
"""Fetch strategy that gets source code from a subversion repository.
Use like this in a package:
version('name', svn='http://www.example.com/svn/trunk')
Optionally, you can provide a revision for the URL:
version('name', svn='http://www.example.com/svn/trunk',
revision='1641')
"""
enabled = True
required_attributes = ['svn']
def __init__(self, **kwargs):
super(SvnFetchStrategy, self).__init__(
'svn', 'revision', **kwargs)
self._svn = None
class SvnFetchStrategy(FetchStrategy):
attributes = ('svn', 'rev', 'revision')
pass
@property
def svn(self):
if not self._svn:
self._svn = which('svn', required=True)
return self._svn
def fetch(self):
assert(self.stage)
self.stage.chdir()
if self.stage.source_path:
tty.msg("Already fetched %s." % self.stage.source_path)
return
tty.msg("Trying to check out svn repository: %s" % self.url)
args = ['checkout', '--force']
if self.revision:
args += ['-r', self.revision]
self.svn(*args)
self.stage.chdir_to_source()
def _remove_untracked_files(self):
"""Removes untracked files in an svn repository."""
status = self.svn('status', '--no-ignore', check_output=True)
for line in status.split('\n'):
if not re.match('^[I?]'):
continue
path = line[8:].strip()
shutil.rmtree(path, ignore_errors=True)
def reset(self):
assert(self.stage)
self.stage.chdir_to_source()
self._remove_untracked_files()
self.svn('revert', '.', '-R')
def from_url(url):
@@ -312,25 +408,31 @@ def from_url(url):
def args_are_for(args, fetcher):
return any(arg in args for arg in fetcher.attributes)
fetcher.matches(args)
def from_args(args, pkg):
"""Determine a fetch strategy based on the arguments supplied to
version() in the package description."""
fetchers = (URLFetchStrategy, GitFetchStrategy)
for fetcher in fetchers:
if args_are_for(args, fetcher):
attrs = {}
for attr in fetcher.attributes:
default = getattr(pkg, attr, None)
if default:
attrs[attr] = default
attrs.update(args)
# Test all strategies against per-version arguments.
for fetcher in all_strategies:
if fetcher.matches(args):
return fetcher(**args)
# If nothing matched for a *specific* version, test all strategies
# against
for fetcher in all_strategies:
attrs = dict((attr, getattr(pkg, attr, None))
for attr in fetcher.required_attributes)
attrs.update(args)
if fetcher.matches(attrs):
return fetcher(**attrs)
return None
raise InvalidArgsError(
"Could not construct fetch strategy for package %s",
pkg.spec.format("%_%@"))
class FetchStrategyError(spack.error.SpackError):
def __init__(self, msg, long_msg):

27
lib/spack/spack/package.py
View File

@@ -453,19 +453,18 @@ def stage(self):
raise ValueError("Can only get a stage for a concrete package.")
if self._stage is None:
if not self.url:
raise PackageVersionError(self.version)
# TODO: move this logic into a mirror module.
# TODO: get rid of dependence on extension.
mirror_path = "%s/%s" % (self.name, "%s-%s.%s" % (
self.name, self.version, extension(self.url)))
self._stage = Stage(
self.fetcher, mirror_path=mirror_path, name=self.spec.short_spec)
self.fetcher, mirror_path=self.mirror_path(), name=self.spec.short_spec)
return self._stage
def mirror_path(self):
"""Get path to this package's archive in a mirror."""
filename = "%s-%s." % (self.name, self.version)
filename += extension(self.url) if self.has_url() else "tar.gz"
return "%s/%s" % (self.name, filename)
def preorder_traversal(self, visited=None, **kwargs):
"""This does a preorder traversal of the package's dependence DAG."""
virtual = kwargs.get("virtual", False)
@@ -617,9 +616,7 @@ def do_fetch(self):
self.stage.fetch()
if spack.do_checksum and self.version in self.versions:
digest = self.versions[self.version].checksum
self.stage.check(digest)
tty.msg("Checksum passed for %s@%s" % (self.name, self.version))
self.stage.check()
def do_stage(self):
@@ -645,8 +642,14 @@ def do_patch(self):
if not self.spec.concrete:
raise ValueError("Can only patch concrete packages.")
# Kick off the stage first.
self.do_stage()
# If there are no patches, note it.
if not self.patches:
tty.msg("No patches needed for %s." % self.name)
return
# Construct paths to special files in the archive dir used to
# keep track of whether patches were successfully applied.
archive_dir = self.stage.source_path

6
lib/spack/spack/stage.py
View File

@@ -84,14 +84,12 @@ def __init__(self, url_or_fetch_strategy, **kwargs):
"""
if isinstance(url_or_fetch_strategy, basestring):
self.fetcher = fetch_strategy.from_url(url_or_fetch_strategy)
self.fetcher.set_stage(self)
elif isinstance(url_or_fetch_strategy, fetch_strategy.FetchStrategy):
self.fetcher = url_or_fetch_strategy
else:
raise ValueError("Can't construct Stage without url or fetch strategy")
self.fetcher.set_stage(self)
self.name = kwargs.get('name')
self.mirror_path = kwargs.get('mirror_path')
@@ -260,7 +258,7 @@ def fetch(self):
continue
def check(self, digest):
def check(self):
"""Check the downloaded archive against a checksum digest.
No-op if this stage checks code out of a repository."""
self.fetcher.check()