Add a mirror module that handles new fetch strategies.
- Uses new fetchers to get source - Add archive() method to fetch strategies to support this. - Updated mirror command to use new mirror module
This commit is contained in:
Todd Gamblin
parent
ee23cc2527
commit
fbd7e96680
@ -23,23 +23,19 @@
|
||||
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
##############################################################################
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from contextlib import closing
|
||||
|
||||
from external import argparse
|
||||
import llnl.util.tty as tty
|
||||
from llnl.util.tty.colify import colify
|
||||
from llnl.util.filesystem import mkdirp, join_path
|
||||
|
||||
import spack
|
||||
import spack.cmd
|
||||
import spack.config
|
||||
import spack.mirror
|
||||
from spack.spec import Spec
|
||||
from spack.error import SpackError
|
||||
from spack.stage import Stage
|
||||
from spack.util.compression import extension
|
||||
|
||||
|
||||
description = "Manage mirrors."
|
||||
|
||||
@ -105,15 +101,8 @@ def mirror_list(args):
|
||||
print fmt % (name, val)
|
||||
|
||||
|
||||
def mirror_create(args):
|
||||
"""Create a directory to be used as a spack mirror, and fill it with
|
||||
package archives."""
|
||||
# try to parse specs from the command line first.
|
||||
args.specs = spack.cmd.parse_specs(args.specs)
|
||||
|
||||
# If there is a file, parse each line as a spec and add it to the list.
|
||||
if args.file:
|
||||
with closing(open(args.file, "r")) as stream:
|
||||
def _read_specs_from_file(filename):
|
||||
with closing(open(filename, "r")) as stream:
|
||||
for i, string in enumerate(stream):
|
||||
try:
|
||||
s = Spec(string)
|
||||
@ -123,8 +112,22 @@ def mirror_create(args):
|
||||
tty.die("Parse error in %s, line %d:" % (args.file, i+1),
|
||||
">>> " + string, str(e))
|
||||
|
||||
if not args.specs:
|
||||
args.specs = [Spec(n) for n in spack.db.all_package_names()]
|
||||
|
||||
def mirror_create(args):
|
||||
"""Create a directory to be used as a spack mirror, and fill it with
|
||||
package archives."""
|
||||
# try to parse specs from the command line first.
|
||||
specs = spack.cmd.parse_specs(args.specs)
|
||||
|
||||
# If there is a file, parse each line as a spec and add it to the list.
|
||||
if args.file:
|
||||
if specs:
|
||||
tty.die("Cannot pass specs on the command line with --file.")
|
||||
specs = _read_specs_from_file(args.file)
|
||||
|
||||
# If nothing is passed, use all packages.
|
||||
if not specs:
|
||||
specs = [Spec(n) for n in spack.db.all_package_names()]
|
||||
|
||||
# Default name for directory is spack-mirror-<DATESTAMP>
|
||||
if not args.directory:
|
||||
@ -132,85 +135,23 @@ def mirror_create(args):
|
||||
args.directory = 'spack-mirror-' + timestamp
|
||||
|
||||
# Make sure nothing is in the way.
|
||||
existed = False
|
||||
if os.path.isfile(args.directory):
|
||||
tty.error("%s already exists and is a file." % args.directory)
|
||||
elif os.path.isdir(args.directory):
|
||||
existed = True
|
||||
|
||||
# Create a directory if none exists
|
||||
if not os.path.isdir(args.directory):
|
||||
mkdirp(args.directory)
|
||||
tty.msg("Created new mirror in %s" % args.directory)
|
||||
else:
|
||||
tty.msg("Adding to existing mirror in %s" % args.directory)
|
||||
# Actually do the work to create the mirror
|
||||
present, mirrored, error = spack.mirror.create(args.directory, specs)
|
||||
p, m, e = len(present), len(mirrored), len(error)
|
||||
|
||||
# Things to keep track of while parsing specs.
|
||||
working_dir = os.getcwd()
|
||||
num_mirrored = 0
|
||||
num_error = 0
|
||||
|
||||
# Iterate through packages and download all the safe tarballs for each of them
|
||||
for spec in args.specs:
|
||||
pkg = spec.package
|
||||
|
||||
# Skip any package that has no checksummed versions.
|
||||
if not pkg.versions:
|
||||
tty.msg("No safe (checksummed) versions for package %s."
|
||||
% pkg.name)
|
||||
continue
|
||||
|
||||
# create a subdir for the current package.
|
||||
pkg_path = join_path(args.directory, pkg.name)
|
||||
mkdirp(pkg_path)
|
||||
|
||||
# Download all the tarballs using Stages, then move them into place
|
||||
for version in pkg.versions:
|
||||
# Skip versions that don't match the spec
|
||||
vspec = Spec('%s@%s' % (pkg.name, version))
|
||||
if not vspec.satisfies(spec):
|
||||
continue
|
||||
|
||||
mirror_path = "%s/%s-%s.%s" % (
|
||||
pkg.name, pkg.name, version, extension(pkg.url))
|
||||
|
||||
os.chdir(working_dir)
|
||||
mirror_file = join_path(args.directory, mirror_path)
|
||||
if os.path.exists(mirror_file):
|
||||
tty.msg("Already fetched %s." % mirror_file)
|
||||
num_mirrored += 1
|
||||
continue
|
||||
|
||||
# Get the URL for the version and set up a stage to download it.
|
||||
url = pkg.url_for_version(version)
|
||||
stage = Stage(url)
|
||||
try:
|
||||
# fetch changes directory into the stage
|
||||
stage.fetch()
|
||||
|
||||
if not args.no_checksum and version in pkg.versions:
|
||||
digest = pkg.versions[version]
|
||||
stage.check(digest)
|
||||
tty.msg("Checksum passed for %s@%s" % (pkg.name, version))
|
||||
|
||||
# change back and move the new archive into place.
|
||||
os.chdir(working_dir)
|
||||
shutil.move(stage.archive_file, mirror_file)
|
||||
tty.msg("Added %s to mirror" % mirror_file)
|
||||
num_mirrored += 1
|
||||
|
||||
except Exception, e:
|
||||
tty.warn("Error while fetching %s." % url, e.message)
|
||||
num_error += 1
|
||||
|
||||
finally:
|
||||
stage.destroy()
|
||||
|
||||
# If nothing happened, try to say why.
|
||||
if not num_mirrored:
|
||||
if num_error:
|
||||
tty.error("No packages added to mirror.",
|
||||
"All packages failed to fetch.")
|
||||
else:
|
||||
tty.error("No packages added to mirror. No versions matched specs:")
|
||||
colify(args.specs, indent=4)
|
||||
verb = "updated" if existed else "created"
|
||||
tty.msg(
|
||||
"Successfully %s mirror in %s." % (verb, args.directory),
|
||||
"Archive stats:",
|
||||
" %-4d already present" % p,
|
||||
" %-4d added" % m,
|
||||
" %-4d failed to fetch." % e)
|
||||
|
||||
|
||||
def mirror(parser, args):
|
||||
@ -218,4 +159,5 @@ def mirror(parser, args):
|
||||
'add' : mirror_add,
|
||||
'remove' : mirror_remove,
|
||||
'list' : mirror_list }
|
||||
|
||||
action[args.mirror_command](args)
|
||||
|
||||
@ -37,6 +37,8 @@
|
||||
Restore original state of downloaded code. Used by clean commands.
|
||||
This may just remove the expanded source and re-expand an archive,
|
||||
or it may run something like git reset --hard.
|
||||
* archive()
|
||||
Archive a source directory, e.g. for creating a mirror.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
@ -91,6 +93,9 @@ def archive(self, destination): pass # Used to create tarball for mirror.
|
||||
def __str__(self): # Should be human readable URL.
|
||||
return "FetchStrategy.__str___"
|
||||
|
||||
@property
|
||||
def unique_name(self): pass
|
||||
|
||||
# This method is used to match fetch strategies to version()
|
||||
# arguments in packages.
|
||||
@classmethod
|
||||
@ -189,7 +194,7 @@ def expand(self):
|
||||
|
||||
|
||||
def archive(self, destination):
|
||||
"""This archive"""
|
||||
"""Just moves this archive to the destination."""
|
||||
if not self.archive_file:
|
||||
raise NoArchiveFileError("Cannot call archive() before fetching.")
|
||||
assert(extension(destination) == extension(self.archive_file))
|
||||
@ -231,6 +236,10 @@ def __str__(self):
|
||||
else:
|
||||
return "URLFetchStrategy<no url>"
|
||||
|
||||
@property
|
||||
def unique_name(self):
|
||||
return "spack-fetch-url:%s" % self
|
||||
|
||||
|
||||
class VCSFetchStrategy(FetchStrategy):
|
||||
def __init__(self, name, *rev_types, **kwargs):
|
||||
@ -384,6 +393,17 @@ def reset(self):
|
||||
self.git('clean', '-f')
|
||||
|
||||
|
||||
@property
|
||||
def unique_name(self):
|
||||
name = "spack-fetch-git:%s" % self.url
|
||||
if self.commit:
|
||||
name += "@" + self.commit
|
||||
elif self.branch:
|
||||
name += "@" + self.branch
|
||||
elif self.tag:
|
||||
name += "@" + self.tag
|
||||
|
||||
|
||||
class SvnFetchStrategy(VCSFetchStrategy):
|
||||
"""Fetch strategy that gets source code from a subversion repository.
|
||||
Use like this in a package:
|
||||
@ -457,6 +477,14 @@ def reset(self):
|
||||
self.svn('revert', '.', '-R')
|
||||
|
||||
|
||||
@property
|
||||
def unique_name(self):
|
||||
name = "spack-fetch-svn:%s" % self.url
|
||||
if self.revision:
|
||||
name += "@" + self.revision
|
||||
|
||||
|
||||
|
||||
class HgFetchStrategy(VCSFetchStrategy):
|
||||
"""Fetch strategy that gets source code from a Mercurial repository.
|
||||
Use like this in a package:
|
||||
@ -532,6 +560,14 @@ def reset(self):
|
||||
self.stage.chdir_to_source()
|
||||
|
||||
|
||||
@property
|
||||
def unique_name(self):
|
||||
name = "spack-fetch-hg:%s" % self.url
|
||||
if self.revision:
|
||||
name += "@" + self.revision
|
||||
|
||||
|
||||
|
||||
def from_url(url):
|
||||
"""Given a URL, find an appropriate fetch strategy for it.
|
||||
Currently just gives you a URLFetchStrategy that uses curl.
|
||||
@ -546,9 +582,18 @@ def args_are_for(args, fetcher):
|
||||
fetcher.matches(args)
|
||||
|
||||
|
||||
def from_args(args, pkg):
|
||||
def for_package_version(pkg, version):
|
||||
"""Determine a fetch strategy based on the arguments supplied to
|
||||
version() in the package description."""
|
||||
# If it's not a known version, extrapolate one.
|
||||
if not version in pkg.versions:
|
||||
url = pkg.url_for_verison(version)
|
||||
if not url:
|
||||
raise InvalidArgsError(pkg, version)
|
||||
return URLFetchStrategy()
|
||||
|
||||
# Grab a dict of args out of the package version dict
|
||||
args = pkg.versions[version]
|
||||
|
||||
# Test all strategies against per-version arguments.
|
||||
for fetcher in all_strategies:
|
||||
@ -564,9 +609,7 @@ def from_args(args, pkg):
|
||||
if fetcher.matches(attrs):
|
||||
return fetcher(**attrs)
|
||||
|
||||
raise InvalidArgsError(
|
||||
"Could not construct fetch strategy for package %s",
|
||||
pkg.spec.format("%_%@"))
|
||||
raise InvalidArgsError(pkg, version)
|
||||
|
||||
|
||||
class FetchStrategyError(spack.error.SpackError):
|
||||
@ -593,5 +636,7 @@ def __init__(self, msg, long_msg):
|
||||
|
||||
|
||||
class InvalidArgsError(FetchStrategyError):
|
||||
def __init__(self, msg, long_msg):
|
||||
super(InvalidArgsError, self).__init__(msg, long_msg)
|
||||
def __init__(self, pkg, version):
|
||||
msg = "Could not construct a fetch strategy for package %s at version %s"
|
||||
msg %= (pkg.name, version)
|
||||
super(InvalidArgsError, self).__init__(msg)
|
||||
|
||||
171
lib/spack/spack/mirror.py
Normal file
171
lib/spack/spack/mirror.py
Normal file
@ -0,0 +1,171 @@
|
||||
##############################################################################
|
||||
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
|
||||
# Produced at the Lawrence Livermore National Laboratory.
|
||||
#
|
||||
# This file is part of Spack.
|
||||
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
|
||||
# LLNL-CODE-647188
|
||||
#
|
||||
# For details, see https://scalability-llnl.github.io/spack
|
||||
# Please also see the LICENSE file for our notice and the LGPL.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License (as published by
|
||||
# the Free Software Foundation) version 2.1 dated February 1999.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
|
||||
# conditions of the GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
##############################################################################
|
||||
"""
|
||||
This file contains code for creating spack mirror directories. A
|
||||
mirror is an organized hierarchy containing specially named archive
|
||||
files. This enabled spack to know where to find files in a mirror if
|
||||
the main server for a particualr package is down. Or, if the computer
|
||||
where spack is run is not connected to the internet, it allows spack
|
||||
to download packages directly from a mirror (e.g., on an intranet).
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
import llnl.util.tty as tty
|
||||
from llnl.util.filesystem import *
|
||||
|
||||
import spack
|
||||
import spack.error
|
||||
import spack.fetch_strategy as fs
|
||||
from spack.spec import Spec
|
||||
from spack.stage import Stage
|
||||
from spack.version import *
|
||||
from spack.util.compression import extension
|
||||
|
||||
|
||||
def mirror_archive_filename(spec):
|
||||
"""Get the path that this spec will live at within a mirror."""
|
||||
if not spec.version.concrete:
|
||||
raise ValueError("mirror.path requires spec with concrete version.")
|
||||
|
||||
url = spec.package.default_url
|
||||
if url is None:
|
||||
ext = 'tar.gz'
|
||||
else:
|
||||
ext = extension(url)
|
||||
|
||||
return "%s-%s.%s" % (spec.package.name, spec.version, ext)
|
||||
|
||||
|
||||
def get_matching_versions(specs):
|
||||
"""Get a spec for EACH known version matching any spec in the list."""
|
||||
matching = []
|
||||
for spec in specs:
|
||||
pkg = spec.package
|
||||
|
||||
# Skip any package that has no known versions.
|
||||
if not pkg.versions:
|
||||
tty.msg("No safe (checksummed) versions for package %s." % pkg.name)
|
||||
continue
|
||||
|
||||
for v in reversed(sorted(pkg.versions)):
|
||||
if v.satisfies(spec.versions):
|
||||
s = Spec(pkg.name)
|
||||
s.versions = VersionList([v])
|
||||
matching.append(s)
|
||||
return matching
|
||||
|
||||
|
||||
def create(path, specs, **kwargs):
|
||||
"""Create a directory to be used as a spack mirror, and fill it with
|
||||
package archives.
|
||||
|
||||
Arguments:
|
||||
path Path to create a mirror directory hierarchy in.
|
||||
specs Any package versions matching these specs will be added
|
||||
to the mirror.
|
||||
|
||||
Return Value:
|
||||
Returns a tuple of lists: (present, mirrored, error)
|
||||
* present: Package specs that were already prsent.
|
||||
* mirrored: Package specs that were successfully mirrored.
|
||||
* error: Package specs that failed to mirror due to some error.
|
||||
|
||||
This routine iterates through all known package versions, and
|
||||
it creates specs for those versions. If the version satisfies any spec
|
||||
in the specs list, it is downloaded and added to the mirror.
|
||||
"""
|
||||
# Make sure nothing is in the way.
|
||||
if os.path.isfile(path):
|
||||
raise MirrorError("%s already exists and is a file." % path)
|
||||
|
||||
# automatically spec-ify anything in the specs array.
|
||||
specs = [s if isinstance(s, Spec) else Spec(s) for s in specs]
|
||||
|
||||
# Get concrete specs for each matching version of these specs.
|
||||
version_specs = get_matching_versions(specs)
|
||||
for s in version_specs:
|
||||
s.concretize()
|
||||
|
||||
# Create a directory if none exists
|
||||
if not os.path.isdir(path):
|
||||
mkdirp(path)
|
||||
|
||||
# Things to keep track of while parsing specs.
|
||||
present = []
|
||||
mirrored = []
|
||||
error = []
|
||||
|
||||
# Iterate through packages and download all the safe tarballs for each of them
|
||||
for spec in version_specs:
|
||||
pkg = spec.package
|
||||
|
||||
stage = None
|
||||
try:
|
||||
# create a subdirectory for the current package@version
|
||||
realpath = os.path.realpath(path)
|
||||
subdir = join_path(realpath, pkg.name)
|
||||
mkdirp(subdir)
|
||||
|
||||
archive_file = mirror_archive_filename(spec)
|
||||
archive_path = join_path(subdir, archive_file)
|
||||
if os.path.exists(archive_path):
|
||||
present.append(spec)
|
||||
continue
|
||||
|
||||
# Set up a stage and a fetcher for the download
|
||||
fetcher = fs.for_package_version(pkg, pkg.version)
|
||||
stage = Stage(fetcher, name=fetcher.unique_name)
|
||||
fetcher.set_stage(stage)
|
||||
|
||||
# Do the fetch and checksum if necessary
|
||||
fetcher.fetch()
|
||||
if not kwargs.get('no_checksum', False):
|
||||
fetcher.check()
|
||||
tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
|
||||
|
||||
# Fetchers have to know how to archive their files. Use
|
||||
# that to move/copy/create an archive in the mirror.
|
||||
fetcher.archive(archive_path)
|
||||
tty.msg("Added %s to mirror" % archive_path)
|
||||
mirrored.append(spec)
|
||||
|
||||
except Exception, e:
|
||||
if spack.debug:
|
||||
sys.excepthook(*sys.exc_info())
|
||||
else:
|
||||
tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message)
|
||||
error.append(spec)
|
||||
|
||||
finally:
|
||||
if stage:
|
||||
stage.destroy()
|
||||
|
||||
return (present, mirrored, error)
|
||||
|
||||
|
||||
class MirrorError(spack.error.SpackError):
|
||||
"""Superclass of all mirror-creation related errors."""
|
||||
def __init__(self, msg, long_msg=None):
|
||||
super(MirrorError, self).__init__(msg, long_msg)
|
||||
@ -385,8 +385,8 @@ def ensure_has_dict(attr_name):
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
if not self.spec.concrete:
|
||||
raise ValueError("Can only get version of concrete package.")
|
||||
if not self.spec.versions.concrete:
|
||||
raise ValueError("Can only get of package with concrete version.")
|
||||
return self.spec.versions[0]
|
||||
|
||||
|
||||
@ -451,18 +451,20 @@ def stage(self):
|
||||
raise ValueError("Can only get a stage for a concrete package.")
|
||||
|
||||
if self._stage is None:
|
||||
self._stage = Stage(
|
||||
self.fetcher, mirror_path=self.mirror_path(), name=self.spec.short_spec)
|
||||
self._stage = Stage(self.fetcher,
|
||||
mirror_path=self.mirror_path(),
|
||||
name=self.spec.short_spec)
|
||||
return self._stage
|
||||
|
||||
|
||||
@property
|
||||
def fetcher(self):
|
||||
if not self.spec.concrete:
|
||||
raise ValueError("Can only get a fetcher for a concrete package.")
|
||||
if not self.spec.versions.concrete:
|
||||
raise ValueError(
|
||||
"Can only get a fetcher for a package with concrete versions.")
|
||||
|
||||
if not self._fetcher:
|
||||
self._fetcher = fs.from_args(self.versions[self.version], self)
|
||||
self._fetcher = fs.for_package_version(self, self.version)
|
||||
return self._fetcher
|
||||
|
||||
|
||||
@ -598,13 +600,14 @@ def url_version(self, version):
|
||||
|
||||
@property
|
||||
def default_url(self):
|
||||
if self.spec.version.concrete:
|
||||
if self.spec.versions.concrete:
|
||||
return self.url_for_version(self.version)
|
||||
else:
|
||||
url = getattr(self, 'url', None)
|
||||
if url:
|
||||
return url
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def remove_prefix(self):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user