zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mirror : deals correctly with variants that optionally enable resources (if they are archive URLs)

Browse Source
This commit is contained in:
Massimiliano Culpo 2015-12-02 16:18:25 +01:00
parent 39a3cfd4d9
commit 50bd4d2e4e
2 changed files with 72 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
lib/spack/spack/mirror.py
View File

@ -26,7 +26,7 @@
This file contains code for creating spack mirror directories. A
mirror is an organized hierarchy containing specially named archive
files. This enabled spack to know where to find files in a mirror if
the main server for a particualr package is down. Or, if the computer
the main server for a particular package is down. Or, if the computer
where spack is run is not connected to the internet, it allows spack
to download packages directly from a mirror (e.g., on an intranet).
"""
@ -42,7 +42,7 @@
from spack.spec import Spec
from spack.stage import Stage
from spack.version import *
from spack.util.compression import extension
from spack.util.compression import extension, allowed_archive
def mirror_archive_filename(spec):
@ -87,11 +87,26 @@ def get_matching_versions(specs, **kwargs):
if v.satisfies(spec.versions):
s = Spec(pkg.name)
s.versions = VersionList([v])
s.variants = spec.variants.copy()
matching.append(s)
return matching
def suggest_archive_basename(resource):
"""
Return a tentative basename for an archive. Raise an exception if the name is among the allowed archive types.
:param fetcher:
:return:
"""
basename = os.path.basename(resource.fetcher.url)
if not allowed_archive(basename):
raise RuntimeError("%s is not an allowed archive tye" % basename)
return basename
def create(path, specs, **kwargs):
"""Create a directory to be used as a spack mirror, and fill it with
package archives.
@ -108,7 +123,7 @@ def create(path, specs, **kwargs):
Return Value:
Returns a tuple of lists: (present, mirrored, error)
* present: Package specs that were already prsent.
* present: Package specs that were already present.
* mirrored: Package specs that were successfully mirrored.
* error: Package specs that failed to mirror due to some error.
@ -140,6 +155,7 @@ def create(path, specs, **kwargs):
error = []
# Iterate through packages and download all the safe tarballs for each of them
everything_already_exists = True
for spec in version_specs:
pkg = spec.package
@ -152,26 +168,47 @@ def create(path, specs, **kwargs):
if os.path.exists(archive_path):
tty.msg("Already added %s" % spec.format("$_$@"))
else:
everything_already_exists = False
# Set up a stage and a fetcher for the download
unique_fetch_name = spec.format("$_$@")
fetcher = fs.for_package_version(pkg, pkg.version)
stage = Stage(fetcher, name=unique_fetch_name)
fetcher.set_stage(stage)
# Do the fetch and checksum if necessary
fetcher.fetch()
if not kwargs.get('no_checksum', False):
fetcher.check()
tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
# Fetchers have to know how to archive their files. Use
# that to move/copy/create an archive in the mirror.
fetcher.archive(archive_path)
tty.msg("Added %s." % spec.format("$_$@"))
# Fetch resources if they are associated with the spec
resources = pkg._get_resources()
for resource in resources:
resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
if os.path.exists(resource_archive_path):
tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
continue
everything_already_exists = False
resource_stage_folder = pkg._resource_stage(resource)
resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
resource.fetcher.set_stage(resource_stage)
resource.fetcher.fetch()
if not kwargs.get('no_checksum', False):
resource.fetcher.check()
tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
resource.fetcher.archive(resource_archive_path)
tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
if everything_already_exists:
present.append(spec)
continue
# Set up a stage and a fetcher for the download
unique_fetch_name = spec.format("$_$@")
fetcher = fs.for_package_version(pkg, pkg.version)
stage = Stage(fetcher, name=unique_fetch_name)
fetcher.set_stage(stage)
# Do the fetch and checksum if necessary
fetcher.fetch()
if not kwargs.get('no_checksum', False):
fetcher.check()
tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
# Fetchers have to know how to archive their files. Use
# that to move/copy/create an archive in the mirror.
fetcher.archive(archive_path)
tty.msg("Added %s." % spec.format("$_$@"))
mirrored.append(spec)
else:
mirrored.append(spec)
except Exception, e:
if spack.debug:

19
lib/spack/spack/package.py
View File

@ -644,12 +644,14 @@ def do_fetch(self):
# Fetch resources
resources = self._get_resources()
for resource in resources:
pieces = ['resource', resource.name, self.spec.dag_hash()]
resource_stage_folder = '-'.join(pieces)
stage = Stage(resource.fetcher, name=resource_stage_folder)
resource.fetcher.set_stage(stage)
resource.fetcher.fetch()
resource.fetcher.check()
resource_stage_folder = self._resource_stage(resource)
# FIXME : works only for URLFetchStrategy
resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
resource.fetcher.set_stage(resource_stage)
# Delegate to stage object to trigger mirror logic
resource_stage.fetch()
resource_stage.check()
##########
self._fetch_time = time.time() - start_time
@ -766,6 +768,11 @@ def _get_resources(self):
resources.extend(resource_list)
return resources
def _resource_stage(self, resource):
pieces = ['resource', resource.name, self.spec.dag_hash()]
resource_stage_folder = '-'.join(pieces)
return resource_stage_folder
def _build_logger(self, log_path):
"""Create a context manager to log build output."""