mirror : deals correctly with variants that optionally enable resources (if they are archive URLs)

lib/spack/spack/mirror.py

@@ -26,7 +26,7 @@
 This file contains code for creating spack mirror directories.  A
 mirror is an organized hierarchy containing specially named archive
 files.  This enabled spack to know where to find files in a mirror if
-the main server for a particualr package is down.  Or, if the computer
+the main server for a particular package is down.  Or, if the computer
 where spack is run is not connected to the internet, it allows spack
 to download packages directly from a mirror (e.g., on an intranet).
 """
@@ -42,7 +42,7 @@
 from spack.spec import Spec
 from spack.stage import Stage
 from spack.version import *
-from spack.util.compression import extension
+from spack.util.compression import extension, allowed_archive
 
 
 def mirror_archive_filename(spec):
@@ -87,11 +87,26 @@ def get_matching_versions(specs, **kwargs):
             if v.satisfies(spec.versions):
                 s = Spec(pkg.name)
                 s.versions = VersionList([v])
+                s.variants = spec.variants.copy()
                 matching.append(s)
 
     return matching
 
 
+def suggest_archive_basename(resource):
+    """
+    Return a tentative basename for an archive. Raise an exception if the name is among the allowed archive types.
+
+    :param fetcher:
+    :return:
+    """
+    basename = os.path.basename(resource.fetcher.url)
+    if not allowed_archive(basename):
+        raise RuntimeError("%s is not an allowed archive tye" % basename)
+    return basename
+
+
+
 def create(path, specs, **kwargs):
     """Create a directory to be used as a spack mirror, and fill it with
        package archives.
@@ -108,7 +123,7 @@ def create(path, specs, **kwargs):
 
        Return Value:
          Returns a tuple of lists: (present, mirrored, error)
-         * present:  Package specs that were already prsent.
+         * present:  Package specs that were already present.
          * mirrored: Package specs that were successfully mirrored.
          * error:    Package specs that failed to mirror due to some error.
 
@@ -140,6 +155,7 @@ def create(path, specs, **kwargs):
     error    = []
 
     # Iterate through packages and download all the safe tarballs for each of them
+    everything_already_exists = True
     for spec in version_specs:
         pkg = spec.package
 
@@ -152,9 +168,8 @@ def create(path, specs, **kwargs):
 
             if os.path.exists(archive_path):
                 tty.msg("Already added %s" % spec.format("$_$@"))
-                present.append(spec)
+            else:
-                continue
+                everything_already_exists = False
-
                 # Set up a stage and a fetcher for the download
                 unique_fetch_name = spec.format("$_$@")
                 fetcher = fs.for_package_version(pkg, pkg.version)
@@ -171,6 +186,28 @@ def create(path, specs, **kwargs):
                 # that to move/copy/create an archive in the mirror.
                 fetcher.archive(archive_path)
                 tty.msg("Added %s." % spec.format("$_$@"))
+
+            # Fetch resources if they are associated with the spec
+            resources = pkg._get_resources()
+            for resource in resources:
+                resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
+                if os.path.exists(resource_archive_path):
+                    tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+                    continue
+                everything_already_exists = False
+                resource_stage_folder = pkg._resource_stage(resource)
+                resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
+                resource.fetcher.set_stage(resource_stage)
+                resource.fetcher.fetch()
+                if not kwargs.get('no_checksum', False):
+                    resource.fetcher.check()
+                    tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
+                resource.fetcher.archive(resource_archive_path)
+                tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+
+            if everything_already_exists:
+                present.append(spec)
+            else:
                 mirrored.append(spec)
 
         except Exception, e:

lib/spack/spack/package.py

@@ -644,12 +644,14 @@ def do_fetch(self):
         # Fetch resources
         resources = self._get_resources()
         for resource in resources:
-            pieces = ['resource', resource.name, self.spec.dag_hash()]
+            resource_stage_folder = self._resource_stage(resource)
-            resource_stage_folder = '-'.join(pieces)
+            # FIXME : works only for URLFetchStrategy
-            stage = Stage(resource.fetcher, name=resource_stage_folder)
+            resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
-            resource.fetcher.set_stage(stage)
+            resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
-            resource.fetcher.fetch()
+            resource.fetcher.set_stage(resource_stage)
-            resource.fetcher.check()
+            # Delegate to stage object to trigger mirror logic
+            resource_stage.fetch()
+            resource_stage.check()
         ##########
 
         self._fetch_time = time.time() - start_time
@@ -766,6 +768,11 @@ def _get_resources(self):
                 resources.extend(resource_list)
         return resources
 
+    def _resource_stage(self, resource):
+        pieces = ['resource', resource.name, self.spec.dag_hash()]
+        resource_stage_folder = '-'.join(pieces)
+        return resource_stage_folder
+
     def _build_logger(self, log_path):
         """Create a context manager to log build output."""