Merge branch 'fixes/md5' of git://github.com/epfl-scitas/spack into epfl-scitas-fixes/md5

lib/spack/spack/cmd/checksum.py

@@ -22,23 +22,18 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import os
-import re
 import argparse
 import hashlib
-from pprint import pprint
-from subprocess import CalledProcessError
 
 import llnl.util.tty as tty
-from llnl.util.tty.colify import colify
-
 import spack
 import spack.cmd
 import spack.util.crypto
 from spack.stage import Stage, FailedDownloadError
 from spack.version import *
 
-description ="Checksum available versions of a package."
+description = "Checksum available versions of a package."
+
 
 def setup_parser(subparser):
     subparser.add_argument(
@@ -60,30 +55,24 @@ def get_checksums(versions, urls, **kwargs):
     hashes = []
     i = 0
     for url, version in zip(urls, versions):
-        stage = Stage(url)
         try:
-            stage.fetch()
-            if i == 0 and first_stage_function:
-                first_stage_function(stage)
+            with Stage(url) as stage:
+                stage.delete_on_exit = not keep_stage
+                stage.fetch()
+                if i == 0 and first_stage_function:
+                    first_stage_function(stage)
 
-            hashes.append((version,
-                spack.util.crypto.checksum(hashlib.md5, stage.archive_file)))
-        except FailedDownloadError, e:
+                hashes.append((version,
+                               spack.util.crypto.checksum(hashlib.md5, stage.archive_file)))
+                i += 1
+        except FailedDownloadError as e:
             tty.msg("Failed to fetch %s" % url)
-            continue
-	except Exception, e:
+        except Exception as e:
             tty.msg('Something failed on %s, skipping.\n    (%s)' % (url, e))
-            continue
-
-        finally:
-            if not keep_stage:
-                stage.destroy()
-        i += 1
 
     return hashes
 
 
-
 def checksum(parser, args):
     # get the package we're going to generate checksums for
     pkg = spack.repo.get(args.package)
@@ -106,8 +95,8 @@ def checksum(parser, args):
 
     tty.msg("Found %s versions of %s" % (len(versions), pkg.name),
             *spack.cmd.elide_list(
-            ["%-10s%s" % (v, versions[v]) for v in sorted_versions]))
-    print
+                ["%-10s%s" % (v, versions[v]) for v in sorted_versions]))
+    print()
     archives_to_fetch = tty.get_number(
         "How many would you like to checksum?", default=5, abort='q')
 

lib/spack/spack/cmd/md5.py

@@ -22,51 +22,51 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import os
 import argparse
 import hashlib
-
-from contextlib import contextmanager
+import os
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import *
-
 import spack.util.crypto
 from spack.stage import Stage, FailedDownloadError
 
 description = "Calculate md5 checksums for files/urls."
 
-@contextmanager
-def stager(url):
-    _cwd = os.getcwd()
-    _stager = Stage(url)
-    try:
-        _stager.fetch()
-        yield _stager
-    except FailedDownloadError:
-        tty.msg("Failed to fetch %s" % url)
-    finally:
-        _stager.destroy()
-        os.chdir(_cwd) # the Stage class changes the current working dir so it has to be restored
 
 def setup_parser(subparser):
     setup_parser.parser = subparser
     subparser.add_argument('files', nargs=argparse.REMAINDER,
                            help="Files to checksum.")
 
+
+def compute_md5_checksum(url):
+    if not os.path.isfile(url):
+        with Stage(url) as stage:
+            stage.fetch()
+            value = spack.util.crypto.checksum(hashlib.md5, stage.archive_file)
+    else:
+        value = spack.util.crypto.checksum(hashlib.md5, url)
+    return value
+
+
 def md5(parser, args):
     if not args.files:
         setup_parser.parser.print_help()
         return 1
 
-    for f in args.files:
-        if not os.path.isfile(f):
-            with stager(f) as stage:
-                checksum = spack.util.crypto.checksum(hashlib.md5, stage.archive_file)
-                print "%s  %s" % (checksum, f)
-        else:
-            if not can_access(f):
-                tty.die("Cannot read file: %s" % f)
+    results = []
+    for url in args.files:
+        try:
+            checksum = compute_md5_checksum(url)
+            results.append((checksum, url))
+        except FailedDownloadError as e:
+            tty.warn("Failed to fetch %s" % url)
+            tty.warn("%s" % e)
+        except IOError as e:
+            tty.warn("Error when reading %s" % url)
+            tty.warn("%s" % e)
 
-            checksum = spack.util.crypto.checksum(hashlib.md5, f)
-            print "%s  %s" % (checksum, f)
+    # Dump the MD5s at last without interleaving them with downloads
+    tty.msg("Number of MD5 check-sums computed: %s " % len(results))
+    for checksum, url in results:
+        tty.msg("%s  %s" % (checksum, url))

lib/spack/spack/cmd/stage.py

@@ -50,4 +50,6 @@ def stage(parser, args):
     specs = spack.cmd.parse_specs(args.specs, concretize=True)
     for spec in specs:
         package = spack.repo.get(spec)
-        package.do_stage()
+        with package.stage as stage:
+            stage.delete_on_exit = False
+            package.do_stage()

lib/spack/spack/stage.py

@@ -441,6 +441,7 @@ def __enter__(self):
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         for item in reversed(self):
+            item.delete_on_exit = getattr(self, 'delete_on_exit', True)
             item.__exit__(exc_type, exc_val, exc_tb)
 
     #