checksums: enforce that all mainline packages use sha256 checksums

- Add a test that verifies checksums on all packages

- Also add an attribute to packages that indicates whether they need a
  manual download or not, and add an exception in the tests for these
  packages until we can verify them.

lib/spack/spack/package.py

@@ -465,10 +465,13 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)):
     #: _spack_build_envfile.
     archive_files = []
 
+    #: Boolean. Set to ``True`` for packages that require a manual download.
+    #: This is currently only used by package sanity tests.
+    manual_download = False
+
     #
     # Set default licensing information
     #
-
     #: Boolean. If set to ``True``, this software requires a license.
     #: If set to ``False``, all of the ``license_*`` attributes will
     #: be ignored. Defaults to ``False``.

lib/spack/spack/test/package_sanity.py

@@ -8,9 +8,10 @@
 
 import pytest
 
+import spack.fetch_strategy
 import spack.paths
 import spack.repo
-import spack.fetch_strategy
+import spack.util.crypto as crypto
 
 
 def check_repo():
@@ -94,3 +95,42 @@ def test_docstring():
     for name in spack.repo.all_package_names():
         pkg = spack.repo.get(name)
         assert pkg.__doc__
+
+
+def test_all_packages_use_sha256_checksums():
+    """Make sure that no packages use md5 checksums."""
+
+    errors = []
+    for name in spack.repo.all_package_names():
+        pkg = spack.repo.path.get(name)
+
+        # for now, don't enforce on packages that require manual downloads
+        # TODO: eventually fix these, too.
+        if pkg.manual_download:
+            continue
+
+        def invalid_sha256_digest(fetcher):
+            if getattr(fetcher, "digest", None):
+                h = crypto.hash_algo_for_digest(fetcher.digest)
+                if h != "sha256":
+                    return h
+
+        for v, args in pkg.versions.items():
+            fetcher = spack.fetch_strategy.for_package_version(pkg, v)
+            bad_digest = invalid_sha256_digest(fetcher)
+            if bad_digest:
+                errors.append(
+                    "All packages must use sha256 checksums. %s@%s uses %s." %
+                    (name, v, bad_digest)
+                )
+
+        for _, resources in pkg.resources.items():
+            for resource in resources:
+                bad_digest = invalid_sha256_digest(resource.fetcher)
+                if bad_digest:
+                    errors.append(
+                        "All packages must use sha256 checksums."
+                        "Resource in %s uses %s." % (name, v, bad_digest)
+                    )
+
+    assert [] == errors

var/spack/repos/builtin/packages/fsl/package.py

@@ -20,6 +20,7 @@ class Fsl(Package):
 
     homepage = "https://fsl.fmrib.ox.ac.uk"
     url      = "file://{0}/fsl-5.0.10-sources.tar.gz".format(os.getcwd())
+    manual_download = True
 
     version('5.0.10', '64823172a08aad679833240ba64c8e30')
 

var/spack/repos/builtin/packages/gams/package.py

@@ -13,6 +13,8 @@ class Gams(Package):
     linear, nonlinear, and mixed-integer optimization problems."""
 
     homepage = "https://www.gams.com/"
+    manual_download = True
+
     version('27.2', '4f3f3484a4389661e0522a4cfe0289fd', expand=False)
 
     def url_for_version(self, version):

var/spack/repos/builtin/packages/gapfiller/package.py

@@ -19,6 +19,7 @@ class Gapfiller(Package):
        http://spack.readthedocs.io/en/latest/mirrors.html"""
 
     homepage = "https://www.baseclear.com/genomics/bioinformatics/basetools/gapfiller"
+    manual_download = True
 
     version('1.10', '54d5e2ada131a1305a66e41c0d380382')
 

var/spack/repos/builtin/packages/gaussian/package.py

@@ -12,6 +12,7 @@ class Gaussian(Package):
 
     homepage = "http://www.gaussian.com/"
     url = "file://{0}/g09.tgz".format(os.getcwd())
+    manual_download = True
 
     version('09', '7d4c95b535e68e48af183920df427e4e')
 

var/spack/repos/builtin/packages/gurobi/package.py

@@ -23,6 +23,7 @@ class Gurobi(Package):
     $GUROBI_HOME/docs/quickstart_linux.pdf for more details."""
 
     homepage = "http://www.gurobi.com/index"
+    manual_download = True
 
     version('7.5.2', '01f6dbb8d165838cca1664a1a14e4a85')
 

var/spack/repos/builtin/packages/matlab/package.py

@@ -23,6 +23,8 @@ class Matlab(Package):
     mirror, see http://spack.readthedocs.io/en/latest/mirrors.html"""
 
     homepage = "https://www.mathworks.com/products/matlab.html"
+    manual_download = True
+
     version('R2018b', sha256='8cfcddd3878d3a69371c4e838773bcabf12aaf0362cc2e1ae7e8820845635cac')
     version('R2016b', 'b0e0b688894282139fa787b5a86a5cf7')
 

var/spack/repos/builtin/packages/molcas/package.py

@@ -15,6 +15,7 @@ class Molcas(CMakePackage):
 
     homepage = "http://www.molcas.org/"
     url = "file://{0}/molcas8.2.tar.gz".format(os.getcwd())
+    manual_download = True
 
     version('8.2', '25b5fb8e1338b458a3eaea0b3d3b5e58')
 

var/spack/repos/builtin/packages/namd/package.py

@@ -15,6 +15,7 @@ class Namd(MakefilePackage):
 
     homepage = "http://www.ks.uiuc.edu/Research/namd/"
     url      = "file://{0}/NAMD_2.12_Source.tar.gz".format(os.getcwd())
+    manual_download = True
 
     version('2.13', '9e3323ed856e36e34d5c17a7b0341e38')
     version('2.12', '2a1191909b1ab03bf0205971ad4d8ee9')

var/spack/repos/builtin/packages/qbank/package.py

@@ -22,6 +22,7 @@ class Qbank(Package):
     # by Scott Jackson <Scott.Jackson@pnl.gov>
     homepage = "http://www.pnnl.gov/"
     url      = "file://{0}/qbank-2.10.4.tar.gz".format(os.getcwd())
+    manual_download = True
 
     version('2.10.4', '0820587353e63d32ddb49689dd4289e7')
 

var/spack/repos/builtin/packages/signalp/package.py

@@ -19,6 +19,7 @@ class Signalp(Package):
 
     homepage = "http://www.cbs.dtu.dk/services/SignalP/"
     url      = "file://{0}/signalp-4.1f.Linux.tar.gz".format(os.getcwd())
+    manual_download = True
 
     version('4.1f', 'a9aeb66259202649c959846f3f4d9744')
 

var/spack/repos/builtin/packages/sspace-longread/package.py

@@ -18,6 +18,7 @@ class SspaceLongread(Package):
        http://spack.readthedocs.io/en/latest/mirrors.html"""
 
     homepage = "https://www.baseclear.com/genomics/bioinformatics/basetools/SSPACE-longread"
+    manual_download = True
 
     version('1.1', '0bb5d8603d7ead4ff1596135a520cc26')
 

var/spack/repos/builtin/packages/sspace-standard/package.py

@@ -20,6 +20,7 @@ class SspaceStandard(Package):
 
     homepage = "https://www.baseclear.com/genomics/bioinformatics/basetools/SSPACE"
     url      = "file://{0}/41SSPACE-STANDARD-3.0_linux-x86_64.tar.gz".format(os.getcwd())
+    manual_download = True
 
     version('3.0', '7e171b4861b9d514e80aafc3d9cdf554')
 

var/spack/repos/builtin/packages/star-ccm-plus/package.py

@@ -13,6 +13,7 @@ class StarCcmPlus(Package):
 
     homepage = "http://mdx.plm.automation.siemens.com/star-ccm-plus"
     url      = "file://{0}/STAR-CCM+11.06.010_02_linux-x86_64.tar.gz".format(os.getcwd())
+    manual_download = True
 
     version('11.06.010_02', 'd349c6ac8293d8e6e7a53533d695588f')
 

var/spack/repos/builtin/packages/supernova/package.py

@@ -26,6 +26,8 @@ class Supernova(Package):
     http://spack.readthedocs.io/en/latest/mirrors.html"""
 
     homepage = "https://support.10xgenomics.com/de-novo-assembly/software/overview/latest/welcome"
+    manual_download = True
+
     version('2.1.1', sha256='2f58eb66951e257b89359134ab8e35ad638c4ed51cb3fb8121625dfcc7761938')
     version('2.0.1', '3697ce043c798fcb672fe0a66c56d6f0')
 

var/spack/repos/builtin/packages/targetp/package.py

@@ -18,6 +18,7 @@ class Targetp(Package):
 
     homepage = "http://www.cbs.dtu.dk/services/TargetP/"
     url      = "file://{0}/targetp-1.1b.Linux.tar.gz".format(os.getcwd())
+    manual_download = True
 
     version('1.1b', '80233d0056e11abfd22a4ce73d1808c6')
 

var/spack/repos/builtin/packages/tecplot/package.py

@@ -14,6 +14,8 @@ class Tecplot(Package):
     by post-processing charge density data."""
 
     homepage = "https://www.tecplot.com/"
+    manual_download = True
+
     version('2017r1', '06a8057d33a519607720d4c621cd3f50', expand=False)
     version('2018r2', 'd3cf54a7555e0259b7ba0d82fef23bc3', expand=False)
 

var/spack/repos/builtin/packages/tmhmm/package.py

@@ -18,6 +18,7 @@ class Tmhmm(Package):
 
     homepage = "http://www.cbs.dtu.dk/cgi-bin/nph-sw_request?tmhmm"
     url      = "file://{0}/tmhmm-2.0c.Linux.tar.gz".format(os.getcwd())
+    manual_download = True
 
     version('2.0c', '359db0c4ecf84d1ade5786abe844d54e')
 

var/spack/repos/builtin/packages/trf/package.py

@@ -18,6 +18,7 @@ class Trf(Package):
        http://spack.readthedocs.io/en/latest/mirrors.html"""
 
     homepage = "https://tandem.bu.edu/trf/trf.html"
+    manual_download = True
 
     version('4.09', '0c594fe666e0332db1df9d160d7fabc8', expand=False,
             url='file://{0}/trf409.linux64'.format(os.getcwd()))

var/spack/repos/builtin/packages/turbomole/package.py

@@ -20,6 +20,7 @@ class Turbomole(Package):
     see http://spack.readthedocs.io/en/latest/mirrors.html"""
 
     homepage = "http://www.turbomole-gmbh.com/"
+    manual_download = True
 
     version('7.0.2', '92b97e1e52e8dcf02a4d9ac0147c09d6',
             url="file://%s/turbolinux702.tar.gz" % os.getcwd())

var/spack/repos/builtin/packages/usearch/package.py

@@ -18,6 +18,7 @@ class Usearch(Package):
        http://spack.readthedocs.io/en/latest/mirrors.html"""
 
     homepage = "http://www.drive5.com/usearch/"
+    manual_download = True
 
     version('10.0.240', '05192b6d5e291530c190a19a3cc82b53', expand=False)
 

var/spack/repos/builtin/packages/vizglow/package.py

@@ -19,6 +19,7 @@ class Vizglow(Package):
     set up a mirror, see http://spack.readthedocs.io/en/latest/mirrors.html"""
 
     homepage = "http://esgeetech.com/products/vizglow-plasma-modeling/"
+    manual_download = True
 
     version('2.2alpha20', '2bef890c66f3a44aaf96f7c96788c89e', expand=False,
             url="file://{0}/VizGlow_v2.2alpha20-Linux-x86_64-R09December2016-Install".format(os.getcwd()))

var/spack/repos/builtin/packages/xplor-nih/package.py

@@ -17,6 +17,7 @@ class XplorNih(Package):
        http://spack.readthedocs.io/en/latest/mirrors.html"""
 
     homepage = "https://nmr.cit.nih.gov/xplor-nih/"
+    manual_download = True
 
     version('2.45', 'ab3e046604beb0effc89a1adb7bab438')