Add dynamic search option for package sources (#2270)

Package provides a 'list_url' attribute which may be searched to find
download links. #1822 created a slowdown for all tests by always
searching this URL. This reenables dynamic search only in cases where
all other fetchers fail. This also only enables dynamic search when
'mirror_only' is set to false.

lib/spack/spack/fetch_strategy.py

@@ -911,7 +911,10 @@ def from_list_url(pkg):
             versions = pkg.fetch_remote_versions()
             try:
                 url_from_list = versions[pkg.version]
-                return URLFetchStrategy(url=url_from_list, digest=None)
+                digest = None
+                if pkg.version in pkg.versions:
+                    digest = pkg.versions[pkg.version].get('md5', None)
+                return URLFetchStrategy(url=url_from_list, digest=digest)
             except KeyError:
                 tty.msg("Can not find version %s in url_list" %
                         self.version)

lib/spack/spack/package.py

@@ -703,7 +703,13 @@ def _make_root_stage(self, fetcher):
         # Construct a path where the stage should build..
         s = self.spec
         stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
-        stage = Stage(fetcher, mirror_path=mp, name=stage_name, path=self.path)
+
+        def download_search():
+            dynamic_fetcher = fs.from_list_url(self)
+            return [dynamic_fetcher] if dynamic_fetcher else []
+
+        stage = Stage(fetcher, mirror_path=mp, name=stage_name, path=self.path,
+                      search_fn=download_search)
         return stage
 
     def _make_stage(self):

lib/spack/spack/stage.py

@@ -162,7 +162,7 @@ class Stage(object):
     def __init__(
             self, url_or_fetch_strategy,
             name=None, mirror_path=None, keep=False, path=None, lock=True,
-            alternate_fetchers=None):
+            search_fn=None):
         """Create a stage object.
            Parameters:
              url_or_fetch_strategy
@@ -198,7 +198,7 @@ def __init__(
         self.fetcher.set_stage(self)
         # self.fetcher can change with mirrors.
         self.default_fetcher = self.fetcher
-        self.alternate_fetchers = alternate_fetchers
+        self.search_fn = search_fn
         # used for mirrored archives of repositories.
         self.skip_checksum_for_mirror = True
 
@@ -416,10 +416,17 @@ def fetch(self, mirror_only=False):
                         self.mirror_path, digest, expand=expand,
                         extension=extension))
 
-        if self.alternate_fetchers:
+        def generate_fetchers():
-            fetchers.extend(self.alternate_fetchers)
-
             for fetcher in fetchers:
+                yield fetcher
+            # The search function may be expensive, so wait until now to
+            # call it so the user can stop if a prior fetcher succeeded
+            if self.search_fn and not mirror_only:
+                dynamic_fetchers = self.search_fn()
+                for fetcher in dynamic_fetchers:
+                    yield fetcher
+
+        for fetcher in generate_fetchers():
             try:
                 fetcher.set_stage(self)
                 self.fetcher = fetcher

lib/spack/spack/test/stage.py

@@ -59,6 +59,26 @@ def use_tmp(use_tmp):
     yield
 
 
+def fail_search_fn():
+    raise Exception("This should not have been called")
+
+
+class FailingFetchStrategy(spack.fetch_strategy.FetchStrategy):
+    def fetch(self):
+        raise spack.fetch_strategy.FailedDownloadError(
+            "<non-existent URL>",
+            "This implementation of FetchStrategy always fails")
+
+
+class MockSearchFunction(object):
+    def __init__(self):
+        self.performed_search = False
+
+    def __call__(self):
+        self.performed_search = True
+        return []
+
+
 class StageTest(MockPackagesTest):
 
     def setUp(self):
@@ -251,6 +271,32 @@ def test_fetch(self):
             self.check_fetch(stage, self.stage_name)
         self.check_destroy(stage, self.stage_name)
 
+    def test_no_search_if_default_succeeds(self):
+        with Stage(self.archive_url, name=self.stage_name,
+                   search_fn=fail_search_fn) as stage:
+            stage.fetch()
+        self.check_destroy(stage, self.stage_name)
+
+    def test_no_search_mirror_only(self):
+        with Stage(FailingFetchStrategy(), name=self.stage_name,
+                   search_fn=fail_search_fn) as stage:
+            try:
+                stage.fetch(mirror_only=True)
+            except spack.fetch_strategy.FetchError:
+                pass
+        self.check_destroy(stage, self.stage_name)
+
+    def test_search_if_default_fails(self):
+        test_search = MockSearchFunction()
+        with Stage(FailingFetchStrategy(), name=self.stage_name,
+                   search_fn=test_search) as stage:
+            try:
+                stage.fetch(mirror_only=False)
+            except spack.fetch_strategy.FetchError:
+                pass
+        self.check_destroy(stage, self.stage_name)
+        self.assertTrue(test_search.performed_search)
+
     def test_expand_archive(self):
         with Stage(self.archive_url, name=self.stage_name) as stage:
             stage.fetch()