Allow more fine-grained control over what submodules are updated (#27293)

The "submodules" argument of the "version" directive can now accept
a callable that returns a list of submodules, in addition to the usual
Boolean values

lib/spack/docs/packaging_guide.rst

@@ -1070,13 +1070,32 @@ Commits
 
 Submodules
   You can supply ``submodules=True`` to cause Spack to fetch submodules
-  recursively along with the repository at fetch time. For more information
+  recursively along with the repository at fetch time.
-  about git submodules see the manpage of git: ``man git-submodule``.
 
   .. code-block:: python
 
      version('1.0.1', tag='v1.0.1', submodules=True)
 
+  If a package has needs more fine-grained control over submodules, define
+  ``submodules`` to be a callable function that takes the package instance as
+  its only argument.  The function should return a list of submodules to be fetched.
+
+  .. code-block:: python
+
+     def submodules(package):
+         submodules = []
+         if "+variant-1" in package.spec:
+             submodules.append("submodule_for_variant_1")
+         if "+variant-2" in package.spec:
+             submodules.append("submodule_for_variant_2")
+         return submodules
+
+
+      class MyPackage(Package):
+          version("0.1.0", submodules=submodules)
+
+  For more information about git submodules see the manpage of git: ``man
+  git-submodule``.
 
 .. _github-fetch:
 

lib/spack/spack/fetch_strategy.py

@@ -120,6 +120,11 @@ def __init__(self, **kwargs):
         # 'no_cache' option from version directive.
         self.cache_enabled = not kwargs.pop('no_cache', False)
 
+        self.package = None
+
+    def set_package(self, package):
+        self.package = package
+
     # Subclasses need to implement these methods
     def fetch(self):
         """Fetch source code archive or repo.
@@ -243,6 +248,10 @@ def source_id(self):
         if all(component_ids):
             return component_ids
 
+    def set_package(self, package):
+        for item in self:
+            item.package = package
+
 
 @fetcher
 class URLFetchStrategy(FetchStrategy):
@@ -976,9 +985,20 @@ def clone(self, dest=None, commit=None, branch=None, tag=None, bare=False):
                     git(*args)
 
         # Init submodules if the user asked for them.
-        if self.submodules:
+        git_commands = []
+        submodules = self.submodules
+        if callable(submodules):
+            submodules = list(submodules(self.package))
+            git_commands.append(["submodule", "init", "--"] + submodules)
+            git_commands.append(['submodule', 'update', '--recursive'])
+        elif submodules:
+            git_commands.append(["submodule", "update", "--init", "--recursive"])
+
+        if not git_commands:
+            return
+
         with working_dir(dest):
-                args = ['submodule', 'update', '--init', '--recursive']
+            for args in git_commands:
                 if not spack.config.get('config:debug'):
                     args.insert(1, '--quiet')
                 git(*args)

lib/spack/spack/package_base.py

@@ -1312,6 +1312,7 @@ def _make_fetcher(self):
         resources = self._get_needed_resources()
         for resource in resources:
             fetcher.append(resource.fetcher)
+        fetcher.set_package(self)
         return fetcher
 
     @property
@@ -1326,6 +1327,7 @@ def fetcher(self):
     @fetcher.setter
     def fetcher(self, f):
         self._fetcher = f
+        self._fetcher.set_package(self)
 
     def dependencies_of_type(self, *deptypes):
         """Get dependencies that can possibly have these deptypes.

lib/spack/spack/test/git_fetch.py

@@ -326,6 +326,37 @@ def test_gitsubmodule(submodules, mock_git_repository, config,
                 assert not os.path.isfile(file_path)
 
 
+@pytest.mark.disable_clean_stage_check
+def test_gitsubmodules_callable(
+        mock_git_repository, config, mutable_mock_repo, monkeypatch
+):
+    """
+    Test GitFetchStrategy behavior with submodules selected after concretization
+    """
+    def submodules_callback(package):
+        name = 'third_party/submodule0'
+        return [name]
+
+    type_of_test = 'tag-branch'
+    t = mock_git_repository.checks[type_of_test]
+
+    # Construct the package under test
+    spec = Spec('git-test')
+    spec.concretize()
+    pkg = spack.repo.get(spec)
+    args = copy.copy(t.args)
+    args['submodules'] = submodules_callback
+    monkeypatch.setitem(pkg.versions, ver('git'), args)
+    pkg.do_stage()
+    with working_dir(pkg.stage.source_path):
+        file_path = os.path.join(pkg.stage.source_path,
+                                 'third_party/submodule0/r0_file_0')
+        assert os.path.isfile(file_path)
+        file_path = os.path.join(pkg.stage.source_path,
+                                 'third_party/submodule1/r0_file_1')
+        assert not os.path.isfile(file_path)
+
+
 @pytest.mark.disable_clean_stage_check
 def test_gitsubmodules_delete(
         mock_git_repository, config, mutable_mock_repo, monkeypatch