ASP-based solver: allow configuring target selection (#29835)

* ASP-based solver: allow configuring target selection

This commit adds a new "concretizer:targets" configuration
section, and two options under it.

 - "concretizer:targets:granularity" allows switching from
considering only generic targets to consider all possible
microarchitectures.

 - "concretizer:targets:host_compatible" instead controls
whether we can concretize for microarchitectures that
are incompatible with the current host.

* Add documentation

* Add unit-tests

etc/spack/defaults/concretizer.yaml

@@ -15,3 +15,16 @@ concretizer:
   # as possible, rather than building. If `false`, we'll always give you a fresh
   # concretization.
   reuse: false
+  # Options that tune which targets are considered for concretization. The
+  # concretization process is very sensitive to the number targets, and the time
+  # needed to reach a solution increases noticeably with the number of targets
+  # considered.
+  targets:
+    # Determine whether we want to target specific or generic microarchitectures.
+    # An example of the first kind might be for instance "skylake" or "bulldozer",
+    # while generic microarchitectures are for instance "aarch64" or "x86_64_v4".
+    granularity: microarchitectures
+    # If "false" allow targets that are incompatible with the current host (for
+    # instance concretize with target "icelake" while running on "haswell").
+    # If "true" only allow targets that are compatible with the host.
+    host_compatible: false

lib/spack/docs/build_settings.rst

@@ -219,26 +219,29 @@ Concretizer options
 but you can also use ``concretizer.yaml`` to customize aspects of the
 algorithm it uses to select the dependencies you install:
 
-.. _code-block: yaml
+.. literalinclude:: _spack_root/etc/spack/defaults/concretizer.yaml
+   :language: yaml
 
-   concretizer:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-     # Whether to consider installed packages or packages from buildcaches when
+Reuse already installed packages
-     # concretizing specs. If `true`, we'll try to use as many installs/binaries
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-     # as possible, rather than building. If `false`, we'll always give you a fresh
-     # concretization.
-     reuse: false
 
-^^^^^^^^^^^^^^^^
+The ``reuse`` attribute controls whether Spack will prefer to use installed packages (``true``), or
-``reuse``
-^^^^^^^^^^^^^^^^
-
-This controls whether Spack will prefer to use installed packages (``true``), or
 whether it will do a "fresh" installation and prefer the latest settings from
-``package.py`` files and ``packages.yaml`` (``false``). .
+``package.py`` files and ``packages.yaml`` (``false``).
+You can use:
 
-You can use ``spack install --reuse`` to enable reuse for a single installation,
+.. code-block:: console
-and you can use ``spack install --fresh`` to do a fresh install if ``reuse`` is
+
-enabled by default.
+   % spack install --reuse <spec>
+
+to enable reuse for a single installation, and you can use:
+
+.. code-block:: console
+
+   spack install --fresh <spec>
+
+to do a fresh install if ``reuse`` is enabled by default.
 
 .. note::
 
@@ -246,6 +249,40 @@ enabled by default.
    in the next Spack release. You will still be able to use ``spack install --fresh``
    to get the old behavior.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Selection of the target microarchitectures
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The options under the ``targets`` attribute control which targets are considered during a solve.
+Currently the options in this section are only configurable from the ``concretization.yaml`` file
+and there are no corresponding command line arguments to enable them for a single solve.
+
+The ``granularity`` option can take two possible values: ``microarchitectures`` and ``generic``.
+If set to:
+
+.. code-block:: yaml
+
+   concretizer:
+     targets:
+       granularity: microarchitectures
+
+Spack will consider all the microarchitectures known to ``archspec`` to label nodes for
+compatibility. If instead the option is set to:
+
+.. code-block:: yaml
+
+   concretizer:
+     targets:
+       granularity: generic
+
+Spack will consider only generic microarchitectures. For instance, when running on an
+Haswell node, Spack will consider ``haswell`` as the best target in the former case and
+``x86_64_v3`` as the best target in the latter case.
+
+The ``host_compatible`` option is a Boolean option that determines whether or not the
+microarchitectures considered during the solve are constrained to be compatible with the
+host Spack is currently running on. For instance, if this option is set to ``true``, a
+user cannot concretize for ``target=icelake`` while running on an Haswell node.
 
 .. _package-preferences:
 

lib/spack/spack/schema/concretizer.py

@@ -15,6 +15,16 @@
         'additionalProperties': False,
         'properties': {
             'reuse': {'type': 'boolean'},
+            'targets': {
+                'type': 'object',
+                'properties': {
+                    'host_compatible': {'type': 'boolean'},
+                    'granularity': {
+                        'type': 'string',
+                        'enum': ['generic', 'microarchitectures']
+                    }
+                }
+            },
         }
     }
 }

lib/spack/spack/solver/asp.py

@@ -1413,23 +1413,48 @@ def target_defaults(self, specs):
 
         self.gen.h2('Target compatibility')
 
-        compatible_targets = [uarch] + uarch.ancestors
+        # Construct the list of targets which are compatible with the host
-        additional_targets_in_family = sorted([
+        candidate_targets = [uarch] + uarch.ancestors
-            t for t in archspec.cpu.TARGETS.values()
+
-            if (t.family.name == uarch.family.name and
+        # Get configuration options
-                t not in compatible_targets)
+        granularity = spack.config.get('concretizer:targets:granularity')
-        ], key=lambda x: len(x.ancestors), reverse=True)
+        host_compatible = spack.config.get('concretizer:targets:host_compatible')
-        compatible_targets += additional_targets_in_family
+
+        # Add targets which are not compatible with the current host
+        if not host_compatible:
+            additional_targets_in_family = sorted([
+                t for t in archspec.cpu.TARGETS.values()
+                if (t.family.name == uarch.family.name and
+                    t not in candidate_targets)
+            ], key=lambda x: len(x.ancestors), reverse=True)
+            candidate_targets += additional_targets_in_family
+
+        # Check if we want only generic architecture
+        if granularity == 'generic':
+            candidate_targets = [t for t in candidate_targets if t.vendor == 'generic']
+
         compilers = self.possible_compilers
 
-        # this loop can be used to limit the number of targets
+        # Add targets explicitly requested from specs
-        # considered. Right now we consider them all, but it seems that
+        for spec in specs:
-        # many targets can make things slow.
+            if not spec.architecture or not spec.architecture.target:
-        # TODO: investigate this.
+                continue
+
+            target = archspec.cpu.TARGETS.get(spec.target.name)
+            if not target:
+                self.target_ranges(spec, None)
+                continue
+
+            if target not in candidate_targets and not host_compatible:
+                candidate_targets.append(target)
+                for ancestor in target.ancestors:
+                    if ancestor not in candidate_targets:
+                        candidate_targets.append(ancestor)
+
         best_targets = set([uarch.family.name])
         for compiler in sorted(compilers):
             supported = self._supported_targets(
-                compiler.name, compiler.version, compatible_targets
+                compiler.name, compiler.version, candidate_targets
             )
 
             # If we can't find supported targets it may be due to custom
@@ -1442,7 +1467,7 @@ def target_defaults(self, specs):
                 supported = self._supported_targets(
                     compiler.name,
                     compiler_obj.real_version,
-                    compatible_targets
+                    candidate_targets
                 )
 
             if not supported:
@@ -1458,21 +1483,8 @@ def target_defaults(self, specs):
                 compiler.name, compiler.version, uarch.family.name
             ))
 
-        # add any targets explicitly mentioned in specs
-        for spec in specs:
-            if not spec.architecture or not spec.architecture.target:
-                continue
-
-            target = archspec.cpu.TARGETS.get(spec.target.name)
-            if not target:
-                self.target_ranges(spec, None)
-                continue
-
-            if target not in compatible_targets:
-                compatible_targets.append(target)
-
         i = 0
-        for target in compatible_targets:
+        for target in candidate_targets:
             self.gen.fact(fn.target(target.name))
             self.gen.fact(fn.target_family(target.name, target.family.name))
             for parent in sorted(target.parents):

lib/spack/spack/test/concretize.py

@@ -1526,6 +1526,48 @@ def test_conditional_values_in_conditional_variant(self):
         s = Spec('conditional-values-in-variant@1.60.0').concretized()
         assert 'cxxstd' in s.variants
 
+    def test_target_granularity(self):
+        if spack.config.get('config:concretizer') == 'original':
+            pytest.skip(
+                'Original concretizer cannot account for target granularity'
+            )
+
+        # The test architecture uses core2 as the default target. Check that when
+        # we configure Spack for "generic" granularity we concretize for x86_64
+        s = Spec('python')
+        assert s.concretized().satisfies('target=core2')
+        with spack.config.override('concretizer:targets', {'granularity': 'generic'}):
+            assert s.concretized().satisfies('target=x86_64')
+
+    def test_host_compatible_concretization(self):
+        if spack.config.get('config:concretizer') == 'original':
+            pytest.skip(
+                'Original concretizer cannot account for host compatibility'
+            )
+
+        # Check that after setting "host_compatible" to false we cannot concretize.
+        # Here we use "k10" to set a target non-compatible with the current host
+        # to avoid a lot of boilerplate when mocking the test platform. The issue
+        # is that the defaults for the test platform are very old, so there's no
+        # compiler supporting e.g. icelake etc.
+        s = Spec('python target=k10')
+        assert s.concretized()
+        with spack.config.override('concretizer:targets', {'host_compatible': True}):
+            with pytest.raises(spack.error.SpackError):
+                s.concretized()
+
+    def test_add_microarchitectures_on_explicit_request(self):
+        if spack.config.get('config:concretizer') == 'original':
+            pytest.skip(
+                'Original concretizer cannot account for host compatibility'
+            )
+
+        # Check that if we consider only "generic" targets, we can still solve for
+        # specific microarchitectures on explicit requests
+        with spack.config.override('concretizer:targets', {'granularity': 'generic'}):
+            s = Spec('python target=k10').concretized()
+        assert s.satisfies('target=k10')
+
     @pytest.mark.regression('29201')
     def test_delete_version_and_reuse(
             self, mutable_database, repo_with_changing_recipe

lib/spack/spack/test/data/config/concretizer.yaml

@@ -0,0 +1,5 @@
+concretizer:
+  # reuse is missing on purpose, see "test_concretizer_arguments"
+  targets:
+    granularity: microarchitectures
+    host_compatible: false