environments: allow sigils to apply to entire reference (#15245)

* environments: allow sigils to apply to entire reference

lib/spack/docs/environments.rst

@@ -602,7 +602,7 @@ files are identical.
    spack:
      definitions:
        - first: [libelf, libdwarf]
-       - compilers: ['%gcc', '^intel']
+       - compilers: ['%gcc', '%intel']
        - second:
            - $first
            - matrix:
@@ -676,6 +676,40 @@ The valid variables for a ``when`` clause are:
 #. ``hostname``. The hostname of the system (if ``hostname`` is an
    executable in the user's PATH).
 
+""""""""""""""""""""""""
+SpecLists as Constraints
+""""""""""""""""""""""""
+
+Dependencies and compilers in Spack can be both packages in an
+environment and constraints on other packages. References to SpecLists
+allow a shorthand to treat packages in a list as either a compiler or
+a dependency using the ``$%`` or ``$^`` syntax respectively.
+
+For example, the following environment has three root packages:
+``gcc@8.1.0``, ``mvapich2@2.3.1 %gcc@8.1.0``, and ``hdf5+mpi
+%gcc@8.1.0 ^mvapich2@2.3.1``.
+
+.. code-block:: yaml
+
+   spack:
+     definitions:
+     - compilers: [gcc@8.1.0]
+     - mpis: [mvapich2@2.3.1]
+     - packages: [hdf5+mpi]
+
+     specs:
+     - $compilers
+     - matrix:
+       - [$mpis]
+       - [$%compilers]
+     - matrix:
+       - [$packages]
+       - [$^mpis]
+       - [$%compilers]
+
+This allows for a much-needed reduction in redundancy between packages
+and constraints.
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 Environment-managed Views
 ^^^^^^^^^^^^^^^^^^^^^^^^^

lib/spack/spack/spec_list.py

@@ -124,12 +124,27 @@ def _expand_references(self, yaml):
         if isinstance(yaml, list):
             for idx, item in enumerate(yaml):
                 if isinstance(item, string_types) and item.startswith('$'):
-                    name = item[1:]
+                    # Reference type can add a constraint to items
+                    if item[1] in ('^', '%'):
+                        name = item[2:]
+                        sigil = item[1]
+                    else:
+                        name = item[1:]
+                        sigil = ''
                     if name in self._reference:
                         ret = [self._expand_references(i) for i in yaml[:idx]]
                         ret += self._reference[name].specs_as_yaml_list
                         ret += self._expand_references(yaml[idx + 1:])
-                        return ret
+
+                        # Add the sigil if we're mapping a sigil to a ref
+                        def sigilify(arg):
+                            if isinstance(arg, dict):
+                                if sigil:
+                                    arg['sigil'] = sigil
+                                return arg
+                            else:
+                                return sigil + arg
+                        return list(map(sigilify, ret))
                     else:
                         msg = 'SpecList %s refers to ' % self.name
                         msg += 'named list %s ' % name
@@ -159,13 +174,17 @@ def _expand_matrix_constraints(object, specify=True):
         new_row = []
         for r in row:
             if isinstance(r, dict):
-                new_row.extend(_expand_matrix_constraints(r, specify=False))
+                new_row.extend(
+                    [[' '.join(c)]
+                     for c in _expand_matrix_constraints(r, specify=False)])
             else:
                 new_row.append([r])
         expanded_rows.append(new_row)
 
-    results = []
     excludes = object.get('exclude', [])  # only compute once
+    sigil = object.get('sigil', '')
+
+    results = []
     for combo in itertools.product(*expanded_rows):
         # Construct a combined spec to test against excludes
         flat_combo = [constraint for list in combo for constraint in list]
@@ -174,6 +193,9 @@ def _expand_matrix_constraints(object, specify=True):
         if any(test_spec.satisfies(x) for x in excludes):
             continue
 
+        if sigil:  # add sigil if necessary
+            ordered_combo[0] = sigil + ordered_combo[0]
+
         # Add to list of constraints
         if specify:
             results.append([Spec(x) for x in ordered_combo])

lib/spack/spack/test/cmd/env.py

@@ -1040,6 +1040,55 @@ def test_stack_yaml_definitions(tmpdir):
         assert Spec('callpath') in test.user_specs
 
 
+def test_stack_yaml_definitions_as_constraints(tmpdir):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [mpileaks, callpath]
+    - mpis: [mpich, openmpi]
+  specs:
+    - matrix:
+      - [$packages]
+      - [$^mpis]
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+        test = ev.read('test')
+
+        assert Spec('mpileaks^mpich') in test.user_specs
+        assert Spec('callpath^mpich') in test.user_specs
+        assert Spec('mpileaks^openmpi') in test.user_specs
+        assert Spec('callpath^openmpi') in test.user_specs
+
+
+def test_stack_yaml_definitions_as_constraints_on_matrix(tmpdir):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [mpileaks, callpath]
+    - mpis:
+      - matrix:
+        - [mpich]
+        - ['@3.0.4', '@3.0.3']
+  specs:
+    - matrix:
+      - [$packages]
+      - [$^mpis]
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+        test = ev.read('test')
+
+        assert Spec('mpileaks^mpich@3.0.4') in test.user_specs
+        assert Spec('callpath^mpich@3.0.4') in test.user_specs
+        assert Spec('mpileaks^mpich@3.0.3') in test.user_specs
+        assert Spec('callpath^mpich@3.0.3') in test.user_specs
+
+
 @pytest.mark.regression('12095')
 def test_stack_yaml_definitions_write_reference(tmpdir):
     filename = str(tmpdir.join('spack.yaml'))