concretizer: virtual entry in packages.yaml, external modules

This commit adds support for specifying rules in
packages.yaml that refer to virtual packages.

The approach is to normalize in memory each
configuration and turn it into an equivalent
configuration without rules on virtual. This
is possible if the set of packages to be handled
is considered fixed.

lib/spack/spack/solver/asp.py

@@ -6,6 +6,8 @@
 from __future__ import print_function
 
 import collections
+import copy
+import itertools
 import os
 import pkgutil
 import pprint
@@ -412,6 +414,30 @@ def _normalize(body):
     return args
 
 
+def _normalize_packages_yaml(packages_yaml):
+    normalized_yaml = copy.copy(packages_yaml)
+    for pkg_name in packages_yaml:
+        is_virtual = spack.repo.path.is_virtual(pkg_name)
+        if pkg_name == 'all' or not is_virtual:
+            continue
+
+        # Remove the virtual entry from the normalized configuration
+        data = normalized_yaml.pop(pkg_name)
+        is_buildable = data.get('buildable', True)
+        if not is_buildable:
+            for provider in spack.repo.path.providers_for(pkg_name):
+                entry = normalized_yaml.setdefault(provider.name, {})
+                entry['buildable'] = False
+
+        externals = data.get('externals', [])
+        keyfn = lambda x: spack.spec.Spec(x['spec']).name
+        for provider, specs in itertools.groupby(externals, key=keyfn):
+            entry = normalized_yaml.setdefault(provider, {})
+            entry.setdefault('externals', []).extend(specs)
+
+    return normalized_yaml
+
+
 class PyclingoDriver(object):
     def __init__(self, cores=True, asp=None):
         """Driver for the Python clingo interface.
@@ -934,7 +960,12 @@ def provider_defaults(self):
 
     def external_packages(self):
         """Facts on external packages, as read from packages.yaml"""
+        # Read packages.yaml and normalize it, so that it
+        # will not contain entries referring to virtual
+        # packages.
         packages_yaml = spack.config.get("packages")
+        packages_yaml = _normalize_packages_yaml(packages_yaml)
+
         self.gen.h1('External packages')
         for pkg_name, data in packages_yaml.items():
             if pkg_name == 'all':
@@ -1526,6 +1557,7 @@ def external_spec(self, pkg, idx):
         has been selected for this package.
         """
         packages_yaml = spack.config.get('packages')
+        packages_yaml = _normalize_packages_yaml(packages_yaml)
         spec_info = packages_yaml[pkg]['externals'][int(idx)]
         self._specs[pkg].external_path = spec_info.get('prefix', None)
         self._specs[pkg].external_modules = spec_info.get('modules', [])
@@ -1623,6 +1655,10 @@ def build_specs(self, function_tuples):
         # fix flags after all specs are constructed
         self.reorder_flags()
 
+        # Add external paths to specs with just external modules
+        for s in self._specs.values():
+            spack.spec.Spec.ensure_external_path_if_external(s)
+
         return self._specs
 
 

lib/spack/spack/spec.py

@@ -2362,17 +2362,9 @@ def _old_concretize(self, tests=False):
                 t[-1] for t in ordered_hashes)
 
         for s in self.traverse():
-            if s.external_modules and not s.external_path:
+            # TODO: Refactor this into a common method to build external specs
-                compiler = spack.compilers.compiler_for_spec(
+            # TODO: or turn external_path into a lazy property
-                    s.compiler, s.architecture)
+            self.ensure_external_path_if_external(s)
-                for mod in compiler.modules:
-                    md.load_module(mod)
-
-                # get the path from the module
-                # the package can override the default
-                s.external_path = getattr(s.package, 'external_prefix',
-                                          md.path_from_modules(
-                                              s.external_modules))
 
         # Mark everything in the spec as concrete, as well.
         self._mark_concrete()
@@ -2419,6 +2411,21 @@ def _old_concretize(self, tests=False):
         # there are declared inconsistencies)
         self.architecture.target.optimization_flags(self.compiler)
 
+    @staticmethod
+    def ensure_external_path_if_external(external_spec):
+        if external_spec.external_modules and not external_spec.external_path:
+            compiler = spack.compilers.compiler_for_spec(
+                external_spec.compiler, external_spec.architecture)
+            for mod in compiler.modules:
+                md.load_module(mod)
+
+            # get the path from the module
+            # the package can override the default
+            external_spec.external_path = getattr(
+                external_spec.package, 'external_prefix',
+                md.path_from_modules(external_spec.external_modules)
+            )
+
     def _new_concretize(self, tests=False):
         import spack.solver.asp