Apply dev specs for dependencies of roots (#30909)

Currently, develop specs that are not roots and are not explicitly listed dependencies 
of the roots are not applied.

- [x] ensure dev specs are applied.

Co-authored-by: Todd Gamblin <tgamblin@llnl.gov>

lib/spack/spack/solver/asp.py

@@ -102,7 +102,15 @@ def getter(node):
 ast_sym = ast_getter("symbol", "term")
 
 #: Order of precedence for version origins. Topmost types are preferred.
-version_origin_fields = ["spec", "external", "packages_yaml", "package_py", "installed"]
+version_origin_fields = [
+    "spec",
+    "dev_spec",
+    "external",
+    "packages_yaml",
+    "package_py",
+    "installed",
+]
+
 #: Look up version precedence strings by enum id
 version_origin_str = {i: name for i, name in enumerate(version_origin_fields)}
 
@@ -1489,7 +1497,7 @@ class Body(object):
 
         return clauses
 
-    def build_version_dict(self, possible_pkgs, specs):
+    def build_version_dict(self, possible_pkgs):
         """Declare any versions in specs not declared in packages."""
         self.declared_versions = collections.defaultdict(list)
         self.possible_versions = collections.defaultdict(set)
@@ -1530,6 +1538,8 @@ def key_fn(item):
                     DeclaredVersion(version=ver, idx=idx, origin=version_provenance.packages_yaml)
                 )
 
+    def add_concrete_versions_from_specs(self, specs, origin):
+        """Add concrete versions to possible versions from lists of CLI/dev specs."""
         for spec in specs:
             for dep in spec.traverse():
                 if not dep.versions.concrete:
@@ -1553,7 +1563,7 @@ def key_fn(item):
                 # about*, add it to the known versions. Use idx=0, which is the
                 # best possible, so they're guaranteed to be used preferentially.
                 self.declared_versions[dep.name].append(
-                    DeclaredVersion(version=dep.version, idx=0, origin=version_provenance.spec)
+                    DeclaredVersion(version=dep.version, idx=0, origin=origin)
                 )
                 self.possible_versions[dep.name].add(dep.version)
 
@@ -1944,11 +1954,28 @@ def setup(self, driver, specs, reuse=None):
         # rules to generate an ASP program.
         self.gen = driver
 
+        # Calculate develop specs
+        # they will be used in addition to command line specs
+        # in determining known versions/targets/os
+        dev_specs = ()
+        env = ev.active_environment()
+        if env:
+            dev_specs = tuple(
+                spack.spec.Spec(info["spec"]).constrained(
+                    "dev_path=%s"
+                    % spack.util.path.canonicalize_path(info["path"], default_wd=env.path)
+                )
+                for name, info in env.dev_specs.items()
+            )
+        specs = tuple(specs)  # ensure compatible types to add
+
         # get possible compilers
         self.possible_compilers = self.generate_possible_compilers(specs)
 
         # traverse all specs and packages to build dict of possible versions
-        self.build_version_dict(possible, specs)
+        self.build_version_dict(possible)
+        self.add_concrete_versions_from_specs(specs, version_provenance.spec)
+        self.add_concrete_versions_from_specs(dev_specs, version_provenance.dev_spec)
 
         self.gen.h1("Concrete input spec definitions")
         self.define_concrete_input_specs(specs, possible)
@@ -1966,8 +1993,8 @@ def setup(self, driver, specs, reuse=None):
 
         # architecture defaults
         self.platform_defaults()
-        self.os_defaults(specs)
+        self.os_defaults(specs + dev_specs)
-        self.target_defaults(specs)
+        self.target_defaults(specs + dev_specs)
 
         self.virtual_providers()
         self.provider_defaults()
@@ -1984,11 +2011,8 @@ def setup(self, driver, specs, reuse=None):
             self.target_preferences(pkg)
 
         # Inject dev_path from environment
-        env = ev.active_environment()
+        for ds in dev_specs:
-        if env:
+            self.condition(spack.spec.Spec(ds.name), ds, msg="%s is a develop spec" % ds.name)
-            for spec in sorted(specs):
-                for dep in spec.traverse():
-                    _develop_specs_from_env(dep, env)
 
         self.gen.h1("Spec Constraints")
         self.literal_specs(specs)
@@ -2311,8 +2335,7 @@ def _develop_specs_from_env(spec, env):
             "Internal Error: The dev_path for spec {name} is not connected to a valid environment"
             "path. Please note that develop specs can only be used inside an environment"
             "These paths should be the same:\n\tdev_path:{dev_path}\n\tenv_based_path:{env_path}"
-        )
+        ).format(name=spec.name, dev_path=spec.variants["dev_path"], env_path=path)
-        error_msg.format(name=spec.name, dev_path=spec.variants["dev_path"], env_path=path)
 
         assert spec.variants["dev_path"].value == path, error_msg
     else:

lib/spack/spack/test/cmd/dev_build.py

@@ -254,13 +254,18 @@ def test_dev_build_env_version_mismatch(
 def test_dev_build_multiple(
     tmpdir, mock_packages, install_mockery, mutable_mock_env_path, mock_fetch
 ):
-    """Test spack install with multiple developer builds"""
+    """Test spack install with multiple developer builds
+
+    Test that only the root needs to be specified in the environment
+    Test that versions known only from the dev specs are included in the solve,
+    even if they come from a non-root
+    """
     # setup dev-build-test-install package for dev build
     # Wait to concretize inside the environment to set dev_path on the specs;
     # without the environment, the user would need to set dev_path for both the
     # root and dependency if they wanted a dev build for both.
     leaf_dir = tmpdir.mkdir("leaf")
-    leaf_spec = spack.spec.Spec("dev-build-test-install@0.0.0")
+    leaf_spec = spack.spec.Spec("dev-build-test-install@1.0.0")
     leaf_pkg_cls = spack.repo.path.get_pkg_class(leaf_spec.name)
     with leaf_dir.as_cwd():
         with open(leaf_pkg_cls.filename, "w") as f:
@@ -283,13 +288,12 @@ def test_dev_build_multiple(
                 """\
 env:
   specs:
-  - dev-build-test-install@0.0.0
   - dev-build-test-dependent@0.0.0
 
   develop:
     dev-build-test-install:
       path: %s
-      spec: dev-build-test-install@0.0.0
+      spec: dev-build-test-install@1.0.0
     dev-build-test-dependent:
       spec: dev-build-test-dependent@0.0.0
       path: %s
@@ -300,6 +304,7 @@ def test_dev_build_multiple(
         env("create", "test", "./spack.yaml")
         with ev.read("test"):
             # Do concretization inside environment for dev info
+            # These specs are the source of truth to compare against the installs
             leaf_spec.concretize()
             root_spec.concretize()