diff --git a/lib/spack/spack/cmd/style.py b/lib/spack/spack/cmd/style.py index d6a5e91a15a..49e03e3ae98 100644 --- a/lib/spack/spack/cmd/style.py +++ b/lib/spack/spack/cmd/style.py @@ -18,8 +18,8 @@ import spack.repo import spack.util.git import spack.util.spack_yaml -from spack.spec_parser import SPEC_TOKENIZER, SpecTokens -from spack.tokenize import Token +from spack.spec_parser import NAME, VERSION_LIST, SpecTokens +from spack.tokenize import Token, TokenBase, Tokenizer from spack.util.executable import Executable, which description = "runs source code style checks on spack" @@ -206,8 +206,8 @@ def setup_parser(subparser): "--spec-strings", action="store_true", help="upgrade spec strings in Python, JSON and YAML files for compatibility with Spack " - "v1.0 and v0.x. Example: spack style --spec-strings $(git ls-files). Note: this flag " - "will be removed in Spack v1.0.", + "v1.0 and v0.x. Example: spack style --spec-strings $(git ls-files). Note: must be " + "used only on specs from spack v0.X.", ) subparser.add_argument("files", nargs=argparse.REMAINDER, help="specific files to check") @@ -521,20 +521,52 @@ def _bootstrap_dev_dependencies(): IS_PROBABLY_COMPILER = re.compile(r"%[a-zA-Z_][a-zA-Z0-9\-]") +class _LegacySpecTokens(TokenBase): + """Reconstructs the tokens for previous specs, so we can reuse code to rotate them""" + + # Dependency + START_EDGE_PROPERTIES = r"(?:\^\[)" + END_EDGE_PROPERTIES = r"(?:\])" + DEPENDENCY = r"(?:\^)" + # Version + VERSION_HASH_PAIR = SpecTokens.VERSION_HASH_PAIR.regex + GIT_VERSION = SpecTokens.GIT_VERSION.regex + VERSION = SpecTokens.VERSION.regex + # Variants + PROPAGATED_BOOL_VARIANT = SpecTokens.PROPAGATED_BOOL_VARIANT.regex + BOOL_VARIANT = SpecTokens.BOOL_VARIANT.regex + PROPAGATED_KEY_VALUE_PAIR = SpecTokens.PROPAGATED_KEY_VALUE_PAIR.regex + KEY_VALUE_PAIR = SpecTokens.KEY_VALUE_PAIR.regex + # Compilers + COMPILER_AND_VERSION = rf"(?:%\s*(?:{NAME})(?:[\s]*)@\s*(?:{VERSION_LIST}))" + COMPILER = rf"(?:%\s*(?:{NAME}))" + # FILENAME + FILENAME = SpecTokens.FILENAME.regex + # Package name + FULLY_QUALIFIED_PACKAGE_NAME = SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME.regex + UNQUALIFIED_PACKAGE_NAME = SpecTokens.UNQUALIFIED_PACKAGE_NAME.regex + # DAG hash + DAG_HASH = SpecTokens.DAG_HASH.regex + # White spaces + WS = SpecTokens.WS.regex + # Unexpected character(s) + UNEXPECTED = SpecTokens.UNEXPECTED.regex + + def _spec_str_reorder_compiler(idx: int, blocks: List[List[Token]]) -> None: # only move the compiler to the back if it exists and is not already at the end if not 0 <= idx < len(blocks) - 1: return # if there's only whitespace after the compiler, don't move it - if all(token.kind == SpecTokens.WS for block in blocks[idx + 1 :] for token in block): + if all(token.kind == _LegacySpecTokens.WS for block in blocks[idx + 1 :] for token in block): return # rotate left and always add at least one WS token between compiler and previous token compiler_block = blocks.pop(idx) - if compiler_block[0].kind != SpecTokens.WS: - compiler_block.insert(0, Token(SpecTokens.WS, " ")) + if compiler_block[0].kind != _LegacySpecTokens.WS: + compiler_block.insert(0, Token(_LegacySpecTokens.WS, " ")) # delete the WS tokens from the new first block if it was at the very start, to prevent leading # WS tokens. - while idx == 0 and blocks[0][0].kind == SpecTokens.WS: + while idx == 0 and blocks[0][0].kind == _LegacySpecTokens.WS: blocks[0].pop(0) blocks.append(compiler_block) @@ -552,11 +584,13 @@ def _spec_str_format(spec_str: str) -> Optional[str]: compiler_block_idx = -1 in_edge_attr = False - for token in SPEC_TOKENIZER.tokenize(spec_str): - if token.kind == SpecTokens.UNEXPECTED: + legacy_tokenizer = Tokenizer(_LegacySpecTokens) + + for token in legacy_tokenizer.tokenize(spec_str): + if token.kind == _LegacySpecTokens.UNEXPECTED: # parsing error, we cannot fix this string. return None - elif token.kind in (SpecTokens.COMPILER, SpecTokens.COMPILER_AND_VERSION): + elif token.kind in (_LegacySpecTokens.COMPILER, _LegacySpecTokens.COMPILER_AND_VERSION): # multiple compilers are not supported in Spack v0.x, so early return if compiler_block_idx != -1: return None @@ -565,19 +599,19 @@ def _spec_str_format(spec_str: str) -> Optional[str]: current_block = [] compiler_block_idx = len(blocks) - 1 elif token.kind in ( - SpecTokens.START_EDGE_PROPERTIES, - SpecTokens.DEPENDENCY, - SpecTokens.UNQUALIFIED_PACKAGE_NAME, - SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, + _LegacySpecTokens.START_EDGE_PROPERTIES, + _LegacySpecTokens.DEPENDENCY, + _LegacySpecTokens.UNQUALIFIED_PACKAGE_NAME, + _LegacySpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, ): _spec_str_reorder_compiler(compiler_block_idx, blocks) compiler_block_idx = -1 - if token.kind == SpecTokens.START_EDGE_PROPERTIES: + if token.kind == _LegacySpecTokens.START_EDGE_PROPERTIES: in_edge_attr = True current_block.append(token) blocks.append(current_block) current_block = [] - elif token.kind == SpecTokens.END_EDGE_PROPERTIES: + elif token.kind == _LegacySpecTokens.END_EDGE_PROPERTIES: in_edge_attr = False current_block.append(token) blocks.append(current_block) @@ -585,19 +619,19 @@ def _spec_str_format(spec_str: str) -> Optional[str]: elif in_edge_attr: current_block.append(token) elif token.kind in ( - SpecTokens.VERSION_HASH_PAIR, - SpecTokens.GIT_VERSION, - SpecTokens.VERSION, - SpecTokens.PROPAGATED_BOOL_VARIANT, - SpecTokens.BOOL_VARIANT, - SpecTokens.PROPAGATED_KEY_VALUE_PAIR, - SpecTokens.KEY_VALUE_PAIR, - SpecTokens.DAG_HASH, + _LegacySpecTokens.VERSION_HASH_PAIR, + _LegacySpecTokens.GIT_VERSION, + _LegacySpecTokens.VERSION, + _LegacySpecTokens.PROPAGATED_BOOL_VARIANT, + _LegacySpecTokens.BOOL_VARIANT, + _LegacySpecTokens.PROPAGATED_KEY_VALUE_PAIR, + _LegacySpecTokens.KEY_VALUE_PAIR, + _LegacySpecTokens.DAG_HASH, ): current_block.append(token) blocks.append(current_block) current_block = [] - elif token.kind == SpecTokens.WS: + elif token.kind == _LegacySpecTokens.WS: current_block.append(token) else: raise ValueError(f"unexpected token {token}") diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index ff75556c22c..9c3b31e2357 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -2556,7 +2556,7 @@ def _spec_clauses( edges = spec.edges_from_dependents() virtuals = [x for x in itertools.chain.from_iterable([edge.virtuals for edge in edges])] - if not body: + if not body and not spec.concrete: for virtual in virtuals: clauses.append(fn.attr("provider_set", spec.name, virtual)) clauses.append(fn.attr("virtual_node", virtual)) diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp index 088a83d1c2c..38f08df0d35 100644 --- a/lib/spack/spack/solver/concretize.lp +++ b/lib/spack/spack/solver/concretize.lp @@ -530,6 +530,32 @@ attr("concrete_variant_set", node(X, A1), Variant, Value, ID) attr("virtual_on_build_edge", ParentNode, BuildDependency, Virtual), not 1 { pkg_fact(BuildDependency, version_satisfies(Constraint, Version)) : hash_attr(BuildDependencyHash, "version", BuildDependency, Version) } 1. +error(100, "Cannot satisfy the request on {0} to have {1}={2}", BuildDependency, Variant, Value) +:- attr("build_requirement", ParentNode, build_requirement("variant_set", BuildDependency, Variant, Value)), + attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), + not hash_attr(BuildDependencyHash, "variant_value", BuildDependency, Variant, Value). + +error(100, "Cannot satisfy the request on {0} to have the target set to {1}", BuildDependency, Target) +:- attr("build_requirement", ParentNode, build_requirement("node_target_set", BuildDependency, Target)), + attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), + not hash_attr(BuildDependencyHash, "node_target", BuildDependency, Target). + +error(100, "Cannot satisfy the request on {0} to have the os set to {1}", BuildDependency, NodeOS) +:- attr("build_requirement", ParentNode, build_requirement("node_os_set", BuildDependency, NodeOS)), + attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), + not hash_attr(BuildDependencyHash, "node_os", BuildDependency, NodeOS). + +error(100, "Cannot satisfy the request on {0} to have the platform set to {1}", BuildDependency, Platform) +:- attr("build_requirement", ParentNode, build_requirement("node_platform_set", BuildDependency, Platform)), + attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), + not hash_attr(BuildDependencyHash, "node_platform", BuildDependency, Platform). + +error(100, "Cannot satisfy the request on {0} to have the following hash {1}", BuildDependency, BuildHash) +:- attr("build_requirement", ParentNode, build_requirement("node_target_set", BuildDependency, Target)), + attr("concrete_build_dependency", ParentNode, BuildDependency, BuildDependencyHash), + attr("build_requirement", ParentNode, build_requirement("hash", BuildDependency, BuildHash)), + BuildHash != BuildDependencyHash. + % External nodes :- attr("build_requirement", ParentNode, build_requirement("node", BuildDependency)), external(ParentNode), @@ -576,6 +602,32 @@ attr("node_version_satisfies", node(X, BuildDependency), Constraint) :- attr("build_requirement", ParentNode, build_requirement("node_version_satisfies", BuildDependency, Constraint)), build_requirement(ParentNode, node(X, BuildDependency)). +% Account for properties on the build requirements +% +% root %gcc@12.0 ^dep +% +attr("variant_set", node(X, BuildDependency), Variant, Value) :- + attr("build_requirement", ParentNode, build_requirement("variant_set", BuildDependency, Variant, Value)), + build_requirement(ParentNode, node(X, BuildDependency)). + +attr("depends_on", node(X, Parent), node(Y, BuildDependency), "build") :- build_requirement(node(X, Parent), node(Y, BuildDependency)). + +attr("node_target_set", node(X, BuildDependency), Target) :- + attr("build_requirement", ParentNode, build_requirement("node_target_set", BuildDependency, Target)), + build_requirement(ParentNode, node(X, BuildDependency)). + +attr("node_os_set", node(X, BuildDependency), NodeOS) :- + attr("build_requirement", ParentNode, build_requirement("node_os_set", BuildDependency, NodeOS)), + build_requirement(ParentNode, node(X, BuildDependency)). + +attr("node_platform_set", node(X, BuildDependency), NodePlatform) :- + attr("build_requirement", ParentNode, build_requirement("node_platform_set", BuildDependency, NodePlatform)), + build_requirement(ParentNode, node(X, BuildDependency)). + +attr("hash", node(X, BuildDependency), BuildHash) :- + attr("build_requirement", ParentNode, build_requirement("hash", BuildDependency, BuildHash)), + build_requirement(ParentNode, node(X, BuildDependency)). + 1 { attr("provider_set", node(X, BuildDependency), node(0..Y-1, Virtual)) : max_dupes(Virtual, Y) } 1 :- attr("build_requirement", ParentNode, build_requirement("provider_set", BuildDependency, Virtual)), diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e2e2d365264..01fb7f18d5d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2233,15 +2233,21 @@ def lookup_hash(self): spec._dup(self._lookup_hash()) return spec - # Get dependencies that need to be replaced - for node in self.traverse(root=False): - if node.abstract_hash: - spec._add_dependency(node._lookup_hash(), depflag=0, virtuals=()) + # Map the dependencies that need to be replaced + node_lookup = { + id(node): node._lookup_hash() + for node in self.traverse(root=False) + if node.abstract_hash + } - # reattach nodes that were not otherwise satisfied by new dependencies - for node in self.traverse(root=False): - if not any(n.satisfies(node) for n in spec.traverse()): - spec._add_dependency(node.copy(), depflag=0, virtuals=()) + # Reconstruct dependencies + for edge in self.traverse_edges(root=False): + key = edge.parent.name + current_node = spec if key == spec.name else spec[key] + child_node = node_lookup.get(id(edge.spec), edge.spec.copy()) + current_node._add_dependency( + child_node, depflag=edge.depflag, virtuals=edge.virtuals, direct=edge.direct + ) return spec diff --git a/lib/spack/spack/spec_parser.py b/lib/spack/spack/spec_parser.py index f6392c33808..aab6dc292b7 100644 --- a/lib/spack/spack/spec_parser.py +++ b/lib/spack/spack/spec_parser.py @@ -101,9 +101,6 @@ SPLIT_KVP = re.compile(rf"^({NAME})(:?==?)(.*)$") -#: Regex with groups to use for splitting %[virtuals=...] tokens -SPLIT_COMPILER_TOKEN = re.compile(rf"^%\[virtuals=({VALUE}|{QUOTED_VALUE})]\s*(.*)$") - #: A filename starts either with a "." or a "/" or a "{name}/, or on Windows, a drive letter #: followed by a colon and "\" or "." or {name}\ WINDOWS_FILENAME = r"(?:\.|[a-zA-Z0-9-_]*\\|[a-zA-Z]:\\)(?:[a-zA-Z0-9-_\.\\]*)(?:\.json|\.yaml)" @@ -124,9 +121,9 @@ class SpecTokens(TokenBase): """ # Dependency - START_EDGE_PROPERTIES = r"(?:\^\[)" + START_EDGE_PROPERTIES = r"(?:[\^%]\[)" END_EDGE_PROPERTIES = r"(?:\])" - DEPENDENCY = r"(?:\^)" + DEPENDENCY = r"(?:[\^\%])" # Version VERSION_HASH_PAIR = rf"(?:@(?:{GIT_VERSION_PATTERN})=(?:{VERSION}))" GIT_VERSION = rf"@(?:{GIT_VERSION_PATTERN})" @@ -136,14 +133,6 @@ class SpecTokens(TokenBase): BOOL_VARIANT = rf"(?:[~+-]\s*{NAME})" PROPAGATED_KEY_VALUE_PAIR = rf"(?:{NAME}:?==(?:{VALUE}|{QUOTED_VALUE}))" KEY_VALUE_PAIR = rf"(?:{NAME}:?=(?:{VALUE}|{QUOTED_VALUE}))" - # Compilers - COMPILER_AND_VERSION = rf"(?:%\s*(?:{NAME})(?:[\s]*)@\s*(?:{VERSION_LIST}))" - COMPILER = rf"(?:%\s*(?:{NAME}))" - COMPILER_AND_VERSION_WITH_VIRTUALS = ( - rf"(?:%\[virtuals=(?:{VALUE}|{QUOTED_VALUE})\]" - rf"\s*(?:{NAME})(?:[\s]*)@\s*(?:{VERSION_LIST}))" - ) - COMPILER_WITH_VIRTUALS = rf"(?:%\[virtuals=(?:{VALUE}|{QUOTED_VALUE})\]\s*(?:{NAME}))" # FILENAME FILENAME = rf"(?:{FILENAME})" # Package name @@ -275,25 +264,58 @@ def next_spec( def add_dependency(dep, **edge_properties): """wrapper around root_spec._add_dependency""" try: - root_spec._add_dependency(dep, **edge_properties) + target_spec._add_dependency(dep, **edge_properties) except spack.error.SpecError as e: raise SpecParsingError(str(e), self.ctx.current_token, self.literal_str) from e initial_spec = initial_spec or spack.spec.Spec() root_spec, parser_warnings = SpecNodeParser(self.ctx, self.literal_str).parse(initial_spec) + current_spec = root_spec while True: if self.ctx.accept(SpecTokens.START_EDGE_PROPERTIES): + is_direct = self.ctx.current_token.value[0] == "%" + edge_properties = EdgeAttributeParser(self.ctx, self.literal_str).parse() - edge_properties.setdefault("depflag", 0) edge_properties.setdefault("virtuals", ()) + edge_properties["direct"] = is_direct + dependency, warnings = self._parse_node(root_spec) + + if is_direct: + target_spec = current_spec + edge_properties.setdefault("depflag", spack.deptypes.BUILD) + if dependency.name in LEGACY_COMPILER_TO_BUILTIN: + dependency.name = LEGACY_COMPILER_TO_BUILTIN[dependency.name] + + else: + current_spec = dependency + target_spec = root_spec + edge_properties.setdefault("depflag", 0) + + # print(f"[{current_spec}], {target_spec}->{dependency} {is_direct}") parser_warnings.extend(warnings) add_dependency(dependency, **edge_properties) elif self.ctx.accept(SpecTokens.DEPENDENCY): + is_direct = self.ctx.current_token.value[0] == "%" dependency, warnings = self._parse_node(root_spec) + edge_properties = {} + edge_properties["direct"] = is_direct + edge_properties["virtuals"] = tuple() + if is_direct: + target_spec = current_spec + edge_properties.setdefault("depflag", spack.deptypes.BUILD) + if dependency.name in LEGACY_COMPILER_TO_BUILTIN: + dependency.name = LEGACY_COMPILER_TO_BUILTIN[dependency.name] + else: + current_spec = dependency + target_spec = root_spec + edge_properties.setdefault("depflag", 0) + + # print(f"[{current_spec}], {target_spec}->{dependency} {is_direct}") + parser_warnings.extend(warnings) - add_dependency(dependency, depflag=0, virtuals=()) + add_dependency(dependency, **edge_properties) else: break @@ -384,34 +406,6 @@ def warn_if_after_compiler(token: str): while True: if ( - self.ctx.accept(SpecTokens.COMPILER) - or self.ctx.accept(SpecTokens.COMPILER_AND_VERSION) - or self.ctx.accept(SpecTokens.COMPILER_WITH_VIRTUALS) - or self.ctx.accept(SpecTokens.COMPILER_AND_VERSION_WITH_VIRTUALS) - ): - current_token = self.ctx.current_token - if current_token.kind in ( - SpecTokens.COMPILER_WITH_VIRTUALS, - SpecTokens.COMPILER_AND_VERSION_WITH_VIRTUALS, - ): - m = SPLIT_COMPILER_TOKEN.match(current_token.value) - assert m, "SPLIT_COMPILER_TOKEN and COMPILER_* do not agree." - virtuals_str, compiler_str = m.groups() - virtuals = tuple(virtuals_str.strip("'\" ").split(",")) - else: - virtuals = tuple() - compiler_str = current_token.value[1:] - - build_dependency = spack.spec.Spec(compiler_str) - if build_dependency.name in LEGACY_COMPILER_TO_BUILTIN: - build_dependency.name = LEGACY_COMPILER_TO_BUILTIN[build_dependency.name] - - initial_spec._add_dependency( - build_dependency, depflag=spack.deptypes.BUILD, virtuals=virtuals, direct=True - ) - last_compiler = self.ctx.current_token.value - - elif ( self.ctx.accept(SpecTokens.VERSION_HASH_PAIR) or self.ctx.accept(SpecTokens.GIT_VERSION) or self.ctx.accept(SpecTokens.VERSION) diff --git a/lib/spack/spack/test/concretization/core.py b/lib/spack/spack/test/concretization/core.py index 04bfbf23f09..3df073e8ad5 100644 --- a/lib/spack/spack/test/concretization/core.py +++ b/lib/spack/spack/test/concretization/core.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os +import platform import sys import _vendoring.jinja2 @@ -3566,3 +3567,144 @@ def test_concrete_multi_valued_variants_when_args(default_mock_concretization): for c in ("foo:=a", "foo:=a,b,c", "foo:=a,b", "foo:=a,c"): s = default_mock_concretization(f"mvdefaults {c}") assert not s.satisfies("^pkg-b") + + +@pytest.mark.usefixtures("mock_packages") +@pytest.mark.parametrize( + "constraint_in_yaml,unsat_request,sat_request", + [ + # Arch parts + pytest.param( + "target=x86_64", + "target=core2", + "target=x86_64", + marks=pytest.mark.skipif( + platform.machine() != "x86_64", reason="only valid for x86_64" + ), + ), + pytest.param( + "target=core2", + "target=x86_64", + "target=core2", + marks=pytest.mark.skipif( + platform.machine() != "x86_64", reason="only valid for x86_64" + ), + ), + ("os=debian6", "os=redhat6", "os=debian6"), + ("platform=test", "platform=linux", "platform=test"), + # Variants + ("~lld", "+lld", "~lld"), + ("+lld", "~lld", "+lld"), + ], +) +def test_spec_parts_on_fresh_compilers( + constraint_in_yaml, unsat_request, sat_request, mutable_config, tmp_path +): + """Tests that spec parts like targets and variants in `% target= ` + are associated with `package` for `%` just as they would be for `^`, when we concretize + without reusing. + """ + packages_yaml = syaml.load_config( + f""" + packages: + llvm:: + buildable: false + externals: + - spec: "llvm+clang@20 {constraint_in_yaml}" + prefix: {tmp_path / 'llvm-20'} + """ + ) + mutable_config.set("packages", packages_yaml["packages"]) + + # Check the abstract spec is formed correctly + abstract_spec = Spec(f"pkg-a %llvm@20 +clang {unsat_request}") + assert abstract_spec["llvm"].satisfies(f"@20 +clang {unsat_request}") + + # Check that we can't concretize the spec, since llvm is not buildable + with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): + spack.concretize.concretize_one(abstract_spec) + + # Check we can instead concretize if we use the correct constraint + s = spack.concretize.concretize_one(f"pkg-a %llvm@20 +clang {sat_request}") + assert s["c"].external and s["c"].satisfies(f"@20 +clang {sat_request}") + + +@pytest.mark.usefixtures("mock_packages", "mutable_database") +@pytest.mark.parametrize( + "constraint_in_yaml,unsat_request,sat_request", + [ + # Arch parts + pytest.param( + "target=x86_64", + "target=core2", + "target=x86_64", + marks=pytest.mark.skipif( + platform.machine() != "x86_64", reason="only valid for x86_64" + ), + ), + pytest.param( + "target=core2", + "target=x86_64", + "target=core2", + marks=pytest.mark.skipif( + platform.machine() != "x86_64", reason="only valid for x86_64" + ), + ), + ("os=debian6", "os=redhat6", "os=debian6"), + ("platform=test", "platform=linux", "platform=test"), + # Variants + ("~lld", "+lld", "~lld"), + ("+lld", "~lld", "+lld"), + ], +) +def test_spec_parts_on_reused_compilers( + constraint_in_yaml, unsat_request, sat_request, mutable_config, tmp_path +): + """Tests that requests of the form % are considered for reused + specs, even though build dependency are not part of the ASP problem. + """ + packages_yaml = syaml.load_config( + f""" + packages: + c: + require: llvm + cxx: + require: llvm + llvm:: + buildable: false + externals: + - spec: "llvm+clang@20 {constraint_in_yaml}" + prefix: {tmp_path / 'llvm-20'} + mpileaks: + buildable: true + """ + ) + mutable_config.set("packages", packages_yaml["packages"]) + + # Install the spec + installed_spec = spack.concretize.concretize_one(f"mpileaks %llvm@20 {sat_request}") + PackageInstaller([installed_spec.package], fake=True, explicit=True).install() + + # Make mpileaks not buildable + mutable_config.set("packages:mpileaks:buildable", False) + + # Check we can't concretize with the unsat request... + with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): + spack.concretize.concretize_one(f"mpileaks %llvm@20 {unsat_request}") + + # ...but we can with the original constraint + with spack.config.override("concretizer:reuse", True): + s = spack.concretize.concretize_one(f"mpileaks %llvm@20 {sat_request}") + + assert s.dag_hash() == installed_spec.dag_hash() + + +def test_use_compiler_by_hash(mock_packages, mutable_database, mutable_config): + """Tests that we can reuse an installed compiler specifying its hash""" + installed_spec = spack.concretize.concretize_one("gcc@14.0") + PackageInstaller([installed_spec.package], fake=True, explicit=True).install() + + with spack.config.override("concretizer:reuse", True): + s = spack.concretize.concretize_one(f"mpileaks %gcc/{installed_spec.dag_hash()}") + + assert s["c"].dag_hash() == installed_spec.dag_hash() diff --git a/lib/spack/spack/test/concretization/flag_mixing.py b/lib/spack/spack/test/concretization/flag_mixing.py index 0b79f88a991..911f53c9e47 100644 --- a/lib/spack/spack/test/concretization/flag_mixing.py +++ b/lib/spack/spack/test/concretization/flag_mixing.py @@ -163,16 +163,15 @@ def test_flag_order_and_grouping( if cmp_flags: compiler_spec = "%gcc@12.100.100" + cmd_flags_str = f'cflags="{cmd_flags}"' if cmd_flags else "" + if dflags: - spec_str = f"x+activatemultiflag {compiler_spec} ^y" + spec_str = f"x+activatemultiflag {compiler_spec} ^y {cmd_flags_str}" expected_dflags = "-d1 -d2" else: - spec_str = f"y {compiler_spec}" + spec_str = f"y {cmd_flags_str} {compiler_spec}" expected_dflags = None - if cmd_flags: - spec_str += f' cflags="{cmd_flags}"' - root_spec = spack.concretize.concretize_one(spec_str) spec = root_spec["y"] satisfy_flags = " ".join(x for x in [cmd_flags, req_flags, cmp_flags, expected_dflags] if x) @@ -277,6 +276,6 @@ def test_flag_injection_different_compilers(mock_packages, mutable_config): """Tests that flag propagation is not activated on nodes with a compiler that is different from the propagation source. """ - s = spack.concretize.concretize_one('mpileaks %gcc cflags=="-O2" ^callpath %llvm') + s = spack.concretize.concretize_one('mpileaks cflags=="-O2" %gcc ^callpath %llvm') assert s.satisfies('cflags="-O2"') and s["c"].name == "gcc" assert not s["callpath"].satisfies('cflags="-O2"') and s["callpath"]["c"].name == "llvm" diff --git a/lib/spack/spack/test/spec_list.py b/lib/spack/spack/test/spec_list.py index e8068643a6c..1a5c51f8df9 100644 --- a/lib/spack/spack/test/spec_list.py +++ b/lib/spack/spack/test/spec_list.py @@ -137,7 +137,14 @@ def test_spec_list_nested_matrices(self, parser_and_speclist): expected_components = itertools.product( ["zlib", "libelf"], ["%gcc", "%intel"], ["+shared", "~shared"] ) - expected = [Spec(" ".join(combo)) for combo in expected_components] + + def _reduce(*, combo): + root = Spec(combo[0]) + for x in combo[1:]: + root.constrain(x) + return root + + expected = [_reduce(combo=combo) for combo in expected_components] assert set(result.specs) == set(expected) @pytest.mark.regression("16897") diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index d3ac7344cce..2cadb89016a 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -51,10 +51,6 @@ def dependency_with_version(text): ) -def compiler_with_version_range(text): - return text, [Token(SpecTokens.COMPILER_AND_VERSION, value=text)], text - - @pytest.fixture() def specfile_for(default_mock_concretization): def _specfile_for(spec_str, filename): @@ -84,7 +80,6 @@ def _specfile_for(spec_str, filename): simple_package_name("3dtk"), simple_package_name("ns-3-dev"), # Single token anonymous specs - ("%intel", [Token(SpecTokens.COMPILER, value="%intel")], "%intel"), ("@2.7", [Token(SpecTokens.VERSION, value="@2.7")], "@2.7"), ("@2.7:", [Token(SpecTokens.VERSION, value="@2.7:")], "@2.7:"), ("@:2.7", [Token(SpecTokens.VERSION, value="@:2.7")], "@:2.7"), @@ -97,6 +92,14 @@ def _specfile_for(spec_str, filename): "arch=test-None-None", ), # Multiple tokens anonymous specs + ( + "%intel", + [ + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "intel"), + ], + "%intel", + ), ( "languages=go @4.2:", [ @@ -159,7 +162,9 @@ def _specfile_for(spec_str, filename): [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="foo"), Token(SpecTokens.VERSION, value="@2.0"), - Token(SpecTokens.COMPILER_AND_VERSION, value="%bar@1.0"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="bar"), + Token(SpecTokens.VERSION, value="@1.0"), ], "foo@2.0 %bar@1.0", ), @@ -178,7 +183,9 @@ def _specfile_for(spec_str, filename): Token(SpecTokens.VERSION, value="@1.2:1.4,1.6"), Token(SpecTokens.BOOL_VARIANT, value="+debug"), Token(SpecTokens.BOOL_VARIANT, value="~qt_4"), - Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), + Token(SpecTokens.VERSION, value="@12.1"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"), Token(SpecTokens.VERSION, value="@8.1_1e"), @@ -194,7 +201,9 @@ def _specfile_for(spec_str, filename): Token(SpecTokens.VERSION, value="@1.2:1.4,1.6"), Token(SpecTokens.BOOL_VARIANT, value="~qt_4"), Token(SpecTokens.KEY_VALUE_PAIR, value="debug=2"), - Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), + Token(SpecTokens.VERSION, value="@12.1"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"), Token(SpecTokens.VERSION, value="@8.1_1e"), @@ -212,7 +221,9 @@ def _specfile_for(spec_str, filename): Token(SpecTokens.KEY_VALUE_PAIR, value="cppflags=-O3"), Token(SpecTokens.BOOL_VARIANT, value="+debug"), Token(SpecTokens.BOOL_VARIANT, value="~qt_4"), - Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), + Token(SpecTokens.VERSION, value="@12.1"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="stackwalker"), Token(SpecTokens.VERSION, value="@8.1_1e"), @@ -226,7 +237,9 @@ def _specfile_for(spec_str, filename): [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="yaml-cpp"), Token(SpecTokens.VERSION, value="@0.1.8"), - Token(SpecTokens.COMPILER_AND_VERSION, value="%intel@12.1"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), + Token(SpecTokens.VERSION, value="@12.1"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="boost"), Token(SpecTokens.VERSION, value="@3.1.4"), @@ -237,7 +250,8 @@ def _specfile_for(spec_str, filename): r"builtin.yaml-cpp%gcc", [ Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value="builtin.yaml-cpp"), - Token(SpecTokens.COMPILER, value="%gcc"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), ], "yaml-cpp %gcc", ), @@ -245,7 +259,8 @@ def _specfile_for(spec_str, filename): r"testrepo.yaml-cpp%gcc", [ Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value="testrepo.yaml-cpp"), - Token(SpecTokens.COMPILER, value="%gcc"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), ], "yaml-cpp %gcc", ), @@ -254,7 +269,9 @@ def _specfile_for(spec_str, filename): [ Token(SpecTokens.FULLY_QUALIFIED_PACKAGE_NAME, value="builtin.yaml-cpp"), Token(SpecTokens.VERSION, value="@0.1.8"), - Token(SpecTokens.COMPILER_AND_VERSION, value="%gcc@7.2.0"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), + Token(SpecTokens.VERSION, value="@7.2.0"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="boost"), Token(SpecTokens.VERSION, value="@3.1.4"), @@ -419,11 +436,51 @@ def _specfile_for(spec_str, filename): f"develop-branch-version@git.{'a' * 40}=develop+var1+var2", ), # Compiler with version ranges - compiler_with_version_range("%gcc@10.2.1:"), - compiler_with_version_range("%gcc@:10.2.1"), - compiler_with_version_range("%gcc@10.2.1:12.1.0"), - compiler_with_version_range("%gcc@10.1.0,12.2.1:"), - compiler_with_version_range("%gcc@:8.4.3,10.2.1:12.1.0"), + ( + "%gcc@10.2.1:", + [ + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), + Token(SpecTokens.VERSION, value="@10.2.1:"), + ], + "%gcc@10.2.1:", + ), + ( + "%gcc@:10.2.1", + [ + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), + Token(SpecTokens.VERSION, value="@:10.2.1"), + ], + "%gcc@:10.2.1", + ), + ( + "%gcc@10.2.1:12.1.0", + [ + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), + Token(SpecTokens.VERSION, value="@10.2.1:12.1.0"), + ], + "%gcc@10.2.1:12.1.0", + ), + ( + "%gcc@10.1.0,12.2.1:", + [ + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), + Token(SpecTokens.VERSION, value="@10.1.0,12.2.1:"), + ], + "%gcc@10.1.0,12.2.1:", + ), + ( + "%gcc@:8.4.3,10.2.1:12.1.0", + [ + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), + Token(SpecTokens.VERSION, value="@:8.4.3,10.2.1:12.1.0"), + ], + "%gcc@:8.4.3,10.2.1:12.1.0", + ), # Special key value arguments ("dev_path=*", [Token(SpecTokens.KEY_VALUE_PAIR, value="dev_path=*")], "dev_path='*'"), ( @@ -484,7 +541,9 @@ def _specfile_for(spec_str, filename): "+ debug % intel @ 12.1:12.6", [ Token(SpecTokens.BOOL_VARIANT, value="+ debug"), - Token(SpecTokens.COMPILER_AND_VERSION, value="% intel @ 12.1:12.6"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), + Token(SpecTokens.VERSION, value="@ 12.1:12.6"), ], "+debug %intel@12.1:12.6", ), @@ -509,7 +568,8 @@ def _specfile_for(spec_str, filename): "@:0.4 % nvhpc", [ Token(SpecTokens.VERSION, value="@:0.4"), - Token(SpecTokens.COMPILER, value="% nvhpc"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="nvhpc"), ], "@:0.4 %nvhpc", ), @@ -602,7 +662,10 @@ def _specfile_for(spec_str, filename): "zlib %[virtuals=c] gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), - Token(SpecTokens.COMPILER_WITH_VIRTUALS, "%[virtuals=c] gcc"), + Token(SpecTokens.START_EDGE_PROPERTIES, value="%["), + Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=c"), + Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), ], "zlib %[virtuals=c] gcc", ), @@ -610,7 +673,10 @@ def _specfile_for(spec_str, filename): "zlib %[virtuals=c,cxx] gcc", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), - Token(SpecTokens.COMPILER_WITH_VIRTUALS, "%[virtuals=c,cxx] gcc"), + Token(SpecTokens.START_EDGE_PROPERTIES, value="%["), + Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=c,cxx"), + Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), ], "zlib %[virtuals=c,cxx] gcc", ), @@ -618,7 +684,11 @@ def _specfile_for(spec_str, filename): "zlib %[virtuals=c,cxx] gcc@14.1", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), - Token(SpecTokens.COMPILER_AND_VERSION_WITH_VIRTUALS, "%[virtuals=c,cxx] gcc@14.1"), + Token(SpecTokens.START_EDGE_PROPERTIES, value="%["), + Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=c,cxx"), + Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), + Token(SpecTokens.VERSION, value="@14.1"), ], "zlib %[virtuals=c,cxx] gcc@14.1", ), @@ -626,10 +696,15 @@ def _specfile_for(spec_str, filename): "zlib %[virtuals=fortran] gcc@14.1 %[virtuals=c,cxx] clang", [ Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "zlib"), - Token( - SpecTokens.COMPILER_AND_VERSION_WITH_VIRTUALS, "%[virtuals=fortran] gcc@14.1" - ), - Token(SpecTokens.COMPILER_WITH_VIRTUALS, "%[virtuals=c,cxx] clang"), + Token(SpecTokens.START_EDGE_PROPERTIES, value="%["), + Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=fortran"), + Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), + Token(SpecTokens.VERSION, value="@14.1"), + Token(SpecTokens.START_EDGE_PROPERTIES, value="%["), + Token(SpecTokens.KEY_VALUE_PAIR, value="virtuals=c,cxx"), + Token(SpecTokens.END_EDGE_PROPERTIES, value="]"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="clang"), ], "zlib %[virtuals=fortran] gcc@14.1 %[virtuals=c,cxx] clang", ), @@ -650,6 +725,18 @@ def _specfile_for(spec_str, filename): ], "gcc languages:=='c,c++'", ), + # test etc. after % + ( + "mvapich %gcc languages:=c,c++ target=x86_64", + [ + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "mvapich"), + Token(SpecTokens.DEPENDENCY, "%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, "gcc"), + Token(SpecTokens.KEY_VALUE_PAIR, "languages:=c,c++"), + Token(SpecTokens.KEY_VALUE_PAIR, "target=x86_64"), + ], + "mvapich %gcc languages:='c,c++' arch=None-None-x86_64", + ), ], ) def test_parse_single_spec(spec_str, tokens, expected_roundtrip, mock_git_test_package): @@ -694,7 +781,8 @@ def test_parse_single_spec(spec_str, tokens, expected_roundtrip, mock_git_test_p Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="emacs"), Token(SpecTokens.VERSION, value="@1.1.1"), Token(SpecTokens.KEY_VALUE_PAIR, value="cflags=-O3"), - Token(SpecTokens.COMPILER, value="%intel"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), ], ["mvapich", "emacs @1.1.1 cflags=-O3 %intel"], ), @@ -706,10 +794,27 @@ def test_parse_single_spec(spec_str, tokens, expected_roundtrip, mock_git_test_p Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="emacs"), Token(SpecTokens.DEPENDENCY, value="^"), Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="ncurses"), - Token(SpecTokens.COMPILER, value="%intel"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="intel"), ], ['mvapich cflags="-O3 -fPIC"', "emacs ^ncurses%intel"], ), + ( + "mvapich %gcc languages=c,c++ emacs ^ncurses%gcc languages:=c", + [ + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="mvapich"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), + Token(SpecTokens.KEY_VALUE_PAIR, value="languages=c,c++"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="emacs"), + Token(SpecTokens.DEPENDENCY, value="^"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="ncurses"), + Token(SpecTokens.DEPENDENCY, value="%"), + Token(SpecTokens.UNQUALIFIED_PACKAGE_NAME, value="gcc"), + Token(SpecTokens.KEY_VALUE_PAIR, value="languages:=c"), + ], + ["mvapich %gcc languages=c,c++", "emacs ^ncurses%gcc languages:=c"], + ), ], ) def test_parse_multiple_specs(text, tokens, expected_specs): diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index a0640f70c4e..c38356a5bdb 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -2935,7 +2935,7 @@ complete -c spack -n '__fish_spack_using_command style' -s t -l tool -r -d 'spec complete -c spack -n '__fish_spack_using_command style' -s s -l skip -r -f -a skip complete -c spack -n '__fish_spack_using_command style' -s s -l skip -r -d 'specify tools to skip (choose from import, isort, black, flake8, mypy)' complete -c spack -n '__fish_spack_using_command style' -l spec-strings -f -a spec_strings -complete -c spack -n '__fish_spack_using_command style' -l spec-strings -d 'upgrade spec strings in Python, JSON and YAML files for compatibility with Spack v1.0 and v0.x. Example: spack style --spec-strings $(git ls-files). Note: this flag will be removed in Spack v1.0.' +complete -c spack -n '__fish_spack_using_command style' -l spec-strings -d 'upgrade spec strings in Python, JSON and YAML files for compatibility with Spack v1.0 and v0.x. Example: spack style --spec-strings $(git ls-files). Note: must be used only on specs from spack v0.X.' # spack tags set -g __fish_spack_optspecs_spack_tags h/help i/installed a/all diff --git a/var/spack/test_repos/builtin.mock/packages/llvm/package.py b/var/spack/test_repos/builtin.mock/packages/llvm/package.py index 2cb1d2dacc6..6086a94a2d0 100644 --- a/var/spack/test_repos/builtin.mock/packages/llvm/package.py +++ b/var/spack/test_repos/builtin.mock/packages/llvm/package.py @@ -17,6 +17,7 @@ class Llvm(Package, CompilerPackage): variant( "clang", default=True, description="Build the LLVM C/C++/Objective-C compiler frontend" ) + variant("lld", default=True, description="Build the LLVM linker") provides("c", "cxx", when="+clang")