From 343580600773eea36949fb2514001c8d040c434f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Jan 2023 09:48:53 -0800 Subject: [PATCH] WIP --- lib/spack/spack/cmd/solve.py | 39 +++-- lib/spack/spack/solver/asp.py | 91 +++++------- lib/spack/spack/solver/concretize.lp | 208 ++++++++++++++------------- lib/spack/spack/solver/display.lp | 8 +- 4 files changed, 173 insertions(+), 173 deletions(-) diff --git a/lib/spack/spack/cmd/solve.py b/lib/spack/spack/cmd/solve.py index accdeb6d05d..4448f3facc7 100644 --- a/lib/spack/spack/cmd/solve.py +++ b/lib/spack/spack/cmd/solve.py @@ -98,27 +98,40 @@ def setup_parser(subparser): def _process_result(result, show, required_format, kwargs): result.raise_if_unsat() opt, *_ = min(result.answers) + + # dump the solutions as concretized specs if ("opt" in show) and (not required_format): tty.msg("Best of %d considered solutions." % result.nmodels) tty.msg("Optimization Criteria:") - maxlen = max(len(s[2]) for s in result.criteria) - color.cprint("@*{ Priority Criterion %sInstalled ToBuild}" % ((maxlen - 10) * " ")) + maxlen = max(len(name) for name in result.criteria) + max_depth = max(len(v) for v in result.criteria.values() if isinstance(v, list)) - fmt = " @K{%%-8d} %%-%ds%%9s %%7s" % maxlen - for i, (installed_cost, build_cost, name) in enumerate(result.criteria, 1): - color.cprint( - fmt - % ( - i, - name, - installed_cost if installed_cost is not None else "-", - build_cost if build_cost is not None else "-", + header = "@*{" + header += "".join(f"{depth:<4}" for depth in range(max_depth)) + header += "Criterion}" + color.cprint(header) + + # make non-zero numbers red + def highlight(n, c): + return color.colorize(f"@{c}{{{n:<4}}}" if n > 0 else f"{n:<4}") + + for i, (name, cost) in enumerate(result.criteria.items(), 1): + colored_name = name.replace("build:", "@c{build:}") + colored_name = colored_name.replace("reuse:", "@B{reuse:}") + colored_name = colored_name.replace("fixed:", "@G{fixed:}") + colored_name = color.colorize(colored_name) + + if isinstance(cost, int): + print(highlight(cost, "G") + " " * (max_depth - 1) + colored_name) + else: + print( + "".join(highlight(c, "c" if "build:" in name else "B") for c in cost) + + colored_name ) - ) + print() - # dump the solutions as concretized specs if "solutions" in show: for spec in result.specs: # With -y, just print YAML to output. diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index 8175b3ac0c8..c6db80e878b 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -154,72 +154,54 @@ def getter(node): #: higher priority for specs we have to build) build_priority_offset = 100_000 +#: max priority for an error +max_error_priority = 3 + def build_criteria_names( - costs: List[int], opt_criteria: List["AspFunction"], max_depth: int -) -> Dict[str, Union[int, List[int]]]: + costs: List[int], opt_criteria: List["AspFunction"], max_depth: int, const_max_depth: int +) -> Dict[str, Union[int, List[Tuple[int, int]]]]: """Construct an ordered mapping from criteria names to costs.""" # ensure names of all criteria are unique names = {criterion.args[0] for criterion in opt_criteria} assert len(names) == len(opt_criteria), "names of optimization criteria must be unique" - # costs contains: - # - error criteria - # - number of input specs not concretized - # - N build criteria - # ... - # - number of packages to build - # - N reuse criteria - # ... - # split opt criteria into two lists fixed_criteria = [oc for oc in opt_criteria if oc.args[1] == "fixed"] leveled_criteria = [oc for oc in opt_criteria if oc.args[1] == "leveled"] - print("FIXED:", len(fixed_criteria)) - print("LEVELED:", len(leveled_criteria)) + # first non-error criterion + solve_index = max_error_priority + 1 - n_depths = max_depth + 1 # depths start at zero - total_criteria = len(fixed_criteria) + len(leveled_criteria) * 2 * n_depths - print(total_criteria) + # each criterion is aggregated for each level in the graph + max_leveled_costs = len(leveled_criteria) * const_max_depth + n_leveled_costs = len(leveled_criteria) * (max_depth + 1) - # opt_criteria has all the named criteria, which is all but the errors. - # So we can figure out how many build criteria there are up front. - # the -2 here accounts for: - # - number of specs not concretized - # - number of packages to build (vs. reuse) - n_leveled_criteria = len(opt_criteria) - 2 + build_index = solve_index + 1 + max_leveled_costs + fixed_costs = [costs[solve_index], costs[build_index]] - n_leveled_criteria = n_build_criteria * 2 * n_depths - total_criteria = n_leveled_criteria + 2 - - assert len(costs) == total_criteria - - # For each level, opt_criteria are in order, highest to lowest, as written in - # concretize.lp put costs in the same order as opt criteria - start = len(costs) - n_named_criteria - - # build criteria count should be divisible by n_depths or we're doing something wrong. - assert n_build_criteria % n_depths == 0 + build_costs = costs[solve_index + 1 : solve_index + 1 + n_leveled_costs] + reuse_costs = costs[build_index + 1 : build_index + 1 + n_leveled_costs] + assert len(build_costs) == len(reuse_costs) == n_leveled_costs criteria = {} - ordered_costs = costs[start:] - ordered_costs.insert(1, ordered_costs.pop(n_build_criteria + 1)) - for i, (priority, type, name) in enumerate(c.args for c in opt_criteria): - if type == "fixed": - criteria[name] = ordered_costs[i] - else: - criteria[name] = ordered_costs[i::n_build_criteria] + def add_fixed(criterion_idx, cost_idx): + name = fixed_criteria[criterion_idx].args[2] + criteria["fixed: " + name] = costs[cost_idx] - # list of build cost, reuse cost, and name of each criterion - criteria: List[Tuple[int, int, str]] = [] - for i, (priority, name) in enumerate(c.args for c in opt_criteria): - priority = int(priority) - build_cost = ordered_costs[i] - reuse_cost = ordered_costs[i + n_build_criteria] if priority < 100_000 else None - criteria.append((reuse_cost, build_cost, name)) + add_fixed(0, solve_index) + + for i, fn in enumerate(leveled_criteria): + name = fn.args[2] + criteria["build: " + name] = build_costs[i :: len(leveled_criteria)] + + add_fixed(1, build_index) + + for i, fn in enumerate(leveled_criteria): + name = fn.args[2] + criteria["reuse: " + name] = reuse_costs[i :: len(leveled_criteria)] return criteria @@ -803,7 +785,7 @@ def on_model(model): # build specs from spec attributes in the model spec_attrs = extract_functions(best_model, "attr") - with timer.measure("build_specs"): + with timer.measure("build"): answers = builder.build_specs(spec_attrs) # add best spec to the results @@ -812,16 +794,11 @@ def on_model(model): # get optimization criteria criteria = extract_functions(best_model, "opt_criterion") depths = extract_functions(best_model, "depth") - print(depths) + const_max_depth, *_ = extract_functions(best_model, "const_max_depth") + const_max_depth = const_max_depth.args[0] max_depth = max(d.args[1] for d in depths) - print(f"COST: {len(min_cost)} {min_cost}") - print(f"PRIO: {len(priorities)} {priorities}") - print(f"MAX_DEPTH: {max_depth}") - print() - print(opt_criteria) - - result.criteria = build_criteria_names(min_cost, criteria, max_depth) + result.criteria = build_criteria_names(min_cost, criteria, max_depth, const_max_depth) # record the number of models the solver considered result.nmodels = len(models) @@ -831,7 +808,7 @@ def on_model(model): # print any unknown functions in the model for sym in best_model: - if sym.name not in ("attr", "error", "opt_criterion"): + if sym.name not in ("attr", "error", "opt_criterion", "depth", "const_max_depth"): tty.debug( "UNKNOWN SYMBOL: %s(%s)" % (sym.name, ", ".join(stringify(sym.arguments))) ) diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp index 00d4d4e37ec..ac1c94ab337 100644 --- a/lib/spack/spack/solver/concretize.lp +++ b/lib/spack/spack/solver/concretize.lp @@ -25,6 +25,7 @@ literal_not_solved(ID) :- not literal_solved(ID), literal(ID). % priority ranges for optimization criteria % note that clingo's weight_t is int32_t, so the max priority we can use is 2,147,483,647 +#const max_error_priority = 3. #const error_prio = 10000000. #const solve_prio = 1000000. #const build_prio = 100000. % n_nodes x depth_offset x max levels needs to be less than this @@ -72,7 +73,8 @@ version_declared(Package, Version, Weight) :- version_declared(Package, Version, version_declared(Package, Version) :- version_declared(Package, Version, _). % a spec with a git hash version is equivalent to one with the same matched version -version_satisfies(Package, Constraint, HashVersion) :- version_satisfies(Package, Constraint, EquivalentVersion), +version_satisfies(Package, Constraint, HashVersion) :- + version_satisfies(Package, Constraint, EquivalentVersion), version_equivalent(Package, HashVersion, EquivalentVersion). #defined version_equivalent/3. @@ -153,7 +155,10 @@ possible_version_weight(Package, Weight) % Otherwise covered by `no_version_error` and `versions_conflict_error`. error(1, "No valid version for '{0}' satisfies '@{1}'", Package, Constraint) :- attr("node_version_satisfies", Package, Constraint), - C = #count{ Version : attr("version", Package, Version), version_satisfies(Package, Constraint, Version)}, + C = #count{ + Version + : attr("version", Package, Version), version_satisfies(Package, Constraint, Version) + }, C < 1. attr("node_version_satisfies", Package, Constraint) @@ -1088,7 +1093,7 @@ build_priority(Package, build_prio) :- build(Package), attr("node", Package), op build_priority(Package, 0) :- not build(Package), attr("node", Package), optimize_for_reuse(). % don't adjust build priorities if reuse is not enabled -build_priority(Package, 0) :- attr("node", Package), not optimize_for_reuse(). +build_priority(Package, build_prio) :- attr("node", Package), not optimize_for_reuse(). % don't assign versions from installed packages unless reuse is enabled % NOTE: that "installed" means the declared version was only included because @@ -1112,7 +1117,8 @@ build_priority(Package, 0) :- attr("node", Package), not optimize_for_reuse(). % Calculate min depth of nodes in the DAG % We use this to optimize nodes closer to roots with higher precedence. %----------------------------------------------------------------------------- -#const max_depth = 10. +#const max_depth = 5. +const_max_depth(max_depth). % roots have depth 0. depth(Package, 0) :- attr("root", Package). @@ -1122,22 +1128,20 @@ min_parent_depth(Package, N) :- N = #min{ D: depends_on(Dependent, Package), depth(Dependent, D), - 0 <= D, - D < max_depth + D = 0..max_depth - 1 }, - 0 <= N, - N < max_depth, + N = 0..max_depth - 1, attr("node", Package). -depth(Package, max_depth) :- +depth(Package, max_depth - 1) :- min_parent_depth(Package, P), - P >= max_depth, + P >= max_depth - 1, attr("node", Package). depth(Package, P+1) :- min_parent_depth(Package, P), P >= 0, - P < max_depth, + P < max_depth - 1, attr("node", Package). @@ -1148,10 +1152,10 @@ depth(Package, P+1) :- % it allows us to explain why something failed. Here we optimize % HEAVILY against the facts generated by those rules. -#minimize{ 0@error_prio: #true}. -#minimize{ 0@error_prio: #true}. -#minimize{ 0@error_prio: #true}. +% ensure that error costs are always in the solution. +#minimize{ 0@error_prio + (0..max_error_priority): #true}. +% TODO: why 1000 and not just 1? 1000 seems unnecessary since priorities are lexicographic. #minimize{ 1000@error_prio+Priority,Msg: error(Priority, Msg) }. #minimize{ 1000@error_prio+Priority,Msg,Arg1: error(Priority, Msg, Arg1) }. #minimize{ 1000@error_prio+Priority,Msg,Arg1,Arg2: error(Priority, Msg, Arg1, Arg2) }. @@ -1177,17 +1181,18 @@ depth(Package, P+1) :- #minimize{ 0@N: opt_criterion(N, "fixed", _) }. % "leveled" criteria sum into a bucket per depth in the graph, per build priority -%#minimize{ -% @(((max_depth - D - 1) * depth_offset) + N + build_prio) -% : opt_criterion(N, "leveled", _), depth(_, D) -%}. +#minimize{ + 0@(((max_depth - D - 1) * depth_offset) + N + build_prio) + : opt_criterion(N, "leveled", _), D = 0..max_depth - 1 +}. #minimize{ 0@(((max_depth - D - 1) * depth_offset) + N) - : opt_criterion(N, "leveled", _), depth(_, D) + : opt_criterion(N, "leveled", _), D = 0..max_depth - 1 }. % Try hard to reuse installed packages (i.e., minimize the number built) opt_criterion(build_prio, "fixed", "number of packages to build (vs. reuse)"). +#minimize { 1@build_prio,Package : build(Package), optimize_for_reuse() }. #defined optimize_for_reuse/0. @@ -1195,118 +1200,118 @@ opt_criterion(build_prio, "fixed", "number of packages to build (vs. reuse)"). % Specs declared first are preferred, so we assign increasing weights and % minimize the weights. opt_criterion(65, "leveled", "requirement weight"). -%#minimize { -% Weight@(65 + ((max_depth - D - 1) * depth_offset) + Priority), Package -% : requirement_weight(Package, Weight), -% build_priority(Package, Priority), -% depth(Package, D) -%}. +#minimize{ + Weight@(65 + ((max_depth - D - 1) * depth_offset) + Priority), Package + : requirement_weight(Package, Weight), + build_priority(Package, Priority), + depth(Package, D) +}. % Minimize the number of deprecated versions being used opt_criterion(60, "leveled", "deprecated versions used"). -%#minimize{ -% 1@(60 + ((max_depth - D - 1) * depth_offset) + Priority), Package -% : attr("deprecated", Package, _), -% build_priority(Package, Priority), -% depth(Package, D) -%}. +#minimize{ + 1@(60 + ((max_depth - D - 1) * depth_offset) + Priority), Package + : attr("deprecated", Package, _), + build_priority(Package, Priority), + depth(Package, D) +}. % Minimize the: % 1. Version weight % 2. Number of variants with a non default value, if not set % for the root package. opt_criterion(55, "leveled", "version badness"). -%#minimize { -% Weight@(55 + ((max_depth - D - 1) * depth_offset) + Priority), Package -% : version_weight(Package, Weight), -% build_priority(Package, Priority), -% depth(Package, D) -%}. +#minimize{ + Weight@(55 + ((max_depth - D - 1) * depth_offset) + Priority), Package + : version_weight(Package, Weight), + build_priority(Package, Priority), + depth(Package, D) +}. opt_criterion(50, "leveled", "number of non-default variants"). -%#minimize { -% 1@(50 + ((max_depth - D - 1) * depth_offset) + Priority), Package, Variant, Value -% : variant_not_default(Package, Variant, Value), -% build_priority(Package, Priority), -% depth(Package, D) -%}. +#minimize{ + 1@(50 + ((max_depth - D - 1) * depth_offset) + Priority), Package, Variant, Value + : variant_not_default(Package, Variant, Value), + build_priority(Package, Priority), + depth(Package, D) +}. opt_criterion(45, "leveled", "preferred providers"). -%#minimize{ -% Weight@(45 + ((max_depth - D - 1) * depth_offset) + Priority), Provider, Virtual -% : provider_weight(Provider, Virtual, Weight), -% build_priority(Provider, Priority), -% depth(Package, D) -%}. +#minimize{ + Weight@(45 + ((max_depth - D - 1) * depth_offset) + Priority), Provider, Virtual + : provider_weight(Provider, Virtual, Weight), + build_priority(Provider, Priority), + depth(Package, D) +}. opt_criterion(40, "leveled", "default values of variants not being used"). -%#minimize{ -% 1@(40 + ((max_depth - D - 1) * depth_offset) + Priority), Package, Variant, Value -% : variant_default_not_used(Package, Variant, Value), -% build_priority(Package, Priority), -% depth(Package, D) -%}. +#minimize{ + 1@(40 + ((max_depth - D - 1) * depth_offset) + Priority), Package, Variant, Value + : variant_default_not_used(Package, Variant, Value), + build_priority(Package, Priority), + depth(Package, D) +}. % Try to use default variants or variants that have been set opt_criterion(35, "leveled", "compiler mismatches (not from CLI)"). -%#minimize{ -% 1@(35 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package -% : compiler_mismatch(Dependent, Package), -% build_priority(Package, Priority), -% depth(Package, D) -%}. +#minimize{ + 1@(35 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package + : compiler_mismatch(Dependent, Package), + build_priority(Package, Priority), + depth(Package, D) +}. opt_criterion(30, "leveled", "compiler mismatches (from CLI)"). -%#minimize{ -% 1@(30 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package -% : compiler_mismatch_required(Dependent, Package), -% build_priority(Package, Priority), -% depth(Package, D) -%}. +#minimize{ + 1@(30 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package + : compiler_mismatch_required(Dependent, Package), + build_priority(Package, Priority), + depth(Package, D) +}. % Try to minimize the number of compiler mismatches in the DAG. opt_criterion(25, "leveled", "OS mismatches"). -%#minimize{ -% 1@(25 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package -% : node_os_mismatch(Dependent, Package), -% build_priority(Package, Priority), -% depth(Package, D) -%}. +#minimize{ + 1@(25 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package + : node_os_mismatch(Dependent, Package), + build_priority(Package, Priority), + depth(Package, D) +}. % Try to use preferred compilers opt_criterion(20, "leveled", "non-preferred compilers"). -%#minimize{ -% Weight@(20 + ((max_depth - D - 1) * depth_offset) + Priority), Package -% : compiler_weight(Package, Weight), -% build_priority(Package, Priority), -% depth(Package, D) -%}. +#minimize{ + Weight@(20 + ((max_depth - D - 1) * depth_offset) + Priority), Package + : compiler_weight(Package, Weight), + build_priority(Package, Priority), + depth(Package, D) +}. opt_criterion(15, "leveled", "non-preferred OS's"). -%#minimize{ -% Weight@(15 + ((max_depth - D - 1) * depth_offset) + Priority), Package -% : node_os_weight(Package, Weight), -% build_priority(Package, Priority), -% depth(Package, D) -%}. +#minimize{ + Weight@(15 + ((max_depth - D - 1) * depth_offset) + Priority), Package + : node_os_weight(Package, Weight), + build_priority(Package, Priority), + depth(Package, D) +}. % Minimize the number of mismatches for targets in the DAG, try % to select the preferred target. opt_criterion(10, "leveled", "target mismatches"). -%#minimize{ -% 1@(10 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package -% : node_target_mismatch(Dependent, Package), -% build_priority(Package, Priority), -% depth(Package, D) -%}. +#minimize{ + 1@(10 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package + : node_target_mismatch(Dependent, Package), + build_priority(Package, Priority), + depth(Package, D) +}. opt_criterion(5, "leveled", "non-preferred targets"). -%#minimize{ -% Weight@(5 + ((max_depth - D - 1) * depth_offset) + Priority), Package -% : node_target_weight(Package, Weight), -% build_priority(Package, Priority), -% depth(Package, D) -%}. +#minimize{ + Weight@(5 + ((max_depth - D - 1) * depth_offset) + Priority), Package + : node_target_weight(Package, Weight), + build_priority(Package, Priority), + depth(Package, D) +}. %----------------- % Domain heuristic @@ -1316,9 +1321,12 @@ opt_criterion(5, "leveled", "non-preferred targets"). #heuristic attr("node_target", Package, Target) : package_target_weight(Target, Package, 0), attr("node", Package). [10, true] #heuristic node_target_weight(Package, 0) : attr("node", Package). [10, true] #heuristic attr("variant_value", Package, Variant, Value) : variant_default_value(Package, Variant, Value), attr("node", Package). [10, true] -#heuristic provider(Package, Virtual) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true] -#heuristic attr("node", Package) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true] -#heuristic attr("node_os", Package, OS) : buildable_os(OS). [10, true] + +%#heuristic provider(Package, Virtual) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true] +%#heuristic attr("node", Package) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true] +%#heuristic attr("node_os", Package, OS) : buildable_os(OS). [10, true] +%#heuristic attr("node_target", Dependency, Target): depends_on(Package, Dependency), attr("node_target", Package, Target). [20, true] +%#heuristic attr("node_os", Dependency, OS): depends_on(Package, Dependency), attr("node_os", Package, OS). [20, true] %----------- % Notes diff --git a/lib/spack/spack/solver/display.lp b/lib/spack/spack/solver/display.lp index c09394951ab..7764c214f76 100644 --- a/lib/spack/spack/solver/display.lp +++ b/lib/spack/spack/solver/display.lp @@ -25,8 +25,12 @@ #show error/6. #show error/7. -% debug +% depths #show depth/2. +#show const_max_depth/1. + +% debug + %#show depends_on/2. %node(Package) :- attr("node", Package). @@ -34,5 +38,3 @@ %version(Package, Version) :- attr("version", Package, Version). %#show version/2. - -#show impose/1.