From b37fab40b51eaad8d9a56633904e7821f5244026 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Nov 2022 14:30:45 -0800 Subject: [PATCH] solver: rework optimization criteria with larger constants To allow room for DAG-ordered optimization, rework the way we write optimization criteria in `concretize.lp`, and convert build offsets to use constants. --- lib/spack/spack/cmd/solve.py | 4 +- lib/spack/spack/solver/asp.py | 123 ++++++++++++--------------- lib/spack/spack/solver/concretize.lp | 76 +++++++++-------- lib/spack/spack/test/concretize.py | 2 +- 4 files changed, 100 insertions(+), 105 deletions(-) diff --git a/lib/spack/spack/cmd/solve.py b/lib/spack/spack/cmd/solve.py index 69caeb421f1..1c92591c303 100644 --- a/lib/spack/spack/cmd/solve.py +++ b/lib/spack/spack/cmd/solve.py @@ -112,8 +112,8 @@ def _process_result(result, show, required_format, kwargs): % ( i, name, - "-" if build_cost is None else installed_cost, - installed_cost if build_cost is None else build_cost, + installed_cost if installed_cost is not None else "-", + build_cost if build_cost is not None else "-", ) ) print() diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index c6a8ea0c89b..74e3a538775 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -13,6 +13,7 @@ import re import types import warnings +from typing import Tuple import archspec.cpu @@ -124,84 +125,72 @@ def getter(node): # The space of possible priorities for optimization targets # is partitioned in the following ranges: -# -# [0-100) Optimization criteria for software being reused -# [100-200) Fixed criteria that are higher priority than reuse, but lower than build -# [200-300) Optimization criteria for software being built -# [300-1000) High-priority fixed criteria -# [1000-inf) Error conditions +# +=============================================================+ +# | Priority | Description | +# +=============================================================+ +# | 10,000,000+ | Error conditions | +# +-------------+-----------------------------------------------+ +# | 9,999,999 | | +# | ... | High-priority criteria | +# | 1,000,000 | | +# +-------------+-----------------------------------------------+ +# | 999,999 | | +# | ... | Standard criteria for built packages | +# | 100,001 | | +# +-------------+-----------------------------------------------+ +# | 100,000 | Number of packages being built | +# +-------------+-----------------------------------------------+ +# | 99,999 | | +# | ... | Standard criteria for reused packages | +# | 0 | | +# +-------------+-----------------------------------------------+ # # Each optimization target is a minimization with optimal value 0. - +# #: High fixed priority offset for criteria that supersede all build criteria -high_fixed_priority_offset = 300 +high_fixed_priority_offset = 10_000_000 #: Priority offset for "build" criteria (regular criterio shifted to #: higher priority for specs we have to build) -build_priority_offset = 200 - -#: Priority offset of "fixed" criteria (those w/o build criteria) -fixed_priority_offset = 100 +build_priority_offset = 100_000 -def build_criteria_names(costs, arg_tuples): +def build_criteria_names(costs, opt_criteria): """Construct an ordered mapping from criteria names to costs.""" - # pull optimization criteria names out of the solution - priorities_names = [] - num_fixed = 0 - num_high_fixed = 0 - for args in arg_tuples: - priority, name = args[:2] + # ensure names of all criteria are unique + names = {name for _, name in opt_criteria} + assert len(names) == len(opt_criteria), "names of optimization criteria must be unique" + + # costs contains: + # - error criteria + # - number of input specs not concretized + # - N build criteria + # ... + # - number of packages to build + # - N reuse criteria + # ... + + # opt_criteria has all the named criteria, which is all but the errors. + # So we can figure out how many build criteria there are up front. + n_build_criteria = len(opt_criteria) - 2 + + # number of criteria *not* including errors + n_named_criteria = len(opt_criteria) + n_build_criteria + + # opt_criteria are in order, highest to lowest, as written in concretize.lp + # put costs in the same order as opt criteria + start = len(costs) - n_named_criteria + ordered_costs = costs[start:] + ordered_costs.insert(1, ordered_costs.pop(n_build_criteria + 1)) + + # list of build cost, reuse cost, and name of each criterion + criteria: List[Tuple[int, int, str]] = [] + for i, (priority, name) in enumerate(opt_criteria): priority = int(priority) - - # add the priority of this opt criterion and its name - priorities_names.append((priority, name)) - - # if the priority is less than fixed_priority_offset, then it - # has an associated build priority -- the same criterion but for - # nodes that we have to build. - if priority < fixed_priority_offset: - build_priority = priority + build_priority_offset - priorities_names.append((build_priority, name)) - elif priority >= high_fixed_priority_offset: - num_high_fixed += 1 - else: - num_fixed += 1 - - # sort the criteria by priority - priorities_names = sorted(priorities_names, reverse=True) - - # We only have opt-criterion values for non-error types - # error type criteria are excluded (they come first) - error_criteria = len(costs) - len(priorities_names) - costs = costs[error_criteria:] - - # split list into three parts: build criteria, fixed criteria, non-build criteria - num_criteria = len(priorities_names) - num_build = (num_criteria - num_fixed - num_high_fixed) // 2 - - build_start_idx = num_high_fixed - fixed_start_idx = num_high_fixed + num_build - installed_start_idx = num_high_fixed + num_build + num_fixed - - high_fixed = priorities_names[:build_start_idx] - build = priorities_names[build_start_idx:fixed_start_idx] - fixed = priorities_names[fixed_start_idx:installed_start_idx] - installed = priorities_names[installed_start_idx:] - - # mapping from priority to index in cost list - indices = dict((p, i) for i, (p, n) in enumerate(priorities_names)) - - # make a list that has each name with its build and non-build costs - criteria = [(cost, None, name) for cost, (p, name) in zip(costs[:build_start_idx], high_fixed)] - criteria += [ - (cost, None, name) - for cost, (p, name) in zip(costs[fixed_start_idx:installed_start_idx], fixed) - ] - - for (i, name), (b, _) in zip(installed, build): - criteria.append((costs[indices[i]], costs[indices[b]], name)) + build_cost = ordered_costs[i] + reuse_cost = ordered_costs[i + n_build_criteria] if priority < 100_000 else None + criteria.append((reuse_cost, build_cost, name)) return criteria diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp index 40a8c0137cb..ca17a3109c0 100644 --- a/lib/spack/spack/solver/concretize.lp +++ b/lib/spack/spack/solver/concretize.lp @@ -23,9 +23,14 @@ literal_not_solved(ID) :- not literal_solved(ID), literal(ID). % in better reporting for users. See #30669 for details. 1 { literal_solved(ID) : literal(ID) }. -opt_criterion(300, "number of input specs not concretized"). -#minimize{ 0@300: #true }. -#minimize { 1@300,ID : literal_not_solved(ID) }. +% priority ranges for optimization criteria +#const error_prio = 10000000. +#const solve_prio = 1000000. +#const build_prio = 100000. + +opt_criterion(solve_prio, "number of input specs not concretized"). +#minimize{ 0@solve_prio: #true }. +#minimize{ 1@solve_prio,ID : literal_not_solved(ID) }. % Map constraint on the literal ID to the correct PSID attr(Name, A1) :- literal(LiteralID, Name, A1), literal_solved(LiteralID). @@ -1077,8 +1082,8 @@ build(Package) :- not attr("hash", Package, _), attr("node", Package). % 200+ Shifted priorities for build nodes; correspond to priorities 0 - 99. % 100 - 199 Unshifted priorities. Currently only includes minimizing #builds. % 0 - 99 Priorities for non-built nodes. -build_priority(Package, 200) :- build(Package), attr("node", Package), optimize_for_reuse(). -build_priority(Package, 0) :- not build(Package), attr("node", Package), optimize_for_reuse(). +build_priority(Package, build_prio) :- build(Package), attr("node", Package), optimize_for_reuse(). +build_priority(Package, 0) :- not build(Package), attr("node", Package), optimize_for_reuse(). % don't adjust build priorities if reuse is not enabled build_priority(Package, 0) :- attr("node", Package), not optimize_for_reuse(). @@ -1107,16 +1112,17 @@ build_priority(Package, 0) :- attr("node", Package), not optimize_for_reuse(). % Some errors are handled as rules instead of constraints because % it allows us to explain why something failed. Here we optimize % HEAVILY against the facts generated by those rules. -#minimize{ 0@1000: #true}. -#minimize{ 0@1001: #true}. -#minimize{ 0@1002: #true}. -#minimize{ 1000@1000+Priority,Msg: error(Priority, Msg) }. -#minimize{ 1000@1000+Priority,Msg,Arg1: error(Priority, Msg, Arg1) }. -#minimize{ 1000@1000+Priority,Msg,Arg1,Arg2: error(Priority, Msg, Arg1, Arg2) }. -#minimize{ 1000@1000+Priority,Msg,Arg1,Arg2,Arg3: error(Priority, Msg, Arg1, Arg2, Arg3) }. -#minimize{ 1000@1000+Priority,Msg,Arg1,Arg2,Arg3,Arg4: error(Priority, Msg, Arg1, Arg2, Arg3, Arg4) }. -#minimize{ 1000@1000+Priority,Msg,Arg1,Arg2,Arg3,Arg4,Arg5: error(Priority, Msg, Arg1, Arg2, Arg3, Arg4, Arg5) }. +#minimize{ 0@error_prio: #true}. +#minimize{ 0@error_prio: #true}. +#minimize{ 0@error_prio: #true}. + +#minimize{ 1000@error_prio+Priority,Msg: error(Priority, Msg) }. +#minimize{ 1000@error_prio+Priority,Msg,Arg1: error(Priority, Msg, Arg1) }. +#minimize{ 1000@error_prio+Priority,Msg,Arg1,Arg2: error(Priority, Msg, Arg1, Arg2) }. +#minimize{ 1000@error_prio+Priority,Msg,Arg1,Arg2,Arg3: error(Priority, Msg, Arg1, Arg2, Arg3) }. +#minimize{ 1000@error_prio+Priority,Msg,Arg1,Arg2,Arg3,Arg4: error(Priority, Msg, Arg1, Arg2, Arg3, Arg4) }. +#minimize{ 1000@error_prio+Priority,Msg,Arg1,Arg2,Arg3,Arg4,Arg5: error(Priority, Msg, Arg1, Arg2, Arg3, Arg4, Arg5) }. %----------------------------------------------------------------------------- % How to optimize the spec (high to low priority) @@ -1127,16 +1133,16 @@ build_priority(Package, 0) :- attr("node", Package), not optimize_for_reuse(). % is displayed (clingo doesn't display sums over empty sets by default) % Try hard to reuse installed packages (i.e., minimize the number built) -opt_criterion(100, "number of packages to build (vs. reuse)"). -#minimize { 0@100: #true }. -#minimize { 1@100,Package : build(Package), optimize_for_reuse() }. +opt_criterion(build_prio, "number of packages to build (vs. reuse)"). +#minimize { 0@build_prio: #true }. +#minimize { 1@build_prio,Package : build(Package), optimize_for_reuse() }. #defined optimize_for_reuse/0. % A condition group specifies one or more specs that must be satisfied. % Specs declared first are preferred, so we assign increasing weights and % minimize the weights. opt_criterion(75, "requirement weight"). -#minimize{ 0@275: #true }. +#minimize{ 0@75+build_prio: #true }. #minimize{ 0@75: #true }. #minimize { Weight@75+Priority @@ -1146,7 +1152,7 @@ opt_criterion(75, "requirement weight"). % Minimize the number of deprecated versions being used opt_criterion(73, "deprecated versions used"). -#minimize{ 0@273: #true }. +#minimize{ 0@73+build_prio: #true }. #minimize{ 0@73: #true }. #minimize{ 1@73+Priority,Package @@ -1159,7 +1165,7 @@ opt_criterion(73, "deprecated versions used"). % 2. Number of variants with a non default value, if not set % for the root package. opt_criterion(70, "version weight"). -#minimize{ 0@270: #true }. +#minimize{ 0@70+build_prio: #true }. #minimize{ 0@70: #true }. #minimize { Weight@70+Priority @@ -1168,7 +1174,7 @@ opt_criterion(70, "version weight"). }. opt_criterion(65, "number of non-default variants (roots)"). -#minimize{ 0@265: #true }. +#minimize{ 0@65+build_prio: #true }. #minimize{ 0@65: #true }. #minimize { 1@65+Priority,Package,Variant,Value @@ -1178,7 +1184,7 @@ opt_criterion(65, "number of non-default variants (roots)"). }. opt_criterion(60, "preferred providers for roots"). -#minimize{ 0@260: #true }. +#minimize{ 0@60+build_prio: #true }. #minimize{ 0@60: #true }. #minimize{ Weight@60+Priority,Provider,Virtual @@ -1188,7 +1194,7 @@ opt_criterion(60, "preferred providers for roots"). }. opt_criterion(55, "default values of variants not being used (roots)"). -#minimize{ 0@255: #true }. +#minimize{ 0@55+build_prio: #true }. #minimize{ 0@55: #true }. #minimize{ 1@55+Priority,Package,Variant,Value @@ -1199,7 +1205,7 @@ opt_criterion(55, "default values of variants not being used (roots)"). % Try to use default variants or variants that have been set opt_criterion(50, "number of non-default variants (non-roots)"). -#minimize{ 0@250: #true }. +#minimize{ 0@50+build_prio: #true }. #minimize{ 0@50: #true }. #minimize { 1@50+Priority,Package,Variant,Value @@ -1211,7 +1217,7 @@ opt_criterion(50, "number of non-default variants (non-roots)"). % Minimize the weights of the providers, i.e. use as much as % possible the most preferred providers opt_criterion(45, "preferred providers (non-roots)"). -#minimize{ 0@245: #true }. +#minimize{ 0@45+build_prio: #true }. #minimize{ 0@45: #true }. #minimize{ Weight@45+Priority,Provider,Virtual @@ -1221,7 +1227,7 @@ opt_criterion(45, "preferred providers (non-roots)"). % Try to minimize the number of compiler mismatches in the DAG. opt_criterion(40, "compiler mismatches that are not from CLI"). -#minimize{ 0@240: #true }. +#minimize{ 0@40+build_prio: #true }. #minimize{ 0@40: #true }. #minimize{ 1@40+Priority,Package,Dependency @@ -1229,8 +1235,8 @@ opt_criterion(40, "compiler mismatches that are not from CLI"). build_priority(Package, Priority) }. -opt_criterion(39, "compiler mismatches that are not from CLI"). -#minimize{ 0@239: #true }. +opt_criterion(39, "compiler mismatches from CLI"). +#minimize{ 0@39+build_prio: #true }. #minimize{ 0@39: #true }. #minimize{ 1@39+Priority,Package,Dependency @@ -1240,7 +1246,7 @@ opt_criterion(39, "compiler mismatches that are not from CLI"). % Try to minimize the number of compiler mismatches in the DAG. opt_criterion(35, "OS mismatches"). -#minimize{ 0@235: #true }. +#minimize{ 0@35+build_prio: #true }. #minimize{ 0@35: #true }. #minimize{ 1@35+Priority,Package,Dependency @@ -1249,7 +1255,7 @@ opt_criterion(35, "OS mismatches"). }. opt_criterion(30, "non-preferred OS's"). -#minimize{ 0@230: #true }. +#minimize{ 0@30+build_prio: #true }. #minimize{ 0@30: #true }. #minimize{ Weight@30+Priority,Package @@ -1259,7 +1265,7 @@ opt_criterion(30, "non-preferred OS's"). % Choose more recent versions for nodes opt_criterion(25, "version badness"). -#minimize{ 0@225: #true }. +#minimize{ 0@25+build_prio: #true }. #minimize{ 0@25: #true }. #minimize{ Weight@25+Priority,Package @@ -1269,7 +1275,7 @@ opt_criterion(25, "version badness"). % Try to use all the default values of variants opt_criterion(20, "default values of variants not being used (non-roots)"). -#minimize{ 0@220: #true }. +#minimize{ 0@20+build_prio: #true }. #minimize{ 0@20: #true }. #minimize{ 1@20+Priority,Package,Variant,Value @@ -1280,7 +1286,7 @@ opt_criterion(20, "default values of variants not being used (non-roots)"). % Try to use preferred compilers opt_criterion(15, "non-preferred compilers"). -#minimize{ 0@215: #true }. +#minimize{ 0@15+build_prio: #true }. #minimize{ 0@15: #true }. #minimize{ Weight@15+Priority,Package @@ -1291,7 +1297,7 @@ opt_criterion(15, "non-preferred compilers"). % Minimize the number of mismatches for targets in the DAG, try % to select the preferred target. opt_criterion(10, "target mismatches"). -#minimize{ 0@210: #true }. +#minimize{ 0@10+build_prio: #true }. #minimize{ 0@10: #true }. #minimize{ 1@10+Priority,Package,Dependency @@ -1300,7 +1306,7 @@ opt_criterion(10, "target mismatches"). }. opt_criterion(5, "non-preferred targets"). -#minimize{ 0@205: #true }. +#minimize{ 0@5+build_prio: #true }. #minimize{ 0@5: #true }. #minimize{ Weight@5+Priority,Package diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 3e816dc63ec..a2d792d9998 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -1779,7 +1779,7 @@ def test_version_weight_and_provenance(self): num_specs = len(list(result_spec.traverse())) criteria = [ - (num_specs - 1, None, "number of packages to build (vs. reuse)"), + (None, num_specs - 1, "number of packages to build (vs. reuse)"), (2, 0, "version badness"), ]