zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

solver: rework optimization criteria with larger constants

Browse Source 
To allow room for DAG-ordered optimization, rework the way we write optimization
criteria in `concretize.lp`, and convert build offsets to use constants.
This commit is contained in:
Todd Gamblin 2022-11-22 14:30:45 -08:00
parent f3e7669400
commit b37fab40b5
No known key found for this signature in database
GPG Key ID: C16729F1AACF66C6
4 changed files with 100 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/spack/spack/cmd/solve.py
View File

@ -112,8 +112,8 @@ def _process_result(result, show, required_format, kwargs):
% ( % (
i, i,
name, name,
"-" if build_cost is None else installed_cost, installed_cost if installed_cost is not None else "-",
installed_cost if build_cost is None else build_cost, build_cost if build_cost is not None else "-",
) )
) )
print() print()

123
lib/spack/spack/solver/asp.py
View File

@ -13,6 +13,7 @@
import re import re
import types import types
import warnings import warnings
from typing import Tuple
import archspec.cpu import archspec.cpu
@ -124,84 +125,72 @@ def getter(node):
# The space of possible priorities for optimization targets # The space of possible priorities for optimization targets
# is partitioned in the following ranges: # is partitioned in the following ranges:
# # +=============================================================+
# [0-100) Optimization criteria for software being reused # | Priority | Description |
# [100-200) Fixed criteria that are higher priority than reuse, but lower than build # +=============================================================+
# [200-300) Optimization criteria for software being built # | 10,000,000+ | Error conditions |
# [300-1000) High-priority fixed criteria # +-------------+-----------------------------------------------+
# [1000-inf) Error conditions # | 9,999,999 | |
# | ... | High-priority criteria |
# | 1,000,000 | |
# +-------------+-----------------------------------------------+
# | 999,999 | |
# | ... | Standard criteria for built packages |
# | 100,001 | |
# +-------------+-----------------------------------------------+
# | 100,000 | Number of packages being built |
# +-------------+-----------------------------------------------+
# | 99,999 | |
# | ... | Standard criteria for reused packages |
# | 0 | |
# +-------------+-----------------------------------------------+
# #
# Each optimization target is a minimization with optimal value 0. # Each optimization target is a minimization with optimal value 0.
#
#: High fixed priority offset for criteria that supersede all build criteria #: High fixed priority offset for criteria that supersede all build criteria
high_fixed_priority_offset = 300 high_fixed_priority_offset = 10_000_000
#: Priority offset for "build" criteria (regular criterio shifted to #: Priority offset for "build" criteria (regular criterio shifted to
#: higher priority for specs we have to build) #: higher priority for specs we have to build)
build_priority_offset = 200 build_priority_offset = 100_000
#: Priority offset of "fixed" criteria (those w/o build criteria)
fixed_priority_offset = 100
def build_criteria_names(costs, arg_tuples): def build_criteria_names(costs, opt_criteria):
"""Construct an ordered mapping from criteria names to costs.""" """Construct an ordered mapping from criteria names to costs."""
# pull optimization criteria names out of the solution
priorities_names = []
num_fixed = 0 # ensure names of all criteria are unique
num_high_fixed = 0 names = {name for _, name in opt_criteria}
for args in arg_tuples: assert len(names) == len(opt_criteria), "names of optimization criteria must be unique"
priority, name = args[:2]
# costs contains:
# - error criteria
# - number of input specs not concretized
# - N build criteria
# ...
# - number of packages to build
# - N reuse criteria
# ...
# opt_criteria has all the named criteria, which is all but the errors.
# So we can figure out how many build criteria there are up front.
n_build_criteria = len(opt_criteria) - 2
# number of criteria *not* including errors
n_named_criteria = len(opt_criteria) + n_build_criteria
# opt_criteria are in order, highest to lowest, as written in concretize.lp
# put costs in the same order as opt criteria
start = len(costs) - n_named_criteria
ordered_costs = costs[start:]
ordered_costs.insert(1, ordered_costs.pop(n_build_criteria + 1))
# list of build cost, reuse cost, and name of each criterion
criteria: List[Tuple[int, int, str]] = []
for i, (priority, name) in enumerate(opt_criteria):
priority = int(priority) priority = int(priority)
build_cost = ordered_costs[i]
# add the priority of this opt criterion and its name reuse_cost = ordered_costs[i + n_build_criteria] if priority < 100_000 else None
priorities_names.append((priority, name)) criteria.append((reuse_cost, build_cost, name))
# if the priority is less than fixed_priority_offset, then it
# has an associated build priority -- the same criterion but for
# nodes that we have to build.
if priority < fixed_priority_offset:
build_priority = priority + build_priority_offset
priorities_names.append((build_priority, name))
elif priority >= high_fixed_priority_offset:
num_high_fixed += 1
else:
num_fixed += 1
# sort the criteria by priority
priorities_names = sorted(priorities_names, reverse=True)
# We only have opt-criterion values for non-error types
# error type criteria are excluded (they come first)
error_criteria = len(costs) - len(priorities_names)
costs = costs[error_criteria:]
# split list into three parts: build criteria, fixed criteria, non-build criteria
num_criteria = len(priorities_names)
num_build = (num_criteria - num_fixed - num_high_fixed) // 2
build_start_idx = num_high_fixed
fixed_start_idx = num_high_fixed + num_build
installed_start_idx = num_high_fixed + num_build + num_fixed
high_fixed = priorities_names[:build_start_idx]
build = priorities_names[build_start_idx:fixed_start_idx]
fixed = priorities_names[fixed_start_idx:installed_start_idx]
installed = priorities_names[installed_start_idx:]
# mapping from priority to index in cost list
indices = dict((p, i) for i, (p, n) in enumerate(priorities_names))
# make a list that has each name with its build and non-build costs
criteria = [(cost, None, name) for cost, (p, name) in zip(costs[:build_start_idx], high_fixed)]
criteria += [
(cost, None, name)
for cost, (p, name) in zip(costs[fixed_start_idx:installed_start_idx], fixed)
]
for (i, name), (b, _) in zip(installed, build):
criteria.append((costs[indices[i]], costs[indices[b]], name))
return criteria return criteria

74
lib/spack/spack/solver/concretize.lp
View File

@ -23,9 +23,14 @@ literal_not_solved(ID) :- not literal_solved(ID), literal(ID).
% in better reporting for users. See #30669 for details. % in better reporting for users. See #30669 for details.
1 { literal_solved(ID) : literal(ID) }. 1 { literal_solved(ID) : literal(ID) }.
opt_criterion(300, "number of input specs not concretized"). % priority ranges for optimization criteria
#minimize{ 0@300: #true }. #const error_prio = 10000000.
#minimize { 1@300,ID : literal_not_solved(ID) }. #const solve_prio = 1000000.
#const build_prio = 100000.
opt_criterion(solve_prio, "number of input specs not concretized").
#minimize{ 0@solve_prio: #true }.
#minimize{ 1@solve_prio,ID : literal_not_solved(ID) }.
% Map constraint on the literal ID to the correct PSID % Map constraint on the literal ID to the correct PSID
attr(Name, A1) :- literal(LiteralID, Name, A1), literal_solved(LiteralID). attr(Name, A1) :- literal(LiteralID, Name, A1), literal_solved(LiteralID).
@ -1077,7 +1082,7 @@ build(Package) :- not attr("hash", Package, _), attr("node", Package).
% 200+ Shifted priorities for build nodes; correspond to priorities 0 - 99. % 200+ Shifted priorities for build nodes; correspond to priorities 0 - 99.
% 100 - 199 Unshifted priorities. Currently only includes minimizing #builds. % 100 - 199 Unshifted priorities. Currently only includes minimizing #builds.
% 0 - 99 Priorities for non-built nodes. % 0 - 99 Priorities for non-built nodes.
build_priority(Package, 200) :- build(Package), attr("node", Package), optimize_for_reuse(). build_priority(Package, build_prio) :- build(Package), attr("node", Package), optimize_for_reuse().
build_priority(Package, 0) :- not build(Package), attr("node", Package), optimize_for_reuse(). build_priority(Package, 0) :- not build(Package), attr("node", Package), optimize_for_reuse().
% don't adjust build priorities if reuse is not enabled % don't adjust build priorities if reuse is not enabled
@ -1107,16 +1112,17 @@ build_priority(Package, 0) :- attr("node", Package), not optimize_for_reuse().
% Some errors are handled as rules instead of constraints because % Some errors are handled as rules instead of constraints because
% it allows us to explain why something failed. Here we optimize % it allows us to explain why something failed. Here we optimize
% HEAVILY against the facts generated by those rules. % HEAVILY against the facts generated by those rules.
#minimize{ 0@1000: #true}.
#minimize{ 0@1001: #true}.
#minimize{ 0@1002: #true}.
#minimize{ 1000@1000+Priority,Msg: error(Priority, Msg) }. #minimize{ 0@error_prio: #true}.
#minimize{ 1000@1000+Priority,Msg,Arg1: error(Priority, Msg, Arg1) }. #minimize{ 0@error_prio: #true}.
#minimize{ 1000@1000+Priority,Msg,Arg1,Arg2: error(Priority, Msg, Arg1, Arg2) }. #minimize{ 0@error_prio: #true}.
#minimize{ 1000@1000+Priority,Msg,Arg1,Arg2,Arg3: error(Priority, Msg, Arg1, Arg2, Arg3) }.
#minimize{ 1000@1000+Priority,Msg,Arg1,Arg2,Arg3,Arg4: error(Priority, Msg, Arg1, Arg2, Arg3, Arg4) }. #minimize{ 1000@error_prio+Priority,Msg: error(Priority, Msg) }.
#minimize{ 1000@1000+Priority,Msg,Arg1,Arg2,Arg3,Arg4,Arg5: error(Priority, Msg, Arg1, Arg2, Arg3, Arg4, Arg5) }. #minimize{ 1000@error_prio+Priority,Msg,Arg1: error(Priority, Msg, Arg1) }.
#minimize{ 1000@error_prio+Priority,Msg,Arg1,Arg2: error(Priority, Msg, Arg1, Arg2) }.
#minimize{ 1000@error_prio+Priority,Msg,Arg1,Arg2,Arg3: error(Priority, Msg, Arg1, Arg2, Arg3) }.
#minimize{ 1000@error_prio+Priority,Msg,Arg1,Arg2,Arg3,Arg4: error(Priority, Msg, Arg1, Arg2, Arg3, Arg4) }.
#minimize{ 1000@error_prio+Priority,Msg,Arg1,Arg2,Arg3,Arg4,Arg5: error(Priority, Msg, Arg1, Arg2, Arg3, Arg4, Arg5) }.
%----------------------------------------------------------------------------- %-----------------------------------------------------------------------------
% How to optimize the spec (high to low priority) % How to optimize the spec (high to low priority)
@ -1127,16 +1133,16 @@ build_priority(Package, 0) :- attr("node", Package), not optimize_for_reuse().
% is displayed (clingo doesn't display sums over empty sets by default) % is displayed (clingo doesn't display sums over empty sets by default)
% Try hard to reuse installed packages (i.e., minimize the number built) % Try hard to reuse installed packages (i.e., minimize the number built)
opt_criterion(100, "number of packages to build (vs. reuse)"). opt_criterion(build_prio, "number of packages to build (vs. reuse)").
#minimize { 0@100: #true }. #minimize { 0@build_prio: #true }.
#minimize { 1@100,Package : build(Package), optimize_for_reuse() }. #minimize { 1@build_prio,Package : build(Package), optimize_for_reuse() }.
#defined optimize_for_reuse/0. #defined optimize_for_reuse/0.
% A condition group specifies one or more specs that must be satisfied. % A condition group specifies one or more specs that must be satisfied.
% Specs declared first are preferred, so we assign increasing weights and % Specs declared first are preferred, so we assign increasing weights and
% minimize the weights. % minimize the weights.
opt_criterion(75, "requirement weight"). opt_criterion(75, "requirement weight").
#minimize{ 0@275: #true }. #minimize{ 0@75+build_prio: #true }.
#minimize{ 0@75: #true }. #minimize{ 0@75: #true }.
#minimize { #minimize {
Weight@75+Priority Weight@75+Priority
@ -1146,7 +1152,7 @@ opt_criterion(75, "requirement weight").
% Minimize the number of deprecated versions being used % Minimize the number of deprecated versions being used
opt_criterion(73, "deprecated versions used"). opt_criterion(73, "deprecated versions used").
#minimize{ 0@273: #true }. #minimize{ 0@73+build_prio: #true }.
#minimize{ 0@73: #true }. #minimize{ 0@73: #true }.
#minimize{ #minimize{
1@73+Priority,Package 1@73+Priority,Package
@ -1159,7 +1165,7 @@ opt_criterion(73, "deprecated versions used").
% 2. Number of variants with a non default value, if not set % 2. Number of variants with a non default value, if not set
% for the root package. % for the root package.
opt_criterion(70, "version weight"). opt_criterion(70, "version weight").
#minimize{ 0@270: #true }. #minimize{ 0@70+build_prio: #true }.
#minimize{ 0@70: #true }. #minimize{ 0@70: #true }.
#minimize { #minimize {
Weight@70+Priority Weight@70+Priority
@ -1168,7 +1174,7 @@ opt_criterion(70, "version weight").
}. }.
opt_criterion(65, "number of non-default variants (roots)"). opt_criterion(65, "number of non-default variants (roots)").
#minimize{ 0@265: #true }. #minimize{ 0@65+build_prio: #true }.
#minimize{ 0@65: #true }. #minimize{ 0@65: #true }.
#minimize { #minimize {
1@65+Priority,Package,Variant,Value 1@65+Priority,Package,Variant,Value
@ -1178,7 +1184,7 @@ opt_criterion(65, "number of non-default variants (roots)").
}. }.
opt_criterion(60, "preferred providers for roots"). opt_criterion(60, "preferred providers for roots").
#minimize{ 0@260: #true }. #minimize{ 0@60+build_prio: #true }.
#minimize{ 0@60: #true }. #minimize{ 0@60: #true }.
#minimize{ #minimize{
Weight@60+Priority,Provider,Virtual Weight@60+Priority,Provider,Virtual
@ -1188,7 +1194,7 @@ opt_criterion(60, "preferred providers for roots").
}. }.
opt_criterion(55, "default values of variants not being used (roots)"). opt_criterion(55, "default values of variants not being used (roots)").
#minimize{ 0@255: #true }. #minimize{ 0@55+build_prio: #true }.
#minimize{ 0@55: #true }. #minimize{ 0@55: #true }.
#minimize{ #minimize{
1@55+Priority,Package,Variant,Value 1@55+Priority,Package,Variant,Value
@ -1199,7 +1205,7 @@ opt_criterion(55, "default values of variants not being used (roots)").
% Try to use default variants or variants that have been set % Try to use default variants or variants that have been set
opt_criterion(50, "number of non-default variants (non-roots)"). opt_criterion(50, "number of non-default variants (non-roots)").
#minimize{ 0@250: #true }. #minimize{ 0@50+build_prio: #true }.
#minimize{ 0@50: #true }. #minimize{ 0@50: #true }.
#minimize { #minimize {
1@50+Priority,Package,Variant,Value 1@50+Priority,Package,Variant,Value
@ -1211,7 +1217,7 @@ opt_criterion(50, "number of non-default variants (non-roots)").
% Minimize the weights of the providers, i.e. use as much as % Minimize the weights of the providers, i.e. use as much as
% possible the most preferred providers % possible the most preferred providers
opt_criterion(45, "preferred providers (non-roots)"). opt_criterion(45, "preferred providers (non-roots)").
#minimize{ 0@245: #true }. #minimize{ 0@45+build_prio: #true }.
#minimize{ 0@45: #true }. #minimize{ 0@45: #true }.
#minimize{ #minimize{
Weight@45+Priority,Provider,Virtual Weight@45+Priority,Provider,Virtual
@ -1221,7 +1227,7 @@ opt_criterion(45, "preferred providers (non-roots)").
% Try to minimize the number of compiler mismatches in the DAG. % Try to minimize the number of compiler mismatches in the DAG.
opt_criterion(40, "compiler mismatches that are not from CLI"). opt_criterion(40, "compiler mismatches that are not from CLI").
#minimize{ 0@240: #true }. #minimize{ 0@40+build_prio: #true }.
#minimize{ 0@40: #true }. #minimize{ 0@40: #true }.
#minimize{ #minimize{
1@40+Priority,Package,Dependency 1@40+Priority,Package,Dependency
@ -1229,8 +1235,8 @@ opt_criterion(40, "compiler mismatches that are not from CLI").
build_priority(Package, Priority) build_priority(Package, Priority)
}. }.
opt_criterion(39, "compiler mismatches that are not from CLI"). opt_criterion(39, "compiler mismatches from CLI").
#minimize{ 0@239: #true }. #minimize{ 0@39+build_prio: #true }.
#minimize{ 0@39: #true }. #minimize{ 0@39: #true }.
#minimize{ #minimize{
1@39+Priority,Package,Dependency 1@39+Priority,Package,Dependency
@ -1240,7 +1246,7 @@ opt_criterion(39, "compiler mismatches that are not from CLI").
% Try to minimize the number of compiler mismatches in the DAG. % Try to minimize the number of compiler mismatches in the DAG.
opt_criterion(35, "OS mismatches"). opt_criterion(35, "OS mismatches").
#minimize{ 0@235: #true }. #minimize{ 0@35+build_prio: #true }.
#minimize{ 0@35: #true }. #minimize{ 0@35: #true }.
#minimize{ #minimize{
1@35+Priority,Package,Dependency 1@35+Priority,Package,Dependency
@ -1249,7 +1255,7 @@ opt_criterion(35, "OS mismatches").
}. }.
opt_criterion(30, "non-preferred OS's"). opt_criterion(30, "non-preferred OS's").
#minimize{ 0@230: #true }. #minimize{ 0@30+build_prio: #true }.
#minimize{ 0@30: #true }. #minimize{ 0@30: #true }.
#minimize{ #minimize{
Weight@30+Priority,Package Weight@30+Priority,Package
@ -1259,7 +1265,7 @@ opt_criterion(30, "non-preferred OS's").
% Choose more recent versions for nodes % Choose more recent versions for nodes
opt_criterion(25, "version badness"). opt_criterion(25, "version badness").
#minimize{ 0@225: #true }. #minimize{ 0@25+build_prio: #true }.
#minimize{ 0@25: #true }. #minimize{ 0@25: #true }.
#minimize{ #minimize{
Weight@25+Priority,Package Weight@25+Priority,Package
@ -1269,7 +1275,7 @@ opt_criterion(25, "version badness").
% Try to use all the default values of variants % Try to use all the default values of variants
opt_criterion(20, "default values of variants not being used (non-roots)"). opt_criterion(20, "default values of variants not being used (non-roots)").
#minimize{ 0@220: #true }. #minimize{ 0@20+build_prio: #true }.
#minimize{ 0@20: #true }. #minimize{ 0@20: #true }.
#minimize{ #minimize{
1@20+Priority,Package,Variant,Value 1@20+Priority,Package,Variant,Value
@ -1280,7 +1286,7 @@ opt_criterion(20, "default values of variants not being used (non-roots)").
% Try to use preferred compilers % Try to use preferred compilers
opt_criterion(15, "non-preferred compilers"). opt_criterion(15, "non-preferred compilers").
#minimize{ 0@215: #true }. #minimize{ 0@15+build_prio: #true }.
#minimize{ 0@15: #true }. #minimize{ 0@15: #true }.
#minimize{ #minimize{
Weight@15+Priority,Package Weight@15+Priority,Package
@ -1291,7 +1297,7 @@ opt_criterion(15, "non-preferred compilers").
% Minimize the number of mismatches for targets in the DAG, try % Minimize the number of mismatches for targets in the DAG, try
% to select the preferred target. % to select the preferred target.
opt_criterion(10, "target mismatches"). opt_criterion(10, "target mismatches").
#minimize{ 0@210: #true }. #minimize{ 0@10+build_prio: #true }.
#minimize{ 0@10: #true }. #minimize{ 0@10: #true }.
#minimize{ #minimize{
1@10+Priority,Package,Dependency 1@10+Priority,Package,Dependency
@ -1300,7 +1306,7 @@ opt_criterion(10, "target mismatches").
}. }.
opt_criterion(5, "non-preferred targets"). opt_criterion(5, "non-preferred targets").
#minimize{ 0@205: #true }. #minimize{ 0@5+build_prio: #true }.
#minimize{ 0@5: #true }. #minimize{ 0@5: #true }.
#minimize{ #minimize{
Weight@5+Priority,Package Weight@5+Priority,Package

2
lib/spack/spack/test/concretize.py
View File

@ -1779,7 +1779,7 @@ def test_version_weight_and_provenance(self):
num_specs = len(list(result_spec.traverse())) num_specs = len(list(result_spec.traverse()))
criteria = [ criteria = [
(num_specs - 1, None, "number of packages to build (vs. reuse)"), (None, num_specs - 1, "number of packages to build (vs. reuse)"),
(2, 0, "version badness"), (2, 0, "version badness"),
] ]