zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Todd Gamblin 2023-01-15 09:48:53 -08:00
parent 06d5abe895
commit 3435806007
No known key found for this signature in database
GPG Key ID: C16729F1AACF66C6
4 changed files with 173 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
lib/spack/spack/cmd/solve.py
View File

@ -98,27 +98,40 @@ def setup_parser(subparser):
def _process_result(result, show, required_format, kwargs):
result.raise_if_unsat()
opt, *_ = min(result.answers)
# dump the solutions as concretized specs
if ("opt" in show) and (not required_format):
tty.msg("Best of %d considered solutions." % result.nmodels)
tty.msg("Optimization Criteria:")
maxlen = max(len(s[2]) for s in result.criteria)
color.cprint("@*{ Priority Criterion %sInstalled ToBuild}" % ((maxlen - 10) * " "))
maxlen = max(len(name) for name in result.criteria)
max_depth = max(len(v) for v in result.criteria.values() if isinstance(v, list))
fmt = " @K{%%-8d} %%-%ds%%9s %%7s" % maxlen
for i, (installed_cost, build_cost, name) in enumerate(result.criteria, 1):
color.cprint(
fmt
% (
i,
name,
installed_cost if installed_cost is not None else "-",
build_cost if build_cost is not None else "-",
header = "@*{"
header += "".join(f"{depth:<4}" for depth in range(max_depth))
header += "Criterion}"
color.cprint(header)
# make non-zero numbers red
def highlight(n, c):
return color.colorize(f"@{c}{{{n:<4}}}" if n > 0 else f"{n:<4}")
for i, (name, cost) in enumerate(result.criteria.items(), 1):
colored_name = name.replace("build:", "@c{build:}")
colored_name = colored_name.replace("reuse:", "@B{reuse:}")
colored_name = colored_name.replace("fixed:", "@G{fixed:}")
colored_name = color.colorize(colored_name)
if isinstance(cost, int):
print(highlight(cost, "G") + " " * (max_depth - 1) + colored_name)
else:
print(
"".join(highlight(c, "c" if "build:" in name else "B") for c in cost)
+ colored_name
)
)
print()
# dump the solutions as concretized specs
if "solutions" in show:
for spec in result.specs:
# With -y, just print YAML to output.

91
lib/spack/spack/solver/asp.py
View File

@ -154,72 +154,54 @@ def getter(node):
#: higher priority for specs we have to build)
build_priority_offset = 100_000
#: max priority for an error
max_error_priority = 3
def build_criteria_names(
costs: List[int], opt_criteria: List["AspFunction"], max_depth: int
) -> Dict[str, Union[int, List[int]]]:
costs: List[int], opt_criteria: List["AspFunction"], max_depth: int, const_max_depth: int
) -> Dict[str, Union[int, List[Tuple[int, int]]]]:
"""Construct an ordered mapping from criteria names to costs."""
# ensure names of all criteria are unique
names = {criterion.args[0] for criterion in opt_criteria}
assert len(names) == len(opt_criteria), "names of optimization criteria must be unique"
# costs contains:
# - error criteria
# - number of input specs not concretized
# - N build criteria
# ...
# - number of packages to build
# - N reuse criteria
# ...
# split opt criteria into two lists
fixed_criteria = [oc for oc in opt_criteria if oc.args[1] == "fixed"]
leveled_criteria = [oc for oc in opt_criteria if oc.args[1] == "leveled"]
print("FIXED:", len(fixed_criteria))
print("LEVELED:", len(leveled_criteria))
# first non-error criterion
solve_index = max_error_priority + 1
n_depths = max_depth + 1 # depths start at zero
total_criteria = len(fixed_criteria) + len(leveled_criteria) * 2 * n_depths
print(total_criteria)
# each criterion is aggregated for each level in the graph
max_leveled_costs = len(leveled_criteria) * const_max_depth
n_leveled_costs = len(leveled_criteria) * (max_depth + 1)
# opt_criteria has all the named criteria, which is all but the errors.
# So we can figure out how many build criteria there are up front.
# the -2 here accounts for:
# - number of specs not concretized
# - number of packages to build (vs. reuse)
n_leveled_criteria = len(opt_criteria) - 2
build_index = solve_index + 1 + max_leveled_costs
fixed_costs = [costs[solve_index], costs[build_index]]
n_leveled_criteria = n_build_criteria * 2 * n_depths
total_criteria = n_leveled_criteria + 2
assert len(costs) == total_criteria
# For each level, opt_criteria are in order, highest to lowest, as written in
# concretize.lp put costs in the same order as opt criteria
start = len(costs) - n_named_criteria
# build criteria count should be divisible by n_depths or we're doing something wrong.
assert n_build_criteria % n_depths == 0
build_costs = costs[solve_index + 1 : solve_index + 1 + n_leveled_costs]
reuse_costs = costs[build_index + 1 : build_index + 1 + n_leveled_costs]
assert len(build_costs) == len(reuse_costs) == n_leveled_costs
criteria = {}
ordered_costs = costs[start:]
ordered_costs.insert(1, ordered_costs.pop(n_build_criteria + 1))
for i, (priority, type, name) in enumerate(c.args for c in opt_criteria):
if type == "fixed":
criteria[name] = ordered_costs[i]
else:
criteria[name] = ordered_costs[i::n_build_criteria]
def add_fixed(criterion_idx, cost_idx):
name = fixed_criteria[criterion_idx].args[2]
criteria["fixed: " + name] = costs[cost_idx]
# list of build cost, reuse cost, and name of each criterion
criteria: List[Tuple[int, int, str]] = []
for i, (priority, name) in enumerate(c.args for c in opt_criteria):
priority = int(priority)
build_cost = ordered_costs[i]
reuse_cost = ordered_costs[i + n_build_criteria] if priority < 100_000 else None
criteria.append((reuse_cost, build_cost, name))
add_fixed(0, solve_index)
for i, fn in enumerate(leveled_criteria):
name = fn.args[2]
criteria["build: " + name] = build_costs[i :: len(leveled_criteria)]
add_fixed(1, build_index)
for i, fn in enumerate(leveled_criteria):
name = fn.args[2]
criteria["reuse: " + name] = reuse_costs[i :: len(leveled_criteria)]
return criteria
@ -803,7 +785,7 @@ def on_model(model):
# build specs from spec attributes in the model
spec_attrs = extract_functions(best_model, "attr")
with timer.measure("build_specs"):
with timer.measure("build"):
answers = builder.build_specs(spec_attrs)
# add best spec to the results
@ -812,16 +794,11 @@ def on_model(model):
# get optimization criteria
criteria = extract_functions(best_model, "opt_criterion")
depths = extract_functions(best_model, "depth")
print(depths)
const_max_depth, *_ = extract_functions(best_model, "const_max_depth")
const_max_depth = const_max_depth.args[0]
max_depth = max(d.args[1] for d in depths)
print(f"COST: {len(min_cost)} {min_cost}")
print(f"PRIO: {len(priorities)} {priorities}")
print(f"MAX_DEPTH: {max_depth}")
print()
print(opt_criteria)
result.criteria = build_criteria_names(min_cost, criteria, max_depth)
result.criteria = build_criteria_names(min_cost, criteria, max_depth, const_max_depth)
# record the number of models the solver considered
result.nmodels = len(models)
@ -831,7 +808,7 @@ def on_model(model):
# print any unknown functions in the model
for sym in best_model:
if sym.name not in ("attr", "error", "opt_criterion"):
if sym.name not in ("attr", "error", "opt_criterion", "depth", "const_max_depth"):
tty.debug(
"UNKNOWN SYMBOL: %s(%s)" % (sym.name, ", ".join(stringify(sym.arguments)))
)

208
lib/spack/spack/solver/concretize.lp
View File

@ -25,6 +25,7 @@ literal_not_solved(ID) :- not literal_solved(ID), literal(ID).
% priority ranges for optimization criteria
% note that clingo's weight_t is int32_t, so the max priority we can use is 2,147,483,647
#const max_error_priority = 3.
#const error_prio = 10000000.
#const solve_prio = 1000000.
#const build_prio = 100000. % n_nodes x depth_offset x max levels needs to be less than this
@ -72,7 +73,8 @@ version_declared(Package, Version, Weight) :- version_declared(Package, Version,
version_declared(Package, Version) :- version_declared(Package, Version, _).
% a spec with a git hash version is equivalent to one with the same matched version
version_satisfies(Package, Constraint, HashVersion) :- version_satisfies(Package, Constraint, EquivalentVersion),
version_satisfies(Package, Constraint, HashVersion) :-
version_satisfies(Package, Constraint, EquivalentVersion),
version_equivalent(Package, HashVersion, EquivalentVersion).
#defined version_equivalent/3.
@ -153,7 +155,10 @@ possible_version_weight(Package, Weight)
% Otherwise covered by `no_version_error` and `versions_conflict_error`.
error(1, "No valid version for '{0}' satisfies '@{1}'", Package, Constraint)
:- attr("node_version_satisfies", Package, Constraint),
C = #count{ Version : attr("version", Package, Version), version_satisfies(Package, Constraint, Version)},
C = #count{
Version
: attr("version", Package, Version), version_satisfies(Package, Constraint, Version)
},
C < 1.
attr("node_version_satisfies", Package, Constraint)
@ -1088,7 +1093,7 @@ build_priority(Package, build_prio) :- build(Package), attr("node", Package), op
build_priority(Package, 0) :- not build(Package), attr("node", Package), optimize_for_reuse().
% don't adjust build priorities if reuse is not enabled
build_priority(Package, 0) :- attr("node", Package), not optimize_for_reuse().
build_priority(Package, build_prio) :- attr("node", Package), not optimize_for_reuse().
% don't assign versions from installed packages unless reuse is enabled
% NOTE: that "installed" means the declared version was only included because
@ -1112,7 +1117,8 @@ build_priority(Package, 0) :- attr("node", Package), not optimize_for_reuse().
% Calculate min depth of nodes in the DAG
% We use this to optimize nodes closer to roots with higher precedence.
%-----------------------------------------------------------------------------
#const max_depth = 10.
#const max_depth = 5.
const_max_depth(max_depth).
% roots have depth 0.
depth(Package, 0) :- attr("root", Package).
@ -1122,22 +1128,20 @@ min_parent_depth(Package, N) :-
N = #min{
D: depends_on(Dependent, Package),
depth(Dependent, D),
0 <= D,
D < max_depth
D = 0..max_depth - 1
},
0 <= N,
N < max_depth,
N = 0..max_depth - 1,
attr("node", Package).
depth(Package, max_depth) :-
depth(Package, max_depth - 1) :-
min_parent_depth(Package, P),
P >= max_depth,
P >= max_depth - 1,
attr("node", Package).
depth(Package, P+1) :-
min_parent_depth(Package, P),
P >= 0,
P < max_depth,
P < max_depth - 1,
attr("node", Package).
@ -1148,10 +1152,10 @@ depth(Package, P+1) :-
% it allows us to explain why something failed. Here we optimize
% HEAVILY against the facts generated by those rules.
#minimize{ 0@error_prio: #true}.
#minimize{ 0@error_prio: #true}.
#minimize{ 0@error_prio: #true}.
% ensure that error costs are always in the solution.
#minimize{ 0@error_prio + (0..max_error_priority): #true}.
% TODO: why 1000 and not just 1? 1000 seems unnecessary since priorities are lexicographic.
#minimize{ 1000@error_prio+Priority,Msg: error(Priority, Msg) }.
#minimize{ 1000@error_prio+Priority,Msg,Arg1: error(Priority, Msg, Arg1) }.
#minimize{ 1000@error_prio+Priority,Msg,Arg1,Arg2: error(Priority, Msg, Arg1, Arg2) }.
@ -1177,17 +1181,18 @@ depth(Package, P+1) :-
#minimize{ 0@N: opt_criterion(N, "fixed", _) }.
% "leveled" criteria sum into a bucket per depth in the graph, per build priority
%#minimize{
% @(((max_depth - D - 1) * depth_offset) + N + build_prio)
% : opt_criterion(N, "leveled", _), depth(_, D)
%}.
#minimize{
0@(((max_depth - D - 1) * depth_offset) + N + build_prio)
: opt_criterion(N, "leveled", _), D = 0..max_depth - 1
}.
#minimize{
0@(((max_depth - D - 1) * depth_offset) + N)
: opt_criterion(N, "leveled", _), depth(_, D)
: opt_criterion(N, "leveled", _), D = 0..max_depth - 1
}.
% Try hard to reuse installed packages (i.e., minimize the number built)
opt_criterion(build_prio, "fixed", "number of packages to build (vs. reuse)").
#minimize { 1@build_prio,Package : build(Package), optimize_for_reuse() }.
#defined optimize_for_reuse/0.
@ -1195,118 +1200,118 @@ opt_criterion(build_prio, "fixed", "number of packages to build (vs. reuse)").
% Specs declared first are preferred, so we assign increasing weights and
% minimize the weights.
opt_criterion(65, "leveled", "requirement weight").
%#minimize {
% Weight@(65 + ((max_depth - D - 1) * depth_offset) + Priority), Package
% : requirement_weight(Package, Weight),
% build_priority(Package, Priority),
% depth(Package, D)
%}.
#minimize{
Weight@(65 + ((max_depth - D - 1) * depth_offset) + Priority), Package
: requirement_weight(Package, Weight),
build_priority(Package, Priority),
depth(Package, D)
}.
% Minimize the number of deprecated versions being used
opt_criterion(60, "leveled", "deprecated versions used").
%#minimize{
% 1@(60 + ((max_depth - D - 1) * depth_offset) + Priority), Package
% : attr("deprecated", Package, _),
% build_priority(Package, Priority),
% depth(Package, D)
%}.
#minimize{
1@(60 + ((max_depth - D - 1) * depth_offset) + Priority), Package
: attr("deprecated", Package, _),
build_priority(Package, Priority),
depth(Package, D)
}.
% Minimize the:
% 1. Version weight
% 2. Number of variants with a non default value, if not set
% for the root package.
opt_criterion(55, "leveled", "version badness").
%#minimize {
% Weight@(55 + ((max_depth - D - 1) * depth_offset) + Priority), Package
% : version_weight(Package, Weight),
% build_priority(Package, Priority),
% depth(Package, D)
%}.
#minimize{
Weight@(55 + ((max_depth - D - 1) * depth_offset) + Priority), Package
: version_weight(Package, Weight),
build_priority(Package, Priority),
depth(Package, D)
}.
opt_criterion(50, "leveled", "number of non-default variants").
%#minimize {
% 1@(50 + ((max_depth - D - 1) * depth_offset) + Priority), Package, Variant, Value
% : variant_not_default(Package, Variant, Value),
% build_priority(Package, Priority),
% depth(Package, D)
%}.
#minimize{
1@(50 + ((max_depth - D - 1) * depth_offset) + Priority), Package, Variant, Value
: variant_not_default(Package, Variant, Value),
build_priority(Package, Priority),
depth(Package, D)
}.
opt_criterion(45, "leveled", "preferred providers").
%#minimize{
% Weight@(45 + ((max_depth - D - 1) * depth_offset) + Priority), Provider, Virtual
% : provider_weight(Provider, Virtual, Weight),
% build_priority(Provider, Priority),
% depth(Package, D)
%}.
#minimize{
Weight@(45 + ((max_depth - D - 1) * depth_offset) + Priority), Provider, Virtual
: provider_weight(Provider, Virtual, Weight),
build_priority(Provider, Priority),
depth(Package, D)
}.
opt_criterion(40, "leveled", "default values of variants not being used").
%#minimize{
% 1@(40 + ((max_depth - D - 1) * depth_offset) + Priority), Package, Variant, Value
% : variant_default_not_used(Package, Variant, Value),
% build_priority(Package, Priority),
% depth(Package, D)
%}.
#minimize{
1@(40 + ((max_depth - D - 1) * depth_offset) + Priority), Package, Variant, Value
: variant_default_not_used(Package, Variant, Value),
build_priority(Package, Priority),
depth(Package, D)
}.
% Try to use default variants or variants that have been set
opt_criterion(35, "leveled", "compiler mismatches (not from CLI)").
%#minimize{
% 1@(35 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package
% : compiler_mismatch(Dependent, Package),
% build_priority(Package, Priority),
% depth(Package, D)
%}.
#minimize{
1@(35 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package
: compiler_mismatch(Dependent, Package),
build_priority(Package, Priority),
depth(Package, D)
}.
opt_criterion(30, "leveled", "compiler mismatches (from CLI)").
%#minimize{
% 1@(30 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package
% : compiler_mismatch_required(Dependent, Package),
% build_priority(Package, Priority),
% depth(Package, D)
%}.
#minimize{
1@(30 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package
: compiler_mismatch_required(Dependent, Package),
build_priority(Package, Priority),
depth(Package, D)
}.
% Try to minimize the number of compiler mismatches in the DAG.
opt_criterion(25, "leveled", "OS mismatches").
%#minimize{
% 1@(25 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package
% : node_os_mismatch(Dependent, Package),
% build_priority(Package, Priority),
% depth(Package, D)
%}.
#minimize{
1@(25 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package
: node_os_mismatch(Dependent, Package),
build_priority(Package, Priority),
depth(Package, D)
}.
% Try to use preferred compilers
opt_criterion(20, "leveled", "non-preferred compilers").
%#minimize{
% Weight@(20 + ((max_depth - D - 1) * depth_offset) + Priority), Package
% : compiler_weight(Package, Weight),
% build_priority(Package, Priority),
% depth(Package, D)
%}.
#minimize{
Weight@(20 + ((max_depth - D - 1) * depth_offset) + Priority), Package
: compiler_weight(Package, Weight),
build_priority(Package, Priority),
depth(Package, D)
}.
opt_criterion(15, "leveled", "non-preferred OS's").
%#minimize{
% Weight@(15 + ((max_depth - D - 1) * depth_offset) + Priority), Package
% : node_os_weight(Package, Weight),
% build_priority(Package, Priority),
% depth(Package, D)
%}.
#minimize{
Weight@(15 + ((max_depth - D - 1) * depth_offset) + Priority), Package
: node_os_weight(Package, Weight),
build_priority(Package, Priority),
depth(Package, D)
}.
% Minimize the number of mismatches for targets in the DAG, try
% to select the preferred target.
opt_criterion(10, "leveled", "target mismatches").
%#minimize{
% 1@(10 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package
% : node_target_mismatch(Dependent, Package),
% build_priority(Package, Priority),
% depth(Package, D)
%}.
#minimize{
1@(10 + ((max_depth - D - 1) * depth_offset) + Priority), Dependent, Package
: node_target_mismatch(Dependent, Package),
build_priority(Package, Priority),
depth(Package, D)
}.
opt_criterion(5, "leveled", "non-preferred targets").
%#minimize{
% Weight@(5 + ((max_depth - D - 1) * depth_offset) + Priority), Package
% : node_target_weight(Package, Weight),
% build_priority(Package, Priority),
% depth(Package, D)
%}.
#minimize{
Weight@(5 + ((max_depth - D - 1) * depth_offset) + Priority), Package
: node_target_weight(Package, Weight),
build_priority(Package, Priority),
depth(Package, D)
}.
%-----------------
% Domain heuristic
@ -1316,9 +1321,12 @@ opt_criterion(5, "leveled", "non-preferred targets").
#heuristic attr("node_target", Package, Target) : package_target_weight(Target, Package, 0), attr("node", Package). [10, true]
#heuristic node_target_weight(Package, 0) : attr("node", Package). [10, true]
#heuristic attr("variant_value", Package, Variant, Value) : variant_default_value(Package, Variant, Value), attr("node", Package). [10, true]
#heuristic provider(Package, Virtual) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true]
#heuristic attr("node", Package) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true]
#heuristic attr("node_os", Package, OS) : buildable_os(OS). [10, true]
%#heuristic provider(Package, Virtual) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true]
%#heuristic attr("node", Package) : possible_provider_weight(Package, Virtual, 0, _), attr("virtual_node", Virtual). [10, true]
%#heuristic attr("node_os", Package, OS) : buildable_os(OS). [10, true]
%#heuristic attr("node_target", Dependency, Target): depends_on(Package, Dependency), attr("node_target", Package, Target). [20, true]
%#heuristic attr("node_os", Dependency, OS): depends_on(Package, Dependency), attr("node_os", Package, OS). [20, true]
%-----------
% Notes

8
lib/spack/spack/solver/display.lp
View File

@ -25,8 +25,12 @@
#show error/6.
#show error/7.
% debug
% depths
#show depth/2.
#show const_max_depth/1.
% debug
%#show depends_on/2.
%node(Package) :- attr("node", Package).
@ -34,5 +38,3 @@
%version(Package, Version) :- attr("version", Package, Version).
%#show version/2.
#show impose/1.