include installed hashes in solve and optimize for reuse
This commit is contained in:
parent
7abe4ab309
commit
3866b3e7d3
@ -43,6 +43,7 @@
|
|||||||
import spack.platforms
|
import spack.platforms
|
||||||
import spack.repo
|
import spack.repo
|
||||||
import spack.spec
|
import spack.spec
|
||||||
|
import spack.store
|
||||||
import spack.util.timer
|
import spack.util.timer
|
||||||
import spack.variant
|
import spack.variant
|
||||||
import spack.version
|
import spack.version
|
||||||
@ -816,18 +817,21 @@ def condition(self, required_spec, imposed_spec=None, name=None):
|
|||||||
)
|
)
|
||||||
|
|
||||||
if imposed_spec:
|
if imposed_spec:
|
||||||
imposed_constraints = self.spec_clauses(
|
self.impose(condition_id, imposed_spec, node=False, name=name)
|
||||||
imposed_spec, body=False, required_from=name)
|
|
||||||
for pred in imposed_constraints:
|
|
||||||
# imposed "node"-like conditions are no-ops
|
|
||||||
if pred.name in ("node", "virtual_node"):
|
|
||||||
continue
|
|
||||||
self.gen.fact(
|
|
||||||
fn.imposed_constraint(condition_id, pred.name, *pred.args)
|
|
||||||
)
|
|
||||||
|
|
||||||
return condition_id
|
return condition_id
|
||||||
|
|
||||||
|
def impose(self, condition_id, imposed_spec, node=True, name=None):
|
||||||
|
imposed_constraints = self.spec_clauses(
|
||||||
|
imposed_spec, body=False, required_from=name)
|
||||||
|
for pred in imposed_constraints:
|
||||||
|
# imposed "node"-like conditions are no-ops
|
||||||
|
if not node and pred.name in ("node", "virtual_node"):
|
||||||
|
continue
|
||||||
|
self.gen.fact(
|
||||||
|
fn.imposed_constraint(condition_id, pred.name, *pred.args)
|
||||||
|
)
|
||||||
|
|
||||||
def package_provider_rules(self, pkg):
|
def package_provider_rules(self, pkg):
|
||||||
for provider_name in sorted(set(s.name for s in pkg.provided.keys())):
|
for provider_name in sorted(set(s.name for s in pkg.provided.keys())):
|
||||||
self.gen.fact(fn.possible_provider(pkg.name, provider_name))
|
self.gen.fact(fn.possible_provider(pkg.name, provider_name))
|
||||||
@ -1127,13 +1131,22 @@ class Body(object):
|
|||||||
|
|
||||||
# dependencies
|
# dependencies
|
||||||
if spec.concrete:
|
if spec.concrete:
|
||||||
clauses.append(fn.concrete(spec.name))
|
clauses.append(fn.hash(spec.name, spec.dag_hash()))
|
||||||
# TODO: add concrete depends_on() facts for concrete dependencies
|
|
||||||
|
|
||||||
# add all clauses from dependencies
|
# add all clauses from dependencies
|
||||||
if transitive:
|
if transitive:
|
||||||
|
if spec.concrete:
|
||||||
|
for dep_name, dep in spec.dependencies_dict().items():
|
||||||
|
for dtype in dep.deptypes:
|
||||||
|
clauses.append(fn.depends_on(spec.name, dep_name, dtype))
|
||||||
|
|
||||||
for dep in spec.traverse(root=False):
|
for dep in spec.traverse(root=False):
|
||||||
clauses.extend(self._spec_clauses(dep, body, transitive=False))
|
if spec.concrete:
|
||||||
|
clauses.append(fn.hash(dep.name, dep.dag_hash()))
|
||||||
|
else:
|
||||||
|
clauses.extend(
|
||||||
|
self._spec_clauses(dep, body, transitive=False)
|
||||||
|
)
|
||||||
|
|
||||||
return clauses
|
return clauses
|
||||||
|
|
||||||
@ -1475,6 +1488,26 @@ def define_variant_values(self):
|
|||||||
for pkg, variant, value in sorted(self.variant_values_from_specs):
|
for pkg, variant, value in sorted(self.variant_values_from_specs):
|
||||||
self.gen.fact(fn.variant_possible_value(pkg, variant, value))
|
self.gen.fact(fn.variant_possible_value(pkg, variant, value))
|
||||||
|
|
||||||
|
def define_installed_packages(self, possible):
|
||||||
|
"""Add facts about all specs already in the database.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
possible (dict): result of Package.possible_dependencies() for
|
||||||
|
specs in this solve.
|
||||||
|
"""
|
||||||
|
with spack.store.db.read_transaction():
|
||||||
|
for spec in spack.store.db.query(installed=True):
|
||||||
|
# tell the solver about any installed packages that could
|
||||||
|
# be dependencies (don't tell it about the others)
|
||||||
|
if spec.name in possible:
|
||||||
|
# this indicates that there is a spec like this installed
|
||||||
|
h = spec.dag_hash()
|
||||||
|
self.gen.fact(fn.installed_hash(spec.name, h))
|
||||||
|
|
||||||
|
# this describes what constraints it imposes on the solve
|
||||||
|
self.impose(h, spec)
|
||||||
|
self.gen.newline()
|
||||||
|
|
||||||
def setup(self, driver, specs, tests=False, reuse=False):
|
def setup(self, driver, specs, tests=False, reuse=False):
|
||||||
"""Generate an ASP program with relevant constraints for specs.
|
"""Generate an ASP program with relevant constraints for specs.
|
||||||
|
|
||||||
@ -1577,6 +1610,10 @@ def setup(self, driver, specs, tests=False, reuse=False):
|
|||||||
self.gen.h1("Target Constraints")
|
self.gen.h1("Target Constraints")
|
||||||
self.define_target_constraints()
|
self.define_target_constraints()
|
||||||
|
|
||||||
|
if reuse:
|
||||||
|
self.gen.h1("Installed packages")
|
||||||
|
self.define_installed_packages(possible)
|
||||||
|
|
||||||
|
|
||||||
class SpecBuilder(object):
|
class SpecBuilder(object):
|
||||||
"""Class with actions to rebuild a spec from ASP results."""
|
"""Class with actions to rebuild a spec from ASP results."""
|
||||||
@ -1586,6 +1623,14 @@ def __init__(self, specs):
|
|||||||
self._flag_sources = collections.defaultdict(lambda: set())
|
self._flag_sources = collections.defaultdict(lambda: set())
|
||||||
self._flag_compiler_defaults = set()
|
self._flag_compiler_defaults = set()
|
||||||
|
|
||||||
|
def hash(self, pkg, h):
|
||||||
|
if pkg not in self._specs:
|
||||||
|
self._specs[pkg] = spack.store.db.get_by_hash(h)[0]
|
||||||
|
else:
|
||||||
|
# ensure that if it's already there, it's correct
|
||||||
|
spec = self._specs[pkg]
|
||||||
|
assert spec.dag_hash() == h
|
||||||
|
|
||||||
def node(self, pkg):
|
def node(self, pkg):
|
||||||
if pkg not in self._specs:
|
if pkg not in self._specs:
|
||||||
self._specs[pkg] = spack.spec.Spec(pkg)
|
self._specs[pkg] = spack.spec.Spec(pkg)
|
||||||
@ -1727,6 +1772,7 @@ def build_specs(self, function_tuples):
|
|||||||
# them here so that directives that build objects (like node and
|
# them here so that directives that build objects (like node and
|
||||||
# node_compiler) are called in the right order.
|
# node_compiler) are called in the right order.
|
||||||
function_tuples.sort(key=lambda f: {
|
function_tuples.sort(key=lambda f: {
|
||||||
|
"hash": -3,
|
||||||
"node": -2,
|
"node": -2,
|
||||||
"node_compiler": -1,
|
"node_compiler": -1,
|
||||||
}.get(f[0], 0))
|
}.get(f[0], 0))
|
||||||
@ -1749,6 +1795,12 @@ def build_specs(self, function_tuples):
|
|||||||
if spack.repo.path.is_virtual(pkg):
|
if spack.repo.path.is_virtual(pkg):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# if we've already gotten a concrete spec for this pkg,
|
||||||
|
# do not bother calling actions on it.
|
||||||
|
spec = self._specs.get(pkg)
|
||||||
|
if spec and spec.concrete:
|
||||||
|
continue
|
||||||
|
|
||||||
action(*args)
|
action(*args)
|
||||||
|
|
||||||
# namespace assignment is done after the fact, as it is not
|
# namespace assignment is done after the fact, as it is not
|
||||||
|
@ -87,18 +87,28 @@ attr(Name, A1, A2, A3) :- impose(ID), imposed_constraint(ID, Name, A1, A2, A3).
|
|||||||
#defined imposed_constraint/4.
|
#defined imposed_constraint/4.
|
||||||
#defined imposed_constraint/5.
|
#defined imposed_constraint/5.
|
||||||
|
|
||||||
|
%-----------------------------------------------------------------------------
|
||||||
|
% Concrete specs
|
||||||
|
%-----------------------------------------------------------------------------
|
||||||
|
% if a package is assigned a hash, it's concrete.
|
||||||
|
concrete(Package) :- hash(Package, _), node(Package).
|
||||||
|
|
||||||
%-----------------------------------------------------------------------------
|
%-----------------------------------------------------------------------------
|
||||||
% Dependency semantics
|
% Dependency semantics
|
||||||
%-----------------------------------------------------------------------------
|
%-----------------------------------------------------------------------------
|
||||||
% Dependencies of any type imply that one package "depends on" another
|
% Dependencies of any type imply that one package "depends on" another
|
||||||
depends_on(Package, Dependency) :- depends_on(Package, Dependency, _).
|
depends_on(Package, Dependency) :- depends_on(Package, Dependency, _).
|
||||||
|
|
||||||
% a dependency holds if its condition holds
|
% a dependency holds if its condition holds and if it is not external or
|
||||||
|
% concrete. We chop off dependencies for externals, and dependencies of
|
||||||
|
% concrete specs don't need to be resolved -- they arise from the concrete
|
||||||
|
% specs themselves.
|
||||||
dependency_holds(Package, Dependency, Type) :-
|
dependency_holds(Package, Dependency, Type) :-
|
||||||
dependency_condition(ID, Package, Dependency),
|
dependency_condition(ID, Package, Dependency),
|
||||||
dependency_type(ID, Type),
|
dependency_type(ID, Type),
|
||||||
condition_holds(ID),
|
condition_holds(ID),
|
||||||
not external(Package).
|
not external(Package),
|
||||||
|
not concrete(Package).
|
||||||
|
|
||||||
% We cut off dependencies of externals (as we don't really know them).
|
% We cut off dependencies of externals (as we don't really know them).
|
||||||
% Don't impose constraints on dependencies that don't exist.
|
% Don't impose constraints on dependencies that don't exist.
|
||||||
@ -251,6 +261,7 @@ possible_provider_weight(Dependency, Virtual, 100, "fallback") :- provider(Depen
|
|||||||
% These allow us to easily define conditional dependency and conflict rules
|
% These allow us to easily define conditional dependency and conflict rules
|
||||||
% without enumerating all spec attributes every time.
|
% without enumerating all spec attributes every time.
|
||||||
node(Package) :- attr("node", Package).
|
node(Package) :- attr("node", Package).
|
||||||
|
hash(Package, Hash) :- attr("hash", Package, Hash).
|
||||||
version(Package, Version) :- attr("version", Package, Version).
|
version(Package, Version) :- attr("version", Package, Version).
|
||||||
version_satisfies(Package, Constraint) :- attr("version_satisfies", Package, Constraint).
|
version_satisfies(Package, Constraint) :- attr("version_satisfies", Package, Constraint).
|
||||||
node_platform(Package, Platform) :- attr("node_platform", Package, Platform).
|
node_platform(Package, Platform) :- attr("node_platform", Package, Platform).
|
||||||
@ -261,12 +272,14 @@ variant_value(Package, Variant, Value) :- attr("variant_value", Package, Variant
|
|||||||
variant_set(Package, Variant, Value) :- attr("variant_set", Package, Variant, Value).
|
variant_set(Package, Variant, Value) :- attr("variant_set", Package, Variant, Value).
|
||||||
node_flag(Package, FlagType, Flag) :- attr("node_flag", Package, FlagType, Flag).
|
node_flag(Package, FlagType, Flag) :- attr("node_flag", Package, FlagType, Flag).
|
||||||
node_compiler(Package, Compiler) :- attr("node_compiler", Package, Compiler).
|
node_compiler(Package, Compiler) :- attr("node_compiler", Package, Compiler).
|
||||||
|
depends_on(Package, Dependency, Type) :- attr("depends_on", Package, Dependency, Type).
|
||||||
node_compiler_version(Package, Compiler, Version)
|
node_compiler_version(Package, Compiler, Version)
|
||||||
:- attr("node_compiler_version", Package, Compiler, Version).
|
:- attr("node_compiler_version", Package, Compiler, Version).
|
||||||
node_compiler_version_satisfies(Package, Compiler, Version)
|
node_compiler_version_satisfies(Package, Compiler, Version)
|
||||||
:- attr("node_compiler_version_satisfies", Package, Compiler, Version).
|
:- attr("node_compiler_version_satisfies", Package, Compiler, Version).
|
||||||
|
|
||||||
attr("node", Package) :- node(Package).
|
attr("node", Package) :- node(Package).
|
||||||
|
attr("hash", Package, Hash) :- hash(Package, Hash).
|
||||||
attr("version", Package, Version) :- version(Package, Version).
|
attr("version", Package, Version) :- version(Package, Version).
|
||||||
attr("version_satisfies", Package, Constraint) :- version_satisfies(Package, Constraint).
|
attr("version_satisfies", Package, Constraint) :- version_satisfies(Package, Constraint).
|
||||||
attr("node_platform", Package, Platform) :- node_platform(Package, Platform).
|
attr("node_platform", Package, Platform) :- node_platform(Package, Platform).
|
||||||
@ -277,6 +290,7 @@ attr("variant_value", Package, Variant, Value) :- variant_value(Package, Variant
|
|||||||
attr("variant_set", Package, Variant, Value) :- variant_set(Package, Variant, Value).
|
attr("variant_set", Package, Variant, Value) :- variant_set(Package, Variant, Value).
|
||||||
attr("node_flag", Package, FlagType, Flag) :- node_flag(Package, FlagType, Flag).
|
attr("node_flag", Package, FlagType, Flag) :- node_flag(Package, FlagType, Flag).
|
||||||
attr("node_compiler", Package, Compiler) :- node_compiler(Package, Compiler).
|
attr("node_compiler", Package, Compiler) :- node_compiler(Package, Compiler).
|
||||||
|
attr("depends_on", Package, Dependency, Type) :- depends_on(Package, Dependency, Type).
|
||||||
attr("node_compiler_version", Package, Compiler, Version)
|
attr("node_compiler_version", Package, Compiler, Version)
|
||||||
:- node_compiler_version(Package, Compiler, Version).
|
:- node_compiler_version(Package, Compiler, Version).
|
||||||
attr("node_compiler_version_satisfies", Package, Compiler, Version)
|
attr("node_compiler_version_satisfies", Package, Compiler, Version)
|
||||||
@ -732,6 +746,21 @@ no_flags(Package, FlagType)
|
|||||||
#defined node_flag/3.
|
#defined node_flag/3.
|
||||||
#defined node_flag_set/3.
|
#defined node_flag_set/3.
|
||||||
|
|
||||||
|
|
||||||
|
%-----------------------------------------------------------------------------
|
||||||
|
% Installed packages
|
||||||
|
%-----------------------------------------------------------------------------
|
||||||
|
% the solver is free to choose at most one installed hash for each package
|
||||||
|
{ hash(Package, Hash) : installed_hash(Package, Hash) } 1 :- node(Package).
|
||||||
|
|
||||||
|
% if a hash is selected, we impose all the constraints that implies
|
||||||
|
impose(Hash) :- hash(Package, Hash).
|
||||||
|
|
||||||
|
% if we haven't selected a hash for a package, we'll be building it
|
||||||
|
build(Package) :- not hash(Package, _), node(Package).
|
||||||
|
|
||||||
|
#defined installed_hash/2.
|
||||||
|
|
||||||
%-----------------------------------------------------------------------------
|
%-----------------------------------------------------------------------------
|
||||||
% How to optimize the spec (high to low priority)
|
% How to optimize the spec (high to low priority)
|
||||||
%-----------------------------------------------------------------------------
|
%-----------------------------------------------------------------------------
|
||||||
@ -740,12 +769,17 @@ no_flags(Package, FlagType)
|
|||||||
% 2. a `#minimize{ 0@2 : #true }.` statement that ensures the criterion
|
% 2. a `#minimize{ 0@2 : #true }.` statement that ensures the criterion
|
||||||
% is displayed (clingo doesn't display sums over empty sets by default)
|
% is displayed (clingo doesn't display sums over empty sets by default)
|
||||||
|
|
||||||
|
% Try hard to reuse installed packages (i.e., minimize the number built)
|
||||||
|
opt_criterion(17, "number of packages to build (vs. reuse)").
|
||||||
|
#minimize { 0@17 : #true }.
|
||||||
|
#minimize { 1@17,Package : build(Package) }.
|
||||||
|
|
||||||
% Minimize the number of deprecated versions being used
|
% Minimize the number of deprecated versions being used
|
||||||
opt_criterion(16, "deprecated versions used").
|
opt_criterion(16, "deprecated versions used").
|
||||||
#minimize{ 0@16 : #true }.
|
#minimize{ 0@16 : #true }.
|
||||||
#minimize{ 1@16,Package : deprecated(Package, _)}.
|
#minimize{ 1@16,Package : deprecated(Package, _)}.
|
||||||
|
|
||||||
% The highest priority is to minimize the:
|
% Minimize the:
|
||||||
% 1. Version weight
|
% 1. Version weight
|
||||||
% 2. Number of variants with a non default value, if not set
|
% 2. Number of variants with a non default value, if not set
|
||||||
% for the root(Package)
|
% for the root(Package)
|
||||||
|
@ -12,6 +12,7 @@
|
|||||||
% Spec-related functions.
|
% Spec-related functions.
|
||||||
% Used to build the result of the solve.
|
% Used to build the result of the solve.
|
||||||
#show node/1.
|
#show node/1.
|
||||||
|
#show hash/2.
|
||||||
#show depends_on/3.
|
#show depends_on/3.
|
||||||
#show version/2.
|
#show version/2.
|
||||||
#show variant_value/3.
|
#show variant_value/3.
|
||||||
@ -26,6 +27,8 @@
|
|||||||
#show no_flags/2.
|
#show no_flags/2.
|
||||||
#show external_spec_selected/2.
|
#show external_spec_selected/2.
|
||||||
|
|
||||||
|
#show build/1.
|
||||||
|
|
||||||
% names of optimization criteria
|
% names of optimization criteria
|
||||||
#show opt_criterion/2.
|
#show opt_criterion/2.
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user