performance: only regenerate env views once in spack install
`spack install` previously concretized, writes the entire environment out, regenerated views, then wrote and regenerated views again. Regenerating views is slow, so ensure that we only do that once. - [x] add an option to env.write() to skip view regeneration - [x] add a note on whether regenerate_views() shouldn't just be a separate operation -- not clear if we want to keep it as part of write to ensure consistency, or take it out to avoid performance issues.
This commit is contained in:
parent
0fb3280011
commit
c83e365c59
@ -259,9 +259,14 @@ def install(parser, args, **kwargs):
|
|||||||
if not args.only_concrete:
|
if not args.only_concrete:
|
||||||
concretized_specs = env.concretize()
|
concretized_specs = env.concretize()
|
||||||
ev.display_specs(concretized_specs)
|
ev.display_specs(concretized_specs)
|
||||||
env.write()
|
|
||||||
|
# save view regeneration for later, so that we only do it
|
||||||
|
# once, as it can be slow.
|
||||||
|
env.write(regenerate_views=False)
|
||||||
|
|
||||||
tty.msg("Installing environment %s" % env.name)
|
tty.msg("Installing environment %s" % env.name)
|
||||||
env.install_all(args)
|
env.install_all(args)
|
||||||
|
env.regenerate_views()
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
tty.die("install requires a package argument or a spack.yaml file")
|
tty.die("install requires a package argument or a spack.yaml file")
|
||||||
|
@ -1182,7 +1182,12 @@ def _add_concrete_spec(self, spec, concrete, new=True):
|
|||||||
self.specs_by_hash[h] = concrete
|
self.specs_by_hash[h] = concrete
|
||||||
|
|
||||||
def install_all(self, args=None):
|
def install_all(self, args=None):
|
||||||
"""Install all concretized specs in an environment."""
|
"""Install all concretized specs in an environment.
|
||||||
|
|
||||||
|
Note: this does not regenerate the views for the environment;
|
||||||
|
that needs to be done separately with a call to write().
|
||||||
|
|
||||||
|
"""
|
||||||
with spack.store.db.read_transaction():
|
with spack.store.db.read_transaction():
|
||||||
for concretized_hash in self.concretized_order:
|
for concretized_hash in self.concretized_order:
|
||||||
spec = self.specs_by_hash[concretized_hash]
|
spec = self.specs_by_hash[concretized_hash]
|
||||||
@ -1203,8 +1208,6 @@ def install_all(self, args=None):
|
|||||||
os.remove(build_log_link)
|
os.remove(build_log_link)
|
||||||
os.symlink(spec.package.build_log_path, build_log_link)
|
os.symlink(spec.package.build_log_path, build_log_link)
|
||||||
|
|
||||||
self.regenerate_views()
|
|
||||||
|
|
||||||
def all_specs_by_hash(self):
|
def all_specs_by_hash(self):
|
||||||
"""Map of hashes to spec for all specs in this environment."""
|
"""Map of hashes to spec for all specs in this environment."""
|
||||||
# Note this uses dag-hashes calculated without build deps as keys,
|
# Note this uses dag-hashes calculated without build deps as keys,
|
||||||
@ -1370,10 +1373,17 @@ def _read_lockfile_dict(self, d):
|
|||||||
self.concretized_order = [
|
self.concretized_order = [
|
||||||
old_hash_to_new.get(h, h) for h in self.concretized_order]
|
old_hash_to_new.get(h, h) for h in self.concretized_order]
|
||||||
|
|
||||||
def write(self):
|
def write(self, regenerate_views=True):
|
||||||
"""Writes an in-memory environment to its location on disk.
|
"""Writes an in-memory environment to its location on disk.
|
||||||
|
|
||||||
This will also write out package files for each newly concretized spec.
|
Write out package files for each newly concretized spec. Also
|
||||||
|
regenerate any views associated with the environment, if
|
||||||
|
regenerate_views is True.
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
regenerate_views (bool): regenerate views as well as
|
||||||
|
writing if True.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
# ensure path in var/spack/environments
|
# ensure path in var/spack/environments
|
||||||
fs.mkdirp(self.path)
|
fs.mkdirp(self.path)
|
||||||
@ -1481,8 +1491,13 @@ def write(self):
|
|||||||
with fs.write_tmp_and_move(self.manifest_path) as f:
|
with fs.write_tmp_and_move(self.manifest_path) as f:
|
||||||
_write_yaml(self.yaml, f)
|
_write_yaml(self.yaml, f)
|
||||||
|
|
||||||
# TODO: for operations that just add to the env (install etc.) this
|
# TODO: rethink where this needs to happen along with
|
||||||
# could just call update_view
|
# writing. For some of the commands (like install, which write
|
||||||
|
# concrete specs AND regen) this might as well be a separate
|
||||||
|
# call. But, having it here makes the views consistent witht the
|
||||||
|
# concretized environment for most operations. Which is the
|
||||||
|
# special case?
|
||||||
|
if regenerate_views:
|
||||||
self.regenerate_views()
|
self.regenerate_views()
|
||||||
|
|
||||||
def __enter__(self):
|
def __enter__(self):
|
||||||
|
Loading…
Reference in New Issue
Block a user