zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more robust switching between atomic update methods

Browse Source
This commit is contained in:
Gregory Becker 2023-03-21 12:35:37 -07:00
parent 17381f0281
commit f6c895ce10
5 changed files with 52 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/spack/external/archspec/json/cpu/microarchitectures.json vendored
View File

@ -2706,7 +2706,7 @@
],
"clang" : [
{
"versions": "9.0:12.0",
"versions": "9.0:12.99",
"flags" : "-march=armv8.4-a"
},
{

28
lib/spack/spack/environment/environment.py
View File

@ -468,11 +468,11 @@ def from_dict(base_path, d):
@property
def _current_root(self):
if spack.util.atomic_update.use_renameat2():
return self.root
if not os.path.islink(self.root):
return None
if os.path.isdir(self.root):
return self.root
else:
return None
root = os.readlink(self.root)
if os.path.isabs(root):
@ -537,6 +537,7 @@ def view(self, new=None):
ignore_conflicts=True,
projections=self.projections,
link=self.link_type,
final_destination=self.root,
)
def __contains__(self, spec):
@ -580,6 +581,15 @@ def specs_for_view(self, concretized_root_specs):
return specs
def use_renameat2(self):
if os.path.islink(self.root):
return False
elif os.path.isdir(self.root):
if not spack.util.atmoic_update.renameat2:
raise Exception
return bool(spack.util.atomic_update.renameat2)
def regenerate(self, concretized_root_specs):
specs = self.specs_for_view(concretized_root_specs)
@ -592,6 +602,9 @@ def regenerate(self, concretized_root_specs):
# will be /dirname/._basename_<hash>.
# This allows for atomic swaps when we update the view
# Check which atomic update method we need
use_renameat2 = self.use_renameat2()
# cache the roots because the way we determine which is which does
# not work while we are updating
new_root = self._next_root(specs)
@ -601,7 +614,7 @@ def regenerate(self, concretized_root_specs):
tty.debug("View at %s does not need regeneration." % self.root)
return
if spack.util.atomic_update.use_renameat2():
if use_renameat2:
if os.path.isdir(new_root):
shutil.rmtree(new_root)
@ -617,7 +630,10 @@ def regenerate(self, concretized_root_specs):
try:
fs.mkdirp(new_root)
view.add_specs(*specs, with_dependencies=False)
spack.util.atomic_update.atomic_update(new_root, self.root)
if use_renameat2:
spack.util.atomic_update.atomic_update_renameat2(new_root, self.root)
else:
spack.util.atomic_update.atomic_update_symlink(new_root, self.root)
except Exception as e:
# Clean up new view and temporary symlink on any failure.
try:

49
lib/spack/spack/filesystem_view.py
View File

@ -82,18 +82,21 @@ def view_copy(src, dst, view, spec=None):
orig_sbang = "#!/bin/bash {0}/bin/sbang".format(spack.paths.spack_root)
new_sbang = sbang.sbang_shebang_line()
root = view.final_destination
prefix_to_projection = collections.OrderedDict(
{spec.prefix: view.get_projection_for_spec(spec)}
{spec.prefix: os.path.join(root, view.get_relative_projection_for_spec(spec))}
)
for dep in spec.traverse():
if not dep.external:
prefix_to_projection[dep.prefix] = view.get_projection_for_spec(dep)
prefix_to_projection[dep.prefix] = os.path.join(
root, view.get_relative_projection_for_spec(dep)
)
if spack.relocate.is_binary(dst):
spack.relocate.relocate_text_bin(binaries=[dst], prefixes=prefix_to_projection)
else:
prefix_to_projection[spack.store.layout.root] = view._root
prefix_to_projection[spack.store.layout.root] = root
prefix_to_projection[orig_sbang] = new_sbang
spack.relocate.relocate_text(files=[dst], prefixes=prefix_to_projection)
try:
@ -154,6 +157,8 @@ def __init__(self, root, layout, **kwargs):
self.ignore_conflicts = kwargs.get("ignore_conflicts", False)
self.verbose = kwargs.get("verbose", False)
self.final_destination = kwargs.get("final_destination", self._root)
# Setup link function to include view
link_func = kwargs.get("link", view_symlink)
self.link = ft.partial(link_func, view=self)
@ -210,12 +215,15 @@ def remove_standalone(self, spec):
"""
raise NotImplementedError
def get_projection_for_spec(self, spec):
def get_relative_projection_for_spec(self, spec):
"""
Get the projection in this view for a spec.
Get the relative projection in this view for a spec.
"""
raise NotImplementedError
def get_projection_for_spec(self, spec):
return os.path.join(self._root, self.get_relative_projection_for_spec(spec))
def get_all_specs(self):
"""
Get all specs currently active in this view.
@ -486,9 +494,9 @@ def remove_standalone(self, spec):
if self.verbose:
tty.info(self._croot + "Removed package: %s" % colorize_spec(spec))
def get_projection_for_spec(self, spec):
def get_relative_projection_for_spec(self, spec):
"""
Return the projection for a spec in this view.
Return the relative projection for a spec in this view.
Relies on the ordering of projections to avoid ambiguity.
"""
@ -499,9 +507,7 @@ def get_projection_for_spec(self, spec):
locator_spec = spec.package.extendee_spec
proj = spack.projections.get_projection(self.projections, locator_spec)
if proj:
return os.path.join(self._root, locator_spec.format(proj))
return self._root
return spec.format(proj) if proj else ""
def get_all_specs(self):
md_dirs = []
@ -765,6 +771,13 @@ def link_metadata(self, specs):
self.link(os.path.join(src_root, src_relpath), os.path.join(self._root, dst_relpath))
def get_relative_projection_for_spec(self, spec):
"""
Return the relative projection for a spec in this view.
Relies on the ordering of projections to avoid ambiguity.
"""
spec = spack.spec.Spec(spec)
# Extensions are placed by their extendee, not by their own spec
if spec.package.extendee_spec:
spec = spec.package.extendee_spec
@ -772,22 +785,6 @@ def get_relative_projection_for_spec(self, spec):
p = spack.projections.get_projection(self.projections, spec)
return spec.format(p) if p else ""
def get_projection_for_spec(self, spec):
"""
Return the projection for a spec in this view.
Relies on the ordering of projections to avoid ambiguity.
"""
spec = spack.spec.Spec(spec)
if spec.package.extendee_spec:
spec = spec.package.extendee_spec
proj = spack.projections.get_projection(self.projections, spec)
if proj:
return os.path.join(self._root, spec.format(proj))
return self._root
#####################
# utility functions #

7
lib/spack/spack/test/cmd/env.py
View File

@ -52,7 +52,7 @@
sep = os.sep
if spack.util.atomic_update.use_renameat2():
if spack.util.atomic_update.renameat2:
use_renameat2 = [True, False]
else:
use_renameat2 = [False]
@ -60,7 +60,8 @@
@pytest.fixture(params=use_renameat2)
def atomic_update_implementations(request, monkeypatch):
monkeypatch.setattr(spack.util.atomic_update, "use_renameat2", lambda: request.param)
if request.param is False:
monkeypatch.setattr(spack.util.atomic_update, "renameat2", None)
yield
@ -2901,7 +2902,7 @@ def test_environment_view_target_already_exists(
the new view dir already exists. If so, it should not be
removed or modified."""
# Only works for symlinked atomic views
monkeypatch.setattr(spack.util.atomic_update, "use_renameat2", lambda: False)
monkeypatch.setattr(spack.util.atomic_update, "renameat2", None)
# Create a new environment
view = str(tmpdir.join("view"))

6
lib/spack/spack/util/atomic_update.py
View File

@ -22,9 +22,7 @@
except BaseException:
pass
def use_renameat2():
return hasattr(libc, "renameat2")
renameat2 = getattr(libc, "renameat2", None)
def atomic_update(oldpath, newpath):
@ -35,7 +33,7 @@ def atomic_update(oldpath, newpath):
on other systems, oldpath is not affected but all paths are abstracted
by a symlink to allow for atomic updates.
"""
if use_renameat2():
if has_renameat2():
return atomic_update_renameat2(oldpath, newpath)
else:
return atomic_update_symlink(oldpath, newpath)