spack env track command (#41897)
This PR adds a sub-command to `spack env` (`track`) which allows users to add/link anonymous environments into their installation as named environments. This allows users to more easily track their installed packages and the environments they're dependencies of. For example, with the addition of #41731 it's now easier to remove all packages not required by any environments with, ``` spack gc -bE ``` #### Usage ``` spack env track /path/to/env ==> Linked environment in /path/to/env ==> You can activate this environment with: ==> spack env activate env ``` By default `track /path/to/env` will use the last directory in the path as the name of the environment. However users may customize the name of the linked environment with `-n | --name`. Shown below. ``` spack env track /path/to/env --name foo ==> Tracking environment in /path/to/env ==> You can activate this environment with: ==> spack env activate foo ``` When removing a linked environment, Spack will remove the link to the environment but will keep the structure of the environment within the directory. This will allow users to remove a linked environment from their installation without deleting it from a shared repository. There is a `spack env untrack` command that can be used to *only* untrack a tracked environment -- it will fail if it is used on a managed environment. Users can also use `spack env remove` to untrack an environment. This allows users to continue to share environments in git repositories while also having the dependencies of those environments be remembered by Spack. --------- Co-authored-by: Todd Gamblin <tgamblin@llnl.gov>
This commit is contained in:
Alec Scott
GitHub
parent
ed916ffe6c
commit
ff26d2f833
@ -10,11 +10,12 @@
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, Set
|
||||
|
||||
import llnl.string as string
|
||||
import llnl.util.filesystem as fs
|
||||
import llnl.util.tty as tty
|
||||
from llnl.util.symlink import islink, symlink
|
||||
from llnl.util.tty.colify import colify
|
||||
from llnl.util.tty.color import cescape, colorize
|
||||
|
||||
@ -50,6 +51,8 @@
|
||||
"update",
|
||||
"revert",
|
||||
"depfile",
|
||||
"track",
|
||||
"untrack",
|
||||
]
|
||||
|
||||
|
||||
@ -446,6 +449,193 @@ def env_deactivate(args):
|
||||
sys.stdout.write(cmds)
|
||||
|
||||
|
||||
#
|
||||
# env track
|
||||
#
|
||||
def env_track_setup_parser(subparser):
|
||||
"""track an environment from a directory in Spack"""
|
||||
subparser.add_argument("-n", "--name", help="custom environment name")
|
||||
subparser.add_argument("dir", help="path to environment")
|
||||
arguments.add_common_arguments(subparser, ["yes_to_all"])
|
||||
|
||||
|
||||
def env_track(args):
|
||||
src_path = os.path.abspath(args.dir)
|
||||
if not ev.is_env_dir(src_path):
|
||||
tty.die("Cannot track environment. Path doesn't contain an environment")
|
||||
|
||||
if args.name:
|
||||
name = args.name
|
||||
else:
|
||||
name = os.path.basename(src_path)
|
||||
|
||||
try:
|
||||
dst_path = ev.environment_dir_from_name(name, exists_ok=False)
|
||||
except ev.SpackEnvironmentError:
|
||||
tty.die(
|
||||
f"An environment named {name} already exists. Set a name with:"
|
||||
"\n\n"
|
||||
f" spack env track --name NAME {src_path}\n"
|
||||
)
|
||||
|
||||
symlink(src_path, dst_path)
|
||||
|
||||
tty.msg(f"Tracking environment in {src_path}")
|
||||
tty.msg(
|
||||
"You can now activate this environment with the following command:\n\n"
|
||||
f" spack env activate {name}\n"
|
||||
)
|
||||
|
||||
|
||||
#
|
||||
# env remove & untrack helpers
|
||||
#
|
||||
def filter_managed_env_names(env_names: Set[str]) -> Set[str]:
|
||||
tracked_env_names = {e for e in env_names if islink(ev.environment_dir_from_name(e))}
|
||||
managed_env_names = env_names - set(tracked_env_names)
|
||||
|
||||
num_managed_envs = len(managed_env_names)
|
||||
managed_envs_str = " ".join(managed_env_names)
|
||||
if num_managed_envs >= 2:
|
||||
tty.error(
|
||||
f"The following are not tracked environments. "
|
||||
"To remove them completely run,"
|
||||
"\n\n"
|
||||
f" spack env rm {managed_envs_str}\n"
|
||||
)
|
||||
|
||||
elif num_managed_envs > 0:
|
||||
tty.error(
|
||||
f"'{managed_envs_str}' is not a tracked env. "
|
||||
"To remove it completely run,"
|
||||
"\n\n"
|
||||
f" spack env rm {managed_envs_str}\n"
|
||||
)
|
||||
|
||||
return tracked_env_names
|
||||
|
||||
|
||||
def get_valid_envs(env_names: Set[str]) -> Set[ev.Environment]:
|
||||
valid_envs = set()
|
||||
for env_name in env_names:
|
||||
try:
|
||||
env = ev.read(env_name)
|
||||
valid_envs.add(env)
|
||||
|
||||
except (spack.config.ConfigFormatError, ev.SpackEnvironmentConfigError):
|
||||
pass
|
||||
|
||||
return valid_envs
|
||||
|
||||
|
||||
def _env_untrack_or_remove(
|
||||
env_names: List[str], remove: bool = False, force: bool = False, yes_to_all: bool = False
|
||||
):
|
||||
all_env_names = set(ev.all_environment_names())
|
||||
known_env_names = set(env_names).intersection(all_env_names)
|
||||
unknown_env_names = set(env_names) - known_env_names
|
||||
|
||||
# print error for unknown environments
|
||||
for env_name in unknown_env_names:
|
||||
tty.error(f"Environment '{env_name}' does not exist")
|
||||
|
||||
# if only unlinking is allowed, remove all environments
|
||||
# which do not point internally at symlinks
|
||||
if not remove:
|
||||
env_names_to_remove = filter_managed_env_names(known_env_names)
|
||||
else:
|
||||
env_names_to_remove = known_env_names
|
||||
|
||||
# initalize all environments with valid spack.yaml configs
|
||||
all_valid_envs = get_valid_envs(all_env_names)
|
||||
|
||||
# build a task list of environments and bad env names to remove
|
||||
envs_to_remove = [e for e in all_valid_envs if e.name in env_names_to_remove]
|
||||
bad_env_names_to_remove = env_names_to_remove - {e.name for e in envs_to_remove}
|
||||
for remove_env in envs_to_remove:
|
||||
for env in all_valid_envs:
|
||||
# don't check if an environment is included to itself
|
||||
if env.name == remove_env.name:
|
||||
continue
|
||||
|
||||
# check if an environment is included un another
|
||||
if remove_env.path in env.included_concrete_envs:
|
||||
msg = f"Environment '{remove_env.name}' is used by environment '{env.name}'"
|
||||
if force:
|
||||
tty.warn(msg)
|
||||
else:
|
||||
tty.error(msg)
|
||||
envs_to_remove.remove(remove_env)
|
||||
|
||||
# ask the user if they really want to remove the known environments
|
||||
# force should do the same as yes to all here following the symantics of rm
|
||||
if not (yes_to_all or force) and (envs_to_remove or bad_env_names_to_remove):
|
||||
environments = string.plural(len(env_names_to_remove), "environment", show_n=False)
|
||||
envs = string.comma_and(list(env_names_to_remove))
|
||||
answer = tty.get_yes_or_no(
|
||||
f"Really {'remove' if remove else 'untrack'} {environments} {envs}?", default=False
|
||||
)
|
||||
if not answer:
|
||||
tty.die("Will not remove any environments")
|
||||
|
||||
# keep track of the environments we remove for later printing the exit code
|
||||
removed_env_names = []
|
||||
for env in envs_to_remove:
|
||||
name = env.name
|
||||
if not force and env.active:
|
||||
tty.error(
|
||||
f"Environment '{name}' can't be "
|
||||
f"{'removed' if remove else 'untracked'} while activated."
|
||||
)
|
||||
continue
|
||||
# Get path to check if environment is a tracked / symlinked environment
|
||||
if islink(env.path):
|
||||
real_env_path = os.path.realpath(env.path)
|
||||
os.unlink(env.path)
|
||||
tty.msg(
|
||||
f"Sucessfully untracked environment '{name}', "
|
||||
"but it can still be found at:\n\n"
|
||||
f" {real_env_path}\n"
|
||||
)
|
||||
else:
|
||||
env.destroy()
|
||||
tty.msg(f"Successfully removed environment '{name}'")
|
||||
|
||||
removed_env_names.append(env.name)
|
||||
|
||||
for bad_env_name in bad_env_names_to_remove:
|
||||
shutil.rmtree(
|
||||
spack.environment.environment.environment_dir_from_name(bad_env_name, exists_ok=True)
|
||||
)
|
||||
tty.msg(f"Successfully removed environment '{bad_env_name}'")
|
||||
removed_env_names.append(env.name)
|
||||
|
||||
# Following the design of linux rm we should exit with a status of 1
|
||||
# anytime we cannot delete every environment the user asks for.
|
||||
# However, we should still process all the environments we know about
|
||||
# and delete them instead of failing on the first unknown enviornment.
|
||||
if len(removed_env_names) < len(known_env_names):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
#
|
||||
# env untrack
|
||||
#
|
||||
def env_untrack_setup_parser(subparser):
|
||||
"""track an environment from a directory in Spack"""
|
||||
subparser.add_argument("env", nargs="+", help="tracked environment name")
|
||||
subparser.add_argument(
|
||||
"-f", "--force", action="store_true", help="force unlink even when environment is active"
|
||||
)
|
||||
arguments.add_common_arguments(subparser, ["yes_to_all"])
|
||||
|
||||
|
||||
def env_untrack(args):
|
||||
_env_untrack_or_remove(
|
||||
env_names=args.env, force=args.force, yes_to_all=args.yes_to_all, remove=False
|
||||
)
|
||||
|
||||
|
||||
#
|
||||
# env remove
|
||||
#
|
||||
@ -471,54 +661,9 @@ def env_remove_setup_parser(subparser):
|
||||
|
||||
def env_remove(args):
|
||||
"""remove existing environment(s)"""
|
||||
remove_envs = []
|
||||
valid_envs = []
|
||||
bad_envs = []
|
||||
|
||||
for env_name in ev.all_environment_names():
|
||||
try:
|
||||
env = ev.read(env_name)
|
||||
valid_envs.append(env)
|
||||
|
||||
if env_name in args.rm_env:
|
||||
remove_envs.append(env)
|
||||
except (spack.config.ConfigFormatError, ev.SpackEnvironmentConfigError):
|
||||
if env_name in args.rm_env:
|
||||
bad_envs.append(env_name)
|
||||
|
||||
# Check if remove_env is included from another env before trying to remove
|
||||
for env in valid_envs:
|
||||
for remove_env in remove_envs:
|
||||
# don't check if environment is included to itself
|
||||
if env.name == remove_env.name:
|
||||
continue
|
||||
|
||||
if remove_env.path in env.included_concrete_envs:
|
||||
msg = f'Environment "{remove_env.name}" is being used by environment "{env.name}"'
|
||||
if args.force:
|
||||
tty.warn(msg)
|
||||
else:
|
||||
tty.die(msg)
|
||||
|
||||
if not args.yes_to_all:
|
||||
environments = string.plural(len(args.rm_env), "environment", show_n=False)
|
||||
envs = string.comma_and(args.rm_env)
|
||||
answer = tty.get_yes_or_no(f"Really remove {environments} {envs}?", default=False)
|
||||
if not answer:
|
||||
tty.die("Will not remove any environments")
|
||||
|
||||
for env in remove_envs:
|
||||
name = env.name
|
||||
if env.active:
|
||||
tty.die(f"Environment {name} can't be removed while activated.")
|
||||
env.destroy()
|
||||
tty.msg(f"Successfully removed environment '{name}'")
|
||||
|
||||
for bad_env_name in bad_envs:
|
||||
shutil.rmtree(
|
||||
spack.environment.environment.environment_dir_from_name(bad_env_name, exists_ok=True)
|
||||
)
|
||||
tty.msg(f"Successfully removed environment '{bad_env_name}'")
|
||||
_env_untrack_or_remove(
|
||||
env_names=args.rm_env, remove=True, force=args.force, yes_to_all=args.yes_to_all
|
||||
)
|
||||
|
||||
|
||||
#
|
||||
|
||||
@ -20,7 +20,7 @@
|
||||
import llnl.util.tty as tty
|
||||
import llnl.util.tty.color as clr
|
||||
from llnl.util.link_tree import ConflictingSpecsError
|
||||
from llnl.util.symlink import readlink, symlink
|
||||
from llnl.util.symlink import islink, readlink, symlink
|
||||
|
||||
import spack
|
||||
import spack.caches
|
||||
@ -668,7 +668,7 @@ def from_dict(base_path, d):
|
||||
|
||||
@property
|
||||
def _current_root(self):
|
||||
if not os.path.islink(self.root):
|
||||
if not islink(self.root):
|
||||
return None
|
||||
|
||||
root = readlink(self.root)
|
||||
|
||||
@ -117,6 +117,99 @@ def check_viewdir_removal(viewdir):
|
||||
) == ["projections.yaml"]
|
||||
|
||||
|
||||
def test_env_track_nonexistant_path_fails(capfd):
|
||||
with pytest.raises(spack.main.SpackCommandError):
|
||||
env("track", "path/does/not/exist")
|
||||
|
||||
out, _ = capfd.readouterr()
|
||||
assert "doesn't contain an environment" in out
|
||||
|
||||
|
||||
def test_env_track_existing_env_fails(capfd):
|
||||
env("create", "track_test")
|
||||
|
||||
with pytest.raises(spack.main.SpackCommandError):
|
||||
env("track", "--name", "track_test", ev.environment_dir_from_name("track_test"))
|
||||
|
||||
out, _ = capfd.readouterr()
|
||||
assert "environment named track_test already exists" in out
|
||||
|
||||
|
||||
def test_env_track_valid(tmp_path):
|
||||
with fs.working_dir(str(tmp_path)):
|
||||
# create an independent environment
|
||||
env("create", "-d", ".")
|
||||
|
||||
# test tracking an environment in known store
|
||||
env("track", "--name", "test1", ".")
|
||||
|
||||
# test removing environment to ensure independent isn't deleted
|
||||
env("rm", "-y", "test1")
|
||||
|
||||
assert os.path.isfile("spack.yaml")
|
||||
|
||||
|
||||
def test_env_untrack_valid(tmp_path):
|
||||
with fs.working_dir(str(tmp_path)):
|
||||
# create an independent environment
|
||||
env("create", "-d", ".")
|
||||
|
||||
# test tracking an environment in known store
|
||||
env("track", "--name", "test_untrack", ".")
|
||||
env("untrack", "--yes-to-all", "test_untrack")
|
||||
|
||||
# check that environment was sucessfully untracked
|
||||
out = env("ls")
|
||||
assert "test_untrack" not in out
|
||||
|
||||
|
||||
def test_env_untrack_invalid_name():
|
||||
# test untracking an environment that doesn't exist
|
||||
env_name = "invalid_enviornment_untrack"
|
||||
|
||||
out = env("untrack", env_name)
|
||||
|
||||
assert f"Environment '{env_name}' does not exist" in out
|
||||
|
||||
|
||||
def test_env_untrack_when_active(tmp_path, capfd):
|
||||
env_name = "test_untrack_active"
|
||||
|
||||
with fs.working_dir(str(tmp_path)):
|
||||
# create an independent environment
|
||||
env("create", "-d", ".")
|
||||
|
||||
# test tracking an environment in known store
|
||||
env("track", "--name", env_name, ".")
|
||||
|
||||
active_env = ev.read(env_name)
|
||||
with active_env:
|
||||
with pytest.raises(spack.main.SpackCommandError):
|
||||
env("untrack", "--yes-to-all", env_name)
|
||||
|
||||
# check that environment could not be untracked while active
|
||||
out, _ = capfd.readouterr()
|
||||
assert f"'{env_name}' can't be untracked while activated" in out
|
||||
|
||||
env("untrack", "-f", env_name)
|
||||
out = env("ls")
|
||||
assert env_name not in out
|
||||
|
||||
|
||||
def test_env_untrack_managed(tmp_path, capfd):
|
||||
env_name = "test_untrack_managed"
|
||||
|
||||
# create an managed environment
|
||||
env("create", env_name)
|
||||
|
||||
with pytest.raises(spack.main.SpackCommandError):
|
||||
env("untrack", env_name)
|
||||
|
||||
# check that environment could not be untracked while active
|
||||
out, _ = capfd.readouterr()
|
||||
assert f"'{env_name}' is not a tracked env" in out
|
||||
|
||||
|
||||
def test_add():
|
||||
e = ev.create("test")
|
||||
e.add("mpileaks")
|
||||
@ -128,6 +221,7 @@ def test_change_match_spec():
|
||||
|
||||
e = ev.read("test")
|
||||
with e:
|
||||
|
||||
add("mpileaks@2.1")
|
||||
add("mpileaks@2.2")
|
||||
|
||||
@ -688,7 +782,7 @@ def test_force_remove_included_env():
|
||||
rm_output = env("remove", "-f", "-y", "test")
|
||||
list_output = env("list")
|
||||
|
||||
assert '"test" is being used by environment "combined_env"' in rm_output
|
||||
assert "'test' is used by environment 'combined_env'" in rm_output
|
||||
assert "test" not in list_output
|
||||
|
||||
|
||||
@ -4239,13 +4333,13 @@ def test_spack_package_ids_variable(tmpdir, mock_packages):
|
||||
# Include in Makefile and create target that depend on SPACK_PACKAGE_IDS
|
||||
with open(makefile_path, "w") as f:
|
||||
f.write(
|
||||
r"""
|
||||
"""
|
||||
all: post-install
|
||||
|
||||
include include.mk
|
||||
|
||||
example/post-install/%: example/install/%
|
||||
$(info post-install: $(HASH)) # noqa: W191,E101
|
||||
\t$(info post-install: $(HASH)) # noqa: W191,E101
|
||||
|
||||
post-install: $(addprefix example/post-install/,$(example/SPACK_PACKAGE_IDS))
|
||||
"""
|
||||
|
||||
@ -1023,7 +1023,7 @@ _spack_env() {
|
||||
then
|
||||
SPACK_COMPREPLY="-h --help"
|
||||
else
|
||||
SPACK_COMPREPLY="activate deactivate create remove rm rename mv list ls status st loads view update revert depfile"
|
||||
SPACK_COMPREPLY="activate deactivate create remove rm rename mv list ls status st loads view update revert depfile track untrack"
|
||||
fi
|
||||
}
|
||||
|
||||
@ -1141,6 +1141,24 @@ _spack_env_depfile() {
|
||||
fi
|
||||
}
|
||||
|
||||
_spack_env_track() {
|
||||
if $list_options
|
||||
then
|
||||
SPACK_COMPREPLY="-h --help -n --name -y --yes-to-all"
|
||||
else
|
||||
SPACK_COMPREPLY=""
|
||||
fi
|
||||
}
|
||||
|
||||
_spack_env_untrack() {
|
||||
if $list_options
|
||||
then
|
||||
SPACK_COMPREPLY="-h --help -f --force -y --yes-to-all"
|
||||
else
|
||||
_environments
|
||||
fi
|
||||
}
|
||||
|
||||
_spack_extensions() {
|
||||
if $list_options
|
||||
then
|
||||
|
||||
@ -1488,6 +1488,8 @@ complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a view -d 'manag
|
||||
complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a update -d 'update the environment manifest to the latest schema format'
|
||||
complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a revert -d 'restore the environment manifest to its previous format'
|
||||
complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a depfile -d 'generate a depfile to exploit parallel builds across specs'
|
||||
complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a track -d 'track an environment from a directory in Spack'
|
||||
complete -c spack -n '__fish_spack_using_command_pos 0 env' -f -a untrack -d 'track an environment from a directory in Spack'
|
||||
complete -c spack -n '__fish_spack_using_command env' -s h -l help -f -a help
|
||||
complete -c spack -n '__fish_spack_using_command env' -s h -l help -d 'show this help message and exit'
|
||||
|
||||
@ -1669,6 +1671,26 @@ complete -c spack -n '__fish_spack_using_command env depfile' -s o -l output -r
|
||||
complete -c spack -n '__fish_spack_using_command env depfile' -s G -l generator -r -f -a make
|
||||
complete -c spack -n '__fish_spack_using_command env depfile' -s G -l generator -r -d 'specify the depfile type (only supports `make`)'
|
||||
|
||||
# spack env track
|
||||
set -g __fish_spack_optspecs_spack_env_track h/help n/name= y/yes-to-all
|
||||
complete -c spack -n '__fish_spack_using_command_pos 0 env track' -f -a '(__fish_spack_environments)'
|
||||
complete -c spack -n '__fish_spack_using_command env track' -s h -l help -f -a help
|
||||
complete -c spack -n '__fish_spack_using_command env track' -s h -l help -d 'show this help message and exit'
|
||||
complete -c spack -n '__fish_spack_using_command env track' -s n -l name -r -f -a name
|
||||
complete -c spack -n '__fish_spack_using_command env track' -s n -l name -r -d 'custom environment name'
|
||||
complete -c spack -n '__fish_spack_using_command env track' -s y -l yes-to-all -f -a yes_to_all
|
||||
complete -c spack -n '__fish_spack_using_command env track' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
|
||||
|
||||
# spack env untrack
|
||||
set -g __fish_spack_optspecs_spack_env_untrack h/help f/force y/yes-to-all
|
||||
complete -c spack -n '__fish_spack_using_command_pos_remainder 0 env untrack' -f -a '(__fish_spack_environments)'
|
||||
complete -c spack -n '__fish_spack_using_command env untrack' -s h -l help -f -a help
|
||||
complete -c spack -n '__fish_spack_using_command env untrack' -s h -l help -d 'show this help message and exit'
|
||||
complete -c spack -n '__fish_spack_using_command env untrack' -s f -l force -f -a force
|
||||
complete -c spack -n '__fish_spack_using_command env untrack' -s f -l force -d 'force unlink even when environment is active'
|
||||
complete -c spack -n '__fish_spack_using_command env untrack' -s y -l yes-to-all -f -a yes_to_all
|
||||
complete -c spack -n '__fish_spack_using_command env untrack' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
|
||||
|
||||
# spack extensions
|
||||
set -g __fish_spack_optspecs_spack_extensions h/help l/long L/very-long d/deps p/paths s/show=
|
||||
complete -c spack -n '__fish_spack_using_command_pos_remainder 0 extensions' -f -a '(__fish_spack_extensions)'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user