New command, spack change, to change existing env specs (#31995)
If you have an environment like ``` $ cat spack.yaml spack: specs: [openmpi@4.1.0+cuda] ``` this PR provides a new command `spack change` that you can use to adjust environment specs from the command line: ``` $ spack change openmpi~cuda $ cat spack.yaml spack: specs: [openmpi@4.1.0~cuda] ``` in other words, this allows you to tweak the details of environment specs from the command line. Notes: * This is only allowed for environments that do not define matrices * This is possible but not anticipated to be needed immediately * If this were done, it should probably only be done for "named"/not-anonymous specs (i.e. we can change `openmpi+cuda` but not spec like `+cuda` or `@4.0.1~cuda`)
This commit is contained in:
Peter Scheibel
GitHub
parent
92b72f186e
commit
2968ae667f
51
lib/spack/spack/cmd/change.py
Normal file
51
lib/spack/spack/cmd/change.py
Normal file
@ -0,0 +1,51 @@
|
||||
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import spack.cmd
|
||||
import spack.cmd.common.arguments as arguments
|
||||
|
||||
description = "change an existing spec in an environment"
|
||||
section = "environments"
|
||||
level = "long"
|
||||
|
||||
|
||||
def setup_parser(subparser):
|
||||
subparser.add_argument(
|
||||
"-l",
|
||||
"--list-name",
|
||||
dest="list_name",
|
||||
default="specs",
|
||||
help="name of the list to remove specs from",
|
||||
)
|
||||
subparser.add_argument(
|
||||
"--match-spec",
|
||||
dest="match_spec",
|
||||
help="if name is ambiguous, supply a spec to match",
|
||||
)
|
||||
subparser.add_argument(
|
||||
"-a",
|
||||
"--all",
|
||||
action="store_true",
|
||||
help="change all matching specs (allow changing more than one spec)",
|
||||
)
|
||||
arguments.add_common_arguments(subparser, ["specs"])
|
||||
|
||||
|
||||
def change(parser, args):
|
||||
env = spack.cmd.require_active_env(cmd_name="change")
|
||||
|
||||
with env.write_transaction():
|
||||
if args.match_spec:
|
||||
match_spec = spack.spec.Spec(args.match_spec)
|
||||
else:
|
||||
match_spec = None
|
||||
for spec in spack.cmd.parse_specs(args.specs):
|
||||
env.change_existing_spec(
|
||||
spec,
|
||||
list_name=args.list_name,
|
||||
match_spec=match_spec,
|
||||
allow_changing_multiple_specs=args.all,
|
||||
)
|
||||
env.write()
|
||||
@ -1073,6 +1073,58 @@ def add(self, user_spec, list_name=user_speclist_name):
|
||||
|
||||
return bool(not existing)
|
||||
|
||||
def change_existing_spec(
|
||||
self,
|
||||
change_spec,
|
||||
list_name=user_speclist_name,
|
||||
match_spec=None,
|
||||
allow_changing_multiple_specs=False,
|
||||
):
|
||||
"""
|
||||
Find the spec identified by `match_spec` and change it to `change_spec`.
|
||||
|
||||
Arguments:
|
||||
change_spec (spack.spec.Spec): defines the spec properties that
|
||||
need to be changed. This will not change attributes of the
|
||||
matched spec unless they conflict with `change_spec`.
|
||||
list_name (str): identifies the spec list in the environment that
|
||||
should be modified
|
||||
match_spec (spack.spec.Spec): if set, this identifies the spec
|
||||
that should be changed. If not set, it is assumed we are
|
||||
looking for a spec with the same name as `change_spec`.
|
||||
"""
|
||||
if not (change_spec.name or (match_spec and match_spec.name)):
|
||||
raise ValueError(
|
||||
"Must specify a spec name to identify a single spec"
|
||||
" in the environment that will be changed"
|
||||
)
|
||||
match_spec = match_spec or Spec(change_spec.name)
|
||||
|
||||
list_to_change = self.spec_lists[list_name]
|
||||
if list_to_change.is_matrix:
|
||||
raise SpackEnvironmentError(
|
||||
"Cannot directly change specs in matrices:"
|
||||
" specify a named list that is not a matrix"
|
||||
)
|
||||
|
||||
matches = list(x for x in list_to_change if x.satisfies(match_spec))
|
||||
if len(matches) == 0:
|
||||
raise ValueError(
|
||||
"There are no specs named {0} in {1}".format(match_spec.name, list_name)
|
||||
)
|
||||
elif len(matches) > 1 and not allow_changing_multiple_specs:
|
||||
raise ValueError("{0} matches multiple specs".format(str(match_spec)))
|
||||
|
||||
new_speclist = SpecList(list_name)
|
||||
for i, spec in enumerate(list_to_change):
|
||||
if spec.satisfies(match_spec):
|
||||
new_speclist.add(Spec.override(spec, change_spec))
|
||||
else:
|
||||
new_speclist.add(spec)
|
||||
|
||||
self.spec_lists[list_name] = new_speclist
|
||||
self.update_stale_references()
|
||||
|
||||
def remove(self, query_spec, list_name=user_speclist_name, force=False):
|
||||
"""Remove specs from an environment that match a query_spec"""
|
||||
query_spec = Spec(query_spec)
|
||||
|
||||
@ -284,6 +284,22 @@ def _string_or_none(s):
|
||||
|
||||
self.platform, self.os, self.target = platform_tuple
|
||||
|
||||
@staticmethod
|
||||
def override(init_spec, change_spec):
|
||||
if init_spec:
|
||||
new_spec = init_spec.copy()
|
||||
else:
|
||||
new_spec = ArchSpec()
|
||||
if change_spec.platform:
|
||||
new_spec.platform = change_spec.platform
|
||||
# TODO: if the platform is changed to something that is incompatible
|
||||
# with the current os, we should implicitly remove it
|
||||
if change_spec.os:
|
||||
new_spec.os = change_spec.os
|
||||
if change_spec.target:
|
||||
new_spec.target = change_spec.target
|
||||
return new_spec
|
||||
|
||||
def _autospec(self, spec_like):
|
||||
if isinstance(spec_like, ArchSpec):
|
||||
return spec_like
|
||||
@ -2242,6 +2258,33 @@ def read_yaml_dep_specs(deps, hash_type=ht.dag_hash.name):
|
||||
raise spack.error.SpecError("Couldn't parse dependency types in spec.")
|
||||
yield dep_name, dep_hash, list(deptypes), hash_type
|
||||
|
||||
@staticmethod
|
||||
def override(init_spec, change_spec):
|
||||
# TODO: this doesn't account for the case where the changed spec
|
||||
# (and the user spec) have dependencies
|
||||
new_spec = init_spec.copy()
|
||||
package_cls = spack.repo.path.get_pkg_class(new_spec.name)
|
||||
if change_spec.versions and not change_spec.versions == spack.version.ver(":"):
|
||||
new_spec.versions = change_spec.versions
|
||||
for variant, value in change_spec.variants.items():
|
||||
if variant in package_cls.variants:
|
||||
if variant in new_spec.variants:
|
||||
new_spec.variants.substitute(value)
|
||||
else:
|
||||
new_spec.variants[variant] = value
|
||||
else:
|
||||
raise ValueError("{0} is not a variant of {1}".format(variant, new_spec.name))
|
||||
if change_spec.compiler:
|
||||
new_spec.compiler = change_spec.compiler
|
||||
if change_spec.compiler_flags:
|
||||
for flagname, flagvals in change_spec.compiler_flags.items():
|
||||
new_spec.compiler_flags[flagname] = flagvals
|
||||
if change_spec.architecture:
|
||||
new_spec.architecture = ArchSpec.override(
|
||||
new_spec.architecture, change_spec.architecture
|
||||
)
|
||||
return new_spec
|
||||
|
||||
@staticmethod
|
||||
def from_literal(spec_dict, normal=True):
|
||||
"""Builds a Spec from a dictionary containing the spec literal.
|
||||
|
||||
@ -34,6 +34,13 @@ def __init__(self, name="specs", yaml_list=None, reference=None):
|
||||
self._constraints = None
|
||||
self._specs = None
|
||||
|
||||
@property
|
||||
def is_matrix(self):
|
||||
for item in self.specs_as_yaml_list:
|
||||
if isinstance(item, dict):
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def specs_as_yaml_list(self):
|
||||
if self._expanded_list is None:
|
||||
|
||||
@ -43,6 +43,7 @@
|
||||
env = SpackCommand("env")
|
||||
install = SpackCommand("install")
|
||||
add = SpackCommand("add")
|
||||
change = SpackCommand("change")
|
||||
remove = SpackCommand("remove")
|
||||
concretize = SpackCommand("concretize")
|
||||
stage = SpackCommand("stage")
|
||||
@ -71,6 +72,35 @@ def test_add():
|
||||
assert Spec("mpileaks") in e.user_specs
|
||||
|
||||
|
||||
def test_change_match_spec():
|
||||
env("create", "test")
|
||||
|
||||
e = ev.read("test")
|
||||
with e:
|
||||
add("mpileaks@2.1")
|
||||
add("mpileaks@2.2")
|
||||
|
||||
change("--match-spec", "mpileaks@2.2", "mpileaks@2.3")
|
||||
|
||||
assert not any(x.satisfies("mpileaks@2.2") for x in e.user_specs)
|
||||
assert any(x.satisfies("mpileaks@2.3") for x in e.user_specs)
|
||||
|
||||
|
||||
def test_change_multiple_matches():
|
||||
env("create", "test")
|
||||
|
||||
e = ev.read("test")
|
||||
with e:
|
||||
add("mpileaks@2.1")
|
||||
add("mpileaks@2.2")
|
||||
add("libelf@0.8.12%clang")
|
||||
|
||||
change("--match-spec", "mpileaks", "-a", "mpileaks%gcc")
|
||||
|
||||
assert all(x.satisfies("%gcc") for x in e.user_specs if x.name == "mpileaks")
|
||||
assert any(x.satisfies("%clang") for x in e.user_specs if x.name == "libelf")
|
||||
|
||||
|
||||
def test_env_add_virtual():
|
||||
env("create", "test")
|
||||
|
||||
|
||||
@ -4,16 +4,19 @@
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
"""Test environment internals without CLI"""
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from six import StringIO
|
||||
|
||||
import spack.environment as ev
|
||||
import spack.spec
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
str(spack.platforms.host()) == "windows", reason="Not supported on Windows (yet)"
|
||||
pytestmark = pytest.mark.skipif(
|
||||
sys.platform == "win32", reason="Envs are not supported on windows"
|
||||
)
|
||||
|
||||
|
||||
def test_hash_change_no_rehash_concrete(tmpdir, mock_packages, config):
|
||||
# create an environment
|
||||
env_path = tmpdir.mkdir("env_dir").strpath
|
||||
@ -43,6 +46,66 @@ def test_hash_change_no_rehash_concrete(tmpdir, mock_packages, config):
|
||||
assert read_in.specs_by_hash[read_in.concretized_order[0]]._hash == new_hash
|
||||
|
||||
|
||||
def test_env_change_spec(tmpdir, mock_packages, config):
|
||||
env_path = tmpdir.mkdir("env_dir").strpath
|
||||
env = ev.Environment(env_path)
|
||||
env.write()
|
||||
|
||||
spec = spack.spec.Spec("mpileaks@2.1~shared+debug")
|
||||
env.add(spec)
|
||||
env.write()
|
||||
|
||||
change_spec = spack.spec.Spec("mpileaks@2.2")
|
||||
env.change_existing_spec(change_spec)
|
||||
(spec,) = env.added_specs()
|
||||
assert spec == spack.spec.Spec("mpileaks@2.2~shared+debug")
|
||||
|
||||
change_spec = spack.spec.Spec("mpileaks~debug")
|
||||
env.change_existing_spec(change_spec)
|
||||
(spec,) = env.added_specs()
|
||||
assert spec == spack.spec.Spec("mpileaks@2.2~shared~debug")
|
||||
|
||||
|
||||
_test_matrix_yaml = """\
|
||||
env:
|
||||
definitions:
|
||||
- compilers: ["%gcc", "%clang"]
|
||||
- desired_specs: ["mpileaks@2.1"]
|
||||
specs:
|
||||
- matrix:
|
||||
- [$compilers]
|
||||
- [$desired_specs]
|
||||
"""
|
||||
|
||||
|
||||
def test_env_change_spec_in_definition(tmpdir, mock_packages, config, mutable_mock_env_path):
|
||||
initial_yaml = StringIO(_test_matrix_yaml)
|
||||
e = ev.create("test", initial_yaml)
|
||||
e.concretize()
|
||||
e.write()
|
||||
|
||||
assert any(x.satisfies("mpileaks@2.1%gcc") for x in e.user_specs)
|
||||
|
||||
e.change_existing_spec(spack.spec.Spec("mpileaks@2.2"), list_name="desired_specs")
|
||||
e.write()
|
||||
|
||||
assert any(x.satisfies("mpileaks@2.2%gcc") for x in e.user_specs)
|
||||
assert not any(x.satisfies("mpileaks@2.1%gcc") for x in e.user_specs)
|
||||
|
||||
|
||||
def test_env_change_spec_in_matrix_raises_error(
|
||||
tmpdir, mock_packages, config, mutable_mock_env_path
|
||||
):
|
||||
initial_yaml = StringIO(_test_matrix_yaml)
|
||||
e = ev.create("test", initial_yaml)
|
||||
e.concretize()
|
||||
e.write()
|
||||
|
||||
with pytest.raises(spack.environment.SpackEnvironmentError) as error:
|
||||
e.change_existing_spec(spack.spec.Spec("mpileaks@2.2"))
|
||||
assert "Cannot directly change specs in matrices" in str(error)
|
||||
|
||||
|
||||
def test_activate_should_require_an_env():
|
||||
with pytest.raises(TypeError):
|
||||
ev.activate(env="name")
|
||||
|
||||
@ -1111,6 +1111,20 @@ def test_splice_swap_names_mismatch_virtuals(self, transitive):
|
||||
with pytest.raises(spack.spec.SpliceError, match="will not provide the same virtuals."):
|
||||
spec.splice(dep, transitive)
|
||||
|
||||
def test_spec_override(self):
|
||||
init_spec = Spec("a foo=baz foobar=baz cflags=-O3 cxxflags=-O1")
|
||||
change_spec = Spec("a foo=fee cflags=-O2")
|
||||
new_spec = Spec.override(init_spec, change_spec)
|
||||
new_spec.concretize()
|
||||
assert "foo=fee" in new_spec
|
||||
# This check fails without concretizing: apparently if both specs are
|
||||
# abstract, then the spec will always be considered to satisfy
|
||||
# 'variant=value' (regardless of whether it in fact does).
|
||||
assert "foo=baz" not in new_spec
|
||||
assert "foobar=baz" in new_spec
|
||||
assert new_spec.compiler_flags["cflags"] == ["-O2"]
|
||||
assert new_spec.compiler_flags["cxxflags"] == ["-O1"]
|
||||
|
||||
|
||||
@pytest.mark.regression("3887")
|
||||
@pytest.mark.parametrize("spec_str", ["git", "hdf5", "py-flake8"])
|
||||
|
||||
@ -337,7 +337,7 @@ _spack() {
|
||||
then
|
||||
SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars"
|
||||
else
|
||||
SPACK_COMPREPLY="activate add arch audit blame bootstrap build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view"
|
||||
SPACK_COMPREPLY="activate add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view"
|
||||
fi
|
||||
}
|
||||
|
||||
@ -585,6 +585,15 @@ _spack_cd() {
|
||||
fi
|
||||
}
|
||||
|
||||
_spack_change() {
|
||||
if $list_options
|
||||
then
|
||||
SPACK_COMPREPLY="-h --help -l --list-name --match-spec -a --all"
|
||||
else
|
||||
_all_packages
|
||||
fi
|
||||
}
|
||||
|
||||
_spack_checksum() {
|
||||
if $list_options
|
||||
then
|
||||
|
||||
Loading…
Reference in New Issue
Block a user