zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adding spack -c to set one off config arguments (#22251)

Browse Source 
This pull request will add the ability for a user to add a configuration argument on the fly, on the command line, e.g.,:

```bash
$ spack -c config:install_tree:root:/path/to/config.yaml -c packages:all:compiler:[gcc] list --help
```
The above command doesn't do anything (I'm just getting help for list) but you can imagine having another root of packages, and updating it on the fly for a command (something I'd like to do in the near future!)

I've moved the logic for config_add that used to be in spack/cmd/config.py into spack/config.py proper, and now both the main.py (where spack commands live) and spack/cmd/config.py use these functions. I only needed spack config add, so I didn't move the others. We can move the others if there are also needed in multiple places.
This commit is contained in:
Vanessasaurus
2021-03-12 22:31:26 -07:00
committed by GitHub
parent 839af2bd70
commit 746081e933
6 changed files with 133 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
lib/spack/spack/cmd/config.py
View File

@@ -200,71 +200,11 @@ def config_add(args):
scope, section = _get_scope_and_section(args)
# Updates from file
if args.file:
# Get file as config dict
data = spack.config.read_config_file(args.file)
if any(k in data for k in spack.schema.env.keys):
data = ev.config_dict(data)
# update all sections from config dict
# We have to iterate on keys to keep overrides from the file
for section in data.keys():
if section in spack.config.section_schemas.keys():
# Special handling for compiler scope difference
# Has to be handled after we choose a section
if scope is None:
scope = spack.config.default_modify_scope(section)
value = data[section]
existing = spack.config.get(section, scope=scope)
new = spack.config.merge_yaml(existing, value)
spack.config.set(section, new, scope)
spack.config.add_from_file(args.file, scope=scope)
if args.path:
components = spack.config.process_config_path(args.path)
has_existing_value = True
path = ''
override = False
for idx, name in enumerate(components[:-1]):
# First handle double colons in constructing path
colon = '::' if override else ':' if path else ''
path += colon + name
if getattr(name, 'override', False):
override = True
else:
override = False
# Test whether there is an existing value at this level
existing = spack.config.get(path, scope=scope)
if existing is None:
has_existing_value = False
# We've nested further than existing config, so we need the
# type information for validation to know how to handle bare
# values appended to lists.
existing = spack.config.get_valid_type(path)
# construct value from this point down
value = syaml.load_config(components[-1])
for component in reversed(components[idx + 1:-1]):
value = {component: value}
break
if has_existing_value:
path, _, value = args.path.rpartition(':')
value = syaml.load_config(value)
existing = spack.config.get(path, scope=scope)
# append values to lists
if isinstance(existing, list) and not isinstance(value, list):
value = [value]
# merge value into existing
new = spack.config.merge_yaml(existing, value)
spack.config.set(path, new, scope)
spack.config.add(args.path, scope=scope)
def config_remove(args):

75
lib/spack/spack/config.py
View File

@@ -806,6 +806,81 @@ def _config():
config = llnl.util.lang.Singleton(_config)
def add_from_file(filename, scope=None):
"""Add updates to a config from a filename
"""
import spack.environment as ev
# Get file as config dict
data = read_config_file(filename)
if any(k in data for k in spack.schema.env.keys):
data = ev.config_dict(data)
# update all sections from config dict
# We have to iterate on keys to keep overrides from the file
for section in data.keys():
if section in section_schemas.keys():
# Special handling for compiler scope difference
# Has to be handled after we choose a section
if scope is None:
scope = default_modify_scope(section)
value = data[section]
existing = get(section, scope=scope)
new = merge_yaml(existing, value)
# We cannot call config.set directly (set is a type)
config.set(section, new, scope)
def add(fullpath, scope=None):
"""Add the given configuration to the specified config scope.
Add accepts a path. If you want to add from a filename, use add_from_file"""
components = process_config_path(fullpath)
has_existing_value = True
path = ''
override = False
for idx, name in enumerate(components[:-1]):
# First handle double colons in constructing path
colon = '::' if override else ':' if path else ''
path += colon + name
if getattr(name, 'override', False):
override = True
else:
override = False
# Test whether there is an existing value at this level
existing = get(path, scope=scope)
if existing is None:
has_existing_value = False
# We've nested further than existing config, so we need the
# type information for validation to know how to handle bare
# values appended to lists.
existing = get_valid_type(path)
# construct value from this point down
value = syaml.load_config(components[-1])
for component in reversed(components[idx + 1:-1]):
value = {component: value}
break
if has_existing_value:
path, _, value = fullpath.rpartition(':')
value = syaml.load_config(value)
existing = get(path, scope=scope)
# append values to lists
if isinstance(existing, list) and not isinstance(value, list):
value = [value]
# merge value into existing
new = merge_yaml(existing, value)
config.set(path, new, scope)
def get(path, default=None, scope=None):
"""Module-level wrapper for ``Configuration.get()``."""
return config.get(path, default, scope)

8
lib/spack/spack/main.py
View File

@@ -40,7 +40,6 @@
import spack.util.executable as exe
from spack.error import SpackError
#: names of profile statistics
stat_names = pstats.Stats.sort_arg_dict_default
@@ -358,6 +357,9 @@ def make_argument_parser(**kwargs):
'--color', action='store', default='auto',
choices=('always', 'never', 'auto'),
help="when to colorize output (default: auto)")
parser.add_argument(
'-c', '--config', default=None, action="append", dest="config_vars",
help="add one or more custom, one off config settings.")
parser.add_argument(
'-C', '--config-scope', dest='config_scopes', action='append',
metavar='DIR', help="add a custom configuration scope")
@@ -463,6 +465,10 @@ def setup_main_options(args):
tty.warn("You asked for --insecure. Will NOT check SSL certificates.")
spack.config.set('config:verify_ssl', False, scope='command_line')
# Use the spack config command to handle parsing the config strings
for config_var in (args.config_vars or []):
spack.config.add(path=config_var, scope="command_line")
# when to use color (takes always, auto, or never)
color.set_color_when(args.color)

26
lib/spack/spack/test/cmd/config.py
View File

@@ -87,6 +87,7 @@ def test_get_config_scope_merged(mock_low_high_config):
def test_config_edit():
"""Ensure `spack config edit` edits the right paths."""
dms = spack.config.default_modify_scope('compilers')
dms_path = spack.config.config.scopes[dms].path
user_path = spack.config.config.scopes['user'].path
@@ -204,20 +205,27 @@ def test_config_add_override_leaf(mutable_empty_config):
def test_config_add_update_dict(mutable_empty_config):
config('add', 'packages:all:compiler:[gcc]')
config('add', 'packages:all:version:1.0.0')
config('add', 'packages:all:version:[1.0.0]')
output = config('get', 'packages')
expected = """packages:
all:
compiler: [gcc]
version:
- 1.0.0
"""
expected = 'packages:\n all:\n version: [1.0.0]\n'
assert output == expected
def test_config_with_c_argument(mutable_empty_config):
# I don't know how to add a spack argument to a Spack Command, so we test this way
config_file = 'config:install_root:root:/path/to/config.yaml'
parser = spack.main.make_argument_parser()
args = parser.parse_args(['-c', config_file])
assert config_file in args.config_vars
# Add the path to the config
config("add", args.config_vars[0], scope='command_line')
output = config("get", 'config')
assert "config:\n install_root:\n - root: /path/to/config.yaml" in output
def test_config_add_ordered_dict(mutable_empty_config):
config('add', 'mirrors:first:/path/to/first')
config('add', 'mirrors:second:/path/to/second')

31
lib/spack/spack/test/config.py
View File

@@ -258,6 +258,37 @@ def test_write_to_same_priority_file(mock_low_high_config, compiler_specs):
repos_low = {'repos': ["/some/path"]}
repos_high = {'repos': ["/some/other/path"]}
# Test setting config values via path in filename
def test_add_config_path():
# Try setting a new install tree root
path = "config:install_tree:root:/path/to/config.yaml"
spack.config.add(path, scope="command_line")
set_value = spack.config.get('config')['install_tree']['root']
assert set_value == '/path/to/config.yaml'
# Now a package:all setting
path = "packages:all:compiler:[gcc]"
spack.config.add(path, scope="command_line")
compilers = spack.config.get('packages')['all']['compiler']
assert "gcc" in compilers
def test_add_config_filename(mock_low_high_config, tmpdir):
config_yaml = tmpdir.join('config-filename.yaml')
config_yaml.ensure()
with config_yaml.open('w') as f:
syaml.dump_config(config_low, f)
spack.config.add_from_file(str(config_yaml), scope="low")
assert "build_stage" in spack.config.get('config')
build_stages = spack.config.get('config')['build_stage']
for stage in config_low['config']['build_stage']:
assert stage in build_stages
# repos
def test_write_list_in_memory(mock_low_high_config):

2
share/spack/spack-completion.bash
View File

@@ -331,7 +331,7 @@ _spacktivate() {
_spack() {
if $list_options
then
SPACK_COMPREPLY="-h --help -H --all-help --color -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars"
SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars"
else
SPACK_COMPREPLY="activate add arch blame build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop docs edit env extensions external fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style test test-env tutorial undevelop uninstall unit-test unload url verify versions view"
fi