From b442b21751634ff771d7dab990683ee3556d5c86 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 21 Jan 2020 23:36:10 -0800 Subject: [PATCH 1/8] bugfix: hashes should use ordered dictionaries (#14390) Despite trying very hard to keep dicts out of our hash algorithm, we seem to still accidentally add them in ways that the tests can't catch. This can cause errors when hashes are not computed deterministically. This fixes an error we saw with Python 3.5, where dictionary iteration order is random. In this instance, we saw a bug when reading Spack environment lockfiles -- The load would fail like this: ``` ... File "/sw/spack/lib/spack/spack/environment.py", line 1249, in concretized_specs yield (s, self.specs_by_hash[h]) KeyError: 'qcttqplkwgxzjlycbs4rfxxladnt423p' ``` This was because the hashes differed depending on whether we wrote `path` or `module` first when recomputing the build hash as part of reading a Spack lockfile. We can fix it by ensuring a determistic iteration order. - [x] Fix two places (one that caused an issue, and one that did not... yet) where our to_node_dict-like methods were using regular python dicts. - [x] Also add a check that statically analyzes our to_node_dict functions and flags any that use Python dicts. The test found the two errors fixed here, specifically: ``` E AssertionError: assert [] == ['Use syaml_dict instead of ...pack/spack/spec.py:1495:28'] E Right contains more items, first extra item: 'Use syaml_dict instead of dict at /Users/gamblin2/src/spack/lib/spack/spack/spec.py:1495:28' E Full diff: E - [] E + ['Use syaml_dict instead of dict at ' E + '/Users/gamblin2/src/spack/lib/spack/spack/spec.py:1495:28'] ``` and ``` E AssertionError: assert [] == ['Use syaml_dict instead of ...ack/architecture.py:359:15'] E Right contains more items, first extra item: 'Use syaml_dict instead of dict at /Users/gamblin2/src/spack/lib/spack/spack/architecture.py:359:15' E Full diff: E - [] E + ['Use syaml_dict instead of dict at ' E + '/Users/gamblin2/src/spack/lib/spack/spack/architecture.py:359:15'] ``` --- lib/spack/spack/architecture.py | 8 ++-- lib/spack/spack/spec.py | 8 ++-- lib/spack/spack/test/spec_yaml.py | 73 +++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 7552795cd23..378fb5d5d9c 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -351,10 +351,10 @@ def _cmp_key(self): return (self.name, self.version) def to_dict(self): - return { - 'name': self.name, - 'version': self.version - } + return syaml_dict([ + ('name', self.name), + ('version', self.version) + ]) @key_ordering diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index c553da796dc..f983e5c5590 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1506,10 +1506,10 @@ def to_node_dict(self, hash=ht.dag_hash): d['parameters'] = params if self.external: - d['external'] = { - 'path': self.external_path, - 'module': self.external_module - } + d['external'] = syaml.syaml_dict([ + ('path', self.external_path), + ('module', self.external_module), + ]) if not self._concrete: d['concrete'] = False diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index 7fd2a36469a..96bed17b78b 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -8,13 +8,20 @@ YAML format preserves DAG information in the spec. """ +import ast +import inspect import os from collections import Iterable, Mapping +import pytest + +import spack.architecture import spack.hash_types as ht +import spack.spec import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml +import spack.version from spack import repo from spack.spec import Spec, save_dependency_spec_yamls @@ -204,6 +211,72 @@ def test_ordered_read_not_required_for_consistent_dag_hash( assert spec.full_hash() == round_trip_reversed_json_spec.full_hash() +@pytest.mark.parametrize("module", [ + spack.spec, + spack.architecture, + spack.version, +]) +def test_hashes_use_no_python_dicts(module): + """Coarse check to make sure we don't use dicts in Spec.to_node_dict(). + + Python dicts are not guaranteed to iterate in a deterministic order + (at least not in all python versions) so we need to use lists and + syaml_dicts. syaml_dicts are ordered and ensure that hashes in Spack + are deterministic. + + This test is intended to handle cases that are not covered by the + consistency checks above, or that would be missed by a dynamic check. + This test traverses the ASTs of functions that are used in our hash + algorithms, finds instances of dictionaries being constructed, and + prints out the line numbers where they occur. + + """ + class FindFunctions(ast.NodeVisitor): + """Find a function definition called to_node_dict.""" + def __init__(self): + self.nodes = [] + + def visit_FunctionDef(self, node): # noqa + if node.name in ("to_node_dict", "to_dict", "to_dict_or_value"): + self.nodes.append(node) + + class FindDicts(ast.NodeVisitor): + """Find source locations of dicts in an AST.""" + def __init__(self, filename): + self.nodes = [] + self.filename = filename + + def add_error(self, node): + self.nodes.append( + "Use syaml_dict instead of dict at %s:%s:%s" + % (self.filename, node.lineno, node.col_offset) + ) + + def visit_Dict(self, node): # noqa + self.add_error(node) + + def visit_Call(self, node): # noqa + name = None + if isinstance(node.func, ast.Name): + name = node.func.id + elif isinstance(node.func, ast.Attribute): + name = node.func.attr + + if name == 'dict': + self.add_error(node) + + find_functions = FindFunctions() + module_ast = ast.parse(inspect.getsource(module)) + find_functions.visit(module_ast) + + find_dicts = FindDicts(module.__file__) + for node in find_functions.nodes: + find_dicts.visit(node) + + # fail with offending lines if we found some dicts. + assert [] == find_dicts.nodes + + def reverse_all_dicts(data): """Descend into data and reverse all the dictionaries""" if isinstance(data, dict): From 5397d500c831a78d39caa3dd8aff931e1ea8ec4d Mon Sep 17 00:00:00 2001 From: Jeffrey Salmond Date: Wed, 8 Jan 2020 23:52:39 +0000 Subject: [PATCH 2/8] Remove extensions from view in the correct order (#12961) When removing packages from a view, extensions were being deactivated in an arbitrary order. Extensions must be deactivated in preorder traversal (dependents before dependencies), so when this order was violated the view update would fail. This commit ensures that views deactivate extensions based on a preorder traversal and adds a test for it. --- lib/spack/spack/filesystem_view.py | 38 ++++++++++++++++++------------ lib/spack/spack/test/views.py | 14 +++++++++++ 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/filesystem_view.py b/lib/spack/spack/filesystem_view.py index 5455ccb1077..3d17d7e4cec 100644 --- a/lib/spack/spack/filesystem_view.py +++ b/lib/spack/spack/filesystem_view.py @@ -399,23 +399,31 @@ def remove_specs(self, *specs, **kwargs): "The following packages will be unusable: %s" % ", ".join((s.name for s in dependents))) - extensions = set(filter(lambda s: s.package.is_extension, - to_deactivate)) - standalones = to_deactivate - extensions + # Determine the order that packages should be removed from the view; + # dependents come before their dependencies. + to_deactivate_sorted = list() + depmap = dict() + for spec in to_deactivate: + depmap[spec] = set(d for d in spec.traverse(root=False) + if d in to_deactivate) - # Please note that a traversal of the DAG in post-order and then - # forcibly removing each package should remove the need to specify - # with_dependents for deactivating extensions/allow removal without - # additional checks (force=True). If removal performance becomes - # unbearable for whatever reason, this should be the first point of - # attack. - # - # see: https://github.com/spack/spack/pull/3227#discussion_r117147475 - remove_extension = ft.partial(self.remove_extension, - with_dependents=with_dependents) + while depmap: + for spec in [s for s, d in depmap.items() if not d]: + to_deactivate_sorted.append(spec) + for s in depmap.keys(): + depmap[s].discard(spec) + depmap.pop(spec) + to_deactivate_sorted.reverse() - set(map(remove_extension, extensions)) - set(map(self.remove_standalone, standalones)) + # Ensure that the sorted list contains all the packages + assert set(to_deactivate_sorted) == to_deactivate + + # Remove the packages from the view + for spec in to_deactivate_sorted: + if spec.package.is_extension: + self.remove_extension(spec, with_dependents=with_dependents) + else: + self.remove_standalone(spec) self._purge_empty_directories() diff --git a/lib/spack/spack/test/views.py b/lib/spack/spack/test/views.py index a94fa42c21a..52ecb91e734 100644 --- a/lib/spack/spack/test/views.py +++ b/lib/spack/spack/test/views.py @@ -6,6 +6,8 @@ import os from spack.spec import Spec +from spack.directory_layout import YamlDirectoryLayout +from spack.filesystem_view import YamlFilesystemView def test_global_activation(install_mockery, mock_fetch): @@ -27,3 +29,15 @@ def test_global_activation(install_mockery, mock_fetch): extendee_spec.prefix, '.spack', 'extensions.yaml') assert (view.extensions_layout.extension_file_path(extendee_spec) == expected_path) + + +def test_remove_extensions_ordered(install_mockery, mock_fetch, tmpdir): + view_dir = str(tmpdir.join('view')) + layout = YamlDirectoryLayout(view_dir) + view = YamlFilesystemView(view_dir, layout) + e2 = Spec('extension2').concretized() + e2.package.do_install() + view.add_specs(e2) + + e1 = e2['extension1'] + view.remove_specs(e1, e2) From 69e5683ba49d9e3c7e5f21d8cfade2d0539097cf Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 6 Jan 2020 23:18:14 -0600 Subject: [PATCH 3/8] Fix outdated bash tab completion (#14392) --- share/spack/spack-completion.bash | 619 ++++++++++++++++++------------ 1 file changed, 370 insertions(+), 249 deletions(-) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 5e8936125f8..8597cd3d7ef 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -34,6 +34,7 @@ function _bash_completion_spack { # For example, `spack -d install []` will call _spack_install # and `spack compiler add []` will call _spack_compiler_add local subfunction=$(IFS='_'; echo "_${COMP_WORDS_NO_FLAGS[*]}") + # Translate dashes to underscores, as dashes are not permitted in # compatibility mode. See https://github.com/spack/spack/pull/4079 subfunction=${subfunction//-/_} @@ -96,11 +97,11 @@ function _spack { if $list_options then compgen -W "-h --help -H --all-help --color -C --config-scope - -d --debug --pdb -e --env -D --env-dir -E --no-env - --use-env-repo -k --insecure -l --enable-locks - -L --disable-locks -m --mock -p --profile - --sorted-profile --lines -v --verbose --stacktrace - -V --version --print-shell-vars" -- "$cur" + -d --debug --timestamp --pdb -e --env -D --env-dir + -E --no-env --use-env-repo -k --insecure + -l --enable-locks -L --disable-locks -m --mock + -p --profile --sorted-profile --lines -v --verbose + --stacktrace -V --version --print-shell-vars" -- "$cur" else compgen -W "$(_subcommands)" -- "$cur" fi @@ -118,15 +119,16 @@ function _spack_activate { function _spack_add { if $list_options then - compgen -W "-h --help" -- "$cur" + compgen -W "-h --help -l --list-name" -- "$cur" else compgen -W "$(_all_packages)" -- "$cur" fi } function _spack_arch { - compgen -W "-h --help -p --platform -o --operating-system - -t --target --known-targets" -- "$cur" + compgen -W "-h --help --known-targets -p --platform + -o --operating-system -t --target -f --frontend + -b --backend" -- "$cur" } function _spack_blame { @@ -141,7 +143,7 @@ function _spack_blame { function _spack_bootstrap { compgen -W "-h --help -j --jobs --keep-prefix --keep-stage -n --no-checksum -v --verbose --use-cache --no-cache - --clean --dirty" -- "$cur" + --cache-only --clean --dirty" -- "$cur" } function _spack_build { @@ -156,7 +158,7 @@ function _spack_build { function _spack_build_env { if $list_options then - compgen -W "-h --help --clean --dirty" -- "$cur" + compgen -W "-h --help --clean --dirty --dump --pickle" -- "$cur" else compgen -W "$(_all_packages)" -- "$cur" fi @@ -167,15 +169,17 @@ function _spack_buildcache { then compgen -W "-h --help" -- "$cur" else - compgen -W "create install keys list" -- "$cur" + compgen -W "create install list keys preview check download + get-buildcache-name save-yaml copy update-index" -- "$cur" fi } function _spack_buildcache_create { if $list_options then - compgen -W "-h --help -r --rel -f --force -u --unsigned -a --allow-root - -k --key -d --directory" -- "$cur" + compgen -W "-h --help -r --rel -f --force -u --unsigned + -a --allow-root -k --key -d --directory + --no-rebuild-index -y --spec-yaml --no-deps" -- "$cur" else compgen -W "$(_all_packages)" -- "$cur" fi @@ -184,8 +188,18 @@ function _spack_buildcache_create { function _spack_buildcache_install { if $list_options then - compgen -W "-h --help -f --force -m --multiple -a --allow-root -u - --unsigned" -- "$cur" + compgen -W "-h --help -f --force -m --multiple -a --allow-root + -u --unsigned" -- "$cur" + else + compgen -W "$(_all_packages)" -- "$cur" + fi +} + +function _spack_buildcache_list { + if $list_options + then + compgen -W "-h --help -l --long -L --very-long -v --variants + -f --force" -- "$cur" else compgen -W "$(_all_packages)" -- "$cur" fi @@ -195,15 +209,42 @@ function _spack_buildcache_keys { compgen -W "-h --help -i --install -t --trust -f --force" -- "$cur" } -function _spack_buildcache_list { +function _spack_buildcache_preview { if $list_options then - compgen -W "-h --help -f --force" -- "$cur" + compgen -W "-h --help" -- "$cur" else - compgen -W "$(_all_packages)" -- "$cur" + compgen -W "$(_installed_packages)" -- "$cur" fi } +function _spack_buildcache_check { + compgen -W "-h --help -m --mirror-url -o --output-file --scope + -s --spec -y --spec-yaml --rebuild-on-error" -- "$cur" +} + +function _spack_buildcache_download { + compgen -W "-h --help -s --spec -y --spec-yaml -p --path + -c --require-cdashid" -- "$cur" +} + +function _spack_buildcache_get_buildcache_name { + compgen -W "-h --help -s --spec -y --spec-yaml" -- "$cur" +} + +function _spack_buildcache_save_yaml { + compgen -W "-h --help --root-spec --root-spec-yaml -s --specs + -y --yaml-dir" -- "$cur" +} + +function _spack_buildcache_copy { + compgen -W "-h --help --base-dir --spec-yaml --destination-url" -- "$cur" +} + +function _spack_buildcache_update_index { + compgen -W "-h --help -d --mirror-url" -- "$cur" +} + function _spack_cd { if $list_options then @@ -257,29 +298,16 @@ function _spack_compiler { fi } -function _spack_compiler_add { - if $list_options - then - compgen -W "-h --help --scope" -- "$cur" - fi -} - function _spack_compiler_find { - # Alias to `spack compiler add` - _spack_compiler_add -} - -function _spack_compiler_info { if $list_options then compgen -W "-h --help --scope" -- "$cur" - else - compgen -W "$(_installed_compilers)" -- "$cur" fi } -function _spack_compiler_list { - compgen -W "-h --help --scope" -- "$cur" +function _spack_compiler_add { + # Alias to `spack compiler find` + _spack_compiler_find } function _spack_compiler_remove { @@ -296,10 +324,24 @@ function _spack_compiler_rm { _spack_compiler_remove } -function _spack_compilers { +function _spack_compiler_list { compgen -W "-h --help --scope" -- "$cur" } +function _spack_compiler_info { + if $list_options + then + compgen -W "-h --help --scope" -- "$cur" + else + compgen -W "$(_installed_compilers)" -- "$cur" + fi +} + +function _spack_compilers { + # Alias to `spack compiler list` + _spack_compiler_list +} + function _spack_concretize { compgen -W "-h --help -f --force" -- "$cur" } @@ -309,25 +351,7 @@ function _spack_config { then compgen -W "-h --help --scope" -- "$cur" else - compgen -W "blame edit get" -- "$cur" - fi -} - -function _spack_config_blame { - if $list_options - then - compgen -W "-h --help" -- "$cur" - else - compgen -W "mirrors repos modules packages config compilers" -- "$cur" - fi -} - -function _spack_config_edit { - if $list_options - then - compgen -W "-h --help --print-file" -- "$cur" - else - compgen -W "mirrors repos modules packages config compilers" -- "$cur" + compgen -W "get blame edit" -- "$cur" fi } @@ -336,7 +360,28 @@ function _spack_config_get { then compgen -W "-h --help" -- "$cur" else - compgen -W "mirrors repos modules packages config compilers" -- "$cur" + compgen -W "compilers mirrors repos packages modules config + upstreams" -- "$cur" + fi +} + +function _spack_config_blame { + if $list_options + then + compgen -W "-h --help" -- "$cur" + else + compgen -W "compilers mirrors repos packages modules config + upstreams" -- "$cur" + fi +} + +function _spack_config_edit { + if $list_options + then + compgen -W "-h --help --print-file" -- "$cur" + else + compgen -W "compilers mirrors repos packages modules config + upstreams" -- "$cur" fi } @@ -382,8 +427,8 @@ function _spack_debug_create_db_tarball { function _spack_dependencies { if $list_options then - compgen -W "-h --help -i --installed -t --transitive -V - --no-expand-virtuals" -- "$cur" + compgen -W "-h --help -i --installed -t --transitive + --deptype -V --no-expand-virtuals" -- "$cur" else compgen -W "$(_all_packages)" -- "$cur" fi @@ -398,13 +443,36 @@ function _spack_dependents { fi } +function _spack_deprecate { + if $list_options + then + compgen -W "-h --help -y --yes-to-all -d --dependencies + -D --no-dependencies -i --install-deprecator + -I --no-install-deprecator -l --link-type" -- "$cur" + else + compgen -W "$(_all_packages)" -- "$cur" + fi +} + +function _spack_dev_build { + if $list_options + then + compgen -W "-h --help -j --jobs -d --source-path + -i --ignore-dependencies -n --no-checksum + --keep-prefix --skip-patch -q --quiet -u --until + --clean --dirty" -- "$cur" + else + compgen -W "$(_all_packages)" -- "$cur" + fi +} + function _spack_diy { if $list_options then compgen -W "-h --help -j --jobs -d --source-path -i --ignore-dependencies -n --no-checksum - --keep-prefix --skip-patch -q --quiet --clean - --dirty -u --until" -- "$cur" + --keep-prefix --skip-patch -q --quiet -u --until + --clean --dirty" -- "$cur" else compgen -W "$(_all_packages)" -- "$cur" fi @@ -417,8 +485,8 @@ function _spack_docs { function _spack_edit { if $list_options then - compgen -W "-h --help -b --build-system -c --command -d --docs -t - --test -m --module -r --repo -N --namespace" -- "$cur" + compgen -W "-h --help -b --build-system -c --command -d --docs + -t --test -m --module -r --repo -N --namespace" -- "$cur" else compgen -W "$(_all_packages)" -- "$cur" fi @@ -429,47 +497,29 @@ function _spack_env { then compgen -W "-h --help" -- "$cur" else - compgen -W "activate create deactivate list ls loads - remove rm status st" -- "$cur" + compgen -W "activate deactivate create remove rm list ls + status st loads view" -- "$cur" fi } function _spack_env_activate { if $list_options then - compgen -W "-h --help --sh --csh -d --dir -p --prompt" -- "$cur" + compgen -W "-h --help --sh --csh -v --with-view + -V --without-view -d --dir -p --prompt" -- "$cur" else compgen -W "$(_environments)" -- "$cur" fi } -function _spack_env_create { - if $list_options - then - compgen -W "-h --help -d --dir" -- "$cur" - fi -} - function _spack_env_deactivate { compgen -W "-h --help --sh --csh" -- "$cur" } -function _spack_env_list { - compgen -W "-h --help" -- "$cur" -} - -function _spack_env_ls { - # Alias to `spack env list` - _spack_env_list -} - -function _spack_env_loads { +function _spack_env_create { if $list_options then - compgen -W "-h --help -m --module-type --input-only -p --prefix - -x --exclude -r --dependencies" -- "$cur" - else - compgen -W "$(_environments)" -- "$cur" + compgen -W "-h --help -d --dir --without-view --with-view" -- "$cur" fi } @@ -487,6 +537,15 @@ function _spack_env_rm { _spack_env_remove } +function _spack_env_list { + compgen -W "-h --help" -- "$cur" +} + +function _spack_env_ls { + # Alias to `spack env list` + _spack_env_list +} + function _spack_env_status { compgen -W "-h --help" -- "$cur" } @@ -496,15 +555,34 @@ function _spack_env_st { _spack_env_status } +function _spack_env_loads { + if $list_options + then + compgen -W "-h --help -m --module-type --input-only -p --prefix + -x --exclude -r --dependencies" -- "$cur" + else + compgen -W "$(_environments)" -- "$cur" + fi +} + +function _spack_env_view { + if $list_options + then + compgen -W "-h --help" -- "$cur" + else + compgen -W "regenerate enable disable" -- "$cur" + fi +} + function _spack_extensions { if $list_options then - compgen -W "-h --help -l --long -p --paths -d --deps - -s --show -v --view" -- "$cur" + compgen -W "-h --help -l --long -L --very-long -d --deps + -p --paths -s --show -v --view" -- "$cur" else - compgen -W "aspell go-bootstrap go icedtea java jdk lua - matlab mofem-cephas octave perl python r ruby - rust tcl yorick" -- "$cur" + compgen -W "aspell go-bootstrap go icedtea jdk kim-api lua + matlab mofem-cephas octave openjdk perl python r + ruby rust tcl yorick" -- "$cur" fi } @@ -521,13 +599,13 @@ function _spack_fetch { function _spack_find { if $list_options then - compgen -W "-h --help -s --short -d --deps -p --paths - --format --json --groups --no-groups -l --long - -L --very-long -t --tags -c --show-concretized - -f --show-flags --show-full-compiler -x --explicit - -X --implicit -u --unknown -m --missing -v --variants - -M --only-missing -N --namespace --start-date - --end-date" -- "$cur" + compgen -W "-h --help --format --json -d --deps -p --paths + --groups --no-groups -l --long -L --very-long + -t --tags -c --show-concretized -f --show-flags + --show-full-compiler -x --explicit -X --implicit + -u --unknown -m --missing -v --variants + -M --only-missing --deprecated --only-deprecated + -N --namespace --start-date --end-date" -- "$cur" else compgen -W "$(_installed_packages)" -- "$cur" fi @@ -546,36 +624,15 @@ function _spack_gpg { then compgen -W "-h --help" -- "$cur" else - compgen -W "create export init list sign trust untrust verify" -- "$cur" + compgen -W "verify trust untrust sign create list init + export" -- "$cur" fi } -function _spack_gpg_create { - if $list_options - then - compgen -W "-h --help --comment --expires --export" -- "$cur" - fi -} - -function _spack_gpg_export { +function _spack_gpg_verify { if $list_options then compgen -W "-h --help" -- "$cur" - fi -} - -function _spack_gpg_init { - compgen -W "-h --help" -- "$cur" -} - -function _spack_gpg_list { - compgen -W "-h --help --trusted --signing" -- "$cur" -} - -function _spack_gpg_sign { - if $list_options - then - compgen -W "-h --help --output --key --clearsign" -- "$cur" else compgen -W "$(installed_packages)" -- "$cur" fi @@ -592,23 +649,49 @@ function _spack_gpg_untrust { if $list_options then compgen -W "-h --help --signing" -- "$cur" + else + compgen -W "$(_keys)" -- "$cur" fi } -function _spack_gpg_verify { +function _spack_gpg_sign { + if $list_options + then + compgen -W "-h --help --output --key --clearsign" -- "$cur" + else + compgen -W "$(installed_packages)" -- "$cur" + fi +} + +function _spack_gpg_create { + if $list_options + then + compgen -W "-h --help --comment --expires --export" -- "$cur" + fi +} + +function _spack_gpg_list { + compgen -W "-h --help --trusted --signing" -- "$cur" +} + +function _spack_gpg_init { + compgen -W "-h --help" -- "$cur" +} + +function _spack_gpg_export { if $list_options then compgen -W "-h --help" -- "$cur" else - compgen -W "$(installed_packages)" -- "$cur" + compgen -W "$(_keys)" -- "$cur" fi } function _spack_graph { if $list_options then - compgen -W "-h --help -a --ascii -d --dot -n --normalize -s --static - -i --installed -t --deptype" -- "$cur" + compgen -W "-h --help -a --ascii -d --dot -s --static + -i --installed --deptype" -- "$cur" else compgen -W "$(_all_packages)" -- "$cur" fi @@ -635,12 +718,14 @@ function _spack_info { function _spack_install { if $list_options then - compgen -W "-h --help --only -j --jobs -I --install-status - --overwrite --keep-prefix --keep-stage --dont-restage - --use-cache --no-cache --show-log-on-error --source + compgen -W "-h --help --only -u --until -j --jobs --overwrite + --keep-prefix --keep-stage --dont-restage --use-cache + --no-cache --cache-only --show-log-on-error --source -n --no-checksum -v --verbose --fake --only-concrete - -f --file --clean --dirty --test --log-format --log-file - --cdash-upload-url -y --yes-to-all" -- "$cur" + -f --file --clean --dirty --test --run-tests + --log-format --log-file -y --yes-to-all + --cdash-upload-url --cdash-build --cdash-site + --cdash-track --cdash-buildstamp" -- "$cur" else compgen -W "$(_all_packages)" -- "$cur" fi @@ -704,7 +789,7 @@ function _spack_log_parse { function _spack_maintainers { if $list_options then - compgen -W "-h --help -a --all --maintained --unmaintained + compgen -W "-h --help --maintained --unmaintained -a --all --by-user" -- "$cur" else compgen -W "$(_all_packages)" -- "$cur" @@ -716,7 +801,17 @@ function _spack_mirror { then compgen -W "-h --help -n --no-checksum" -- "$cur" else - compgen -W "add create list remove rm" -- "$cur" + compgen -W "create add remove rm set-url list" -- "$cur" + fi +} + +function _spack_mirror_create { + if $list_options + then + compgen -W "-h --help -d --directory -a --all -f --file + -D --dependencies -n --versions-per-spec" -- "$cur" + else + compgen -W "$(_all_packages)" -- "$cur" fi } @@ -727,20 +822,6 @@ function _spack_mirror_add { fi } -function _spack_mirror_create { - if $list_options - then - compgen -W "-h --help -d --directory -f --file - -D --dependencies -n --versions-per-spec" -- "$cur" - else - compgen -W "$(_all_packages)" -- "$cur" - fi -} - -function _spack_mirror_list { - compgen -W "-h --help --scope" -- "$cur" -} - function _spack_mirror_remove { if $list_options then @@ -755,6 +836,19 @@ function _spack_mirror_rm { _spack_mirror_remove } +function _spack_mirror_set_url { + if $list_options + then + compgen -W "-h --help --push --scope" -- "$cur" + else + compgen -W "$(_mirrors)" -- "$cur" + fi +} + +function _spack_mirror_list { + compgen -W "-h --help --scope" -- "$cur" +} + function _spack_module { if $list_options then @@ -764,55 +858,6 @@ function _spack_module { fi } -function _spack_module_tcl { - if $list_options - then - compgen -W "-h --help" -- "$cur" - else - compgen -W "refresh find rm loads" -- "$cur" - fi -} - - -function _spack_module_tcl_find { - if $list_options - then - compgen -W "-h --help --full-path -r --dependencies" -- "$cur" - else - compgen -W "$(_installed_packages)" -- "$cur" - fi -} - -function _spack_module_tcl_loads { - if $list_options - then - compgen -W "-h --help --input-only -p --prefix -x --exclude - -r --dependencies" -- "$cur" - else - compgen -W "$(_installed_packages)" -- "$cur" - fi - -} - -function _spack_module_tcl_refresh { - if $list_options - then - compgen -W "-h --help --delete-tree -y --yes-to-all" -- "$cur" - else - compgen -W "$(_installed_packages)" -- "$cur" - fi -} - -function _spack_module_tcl_rm { - if $list_options - then - compgen -W "-h --help -y --yes-to-all" -- "$cur" - else - compgen -W "$(_installed_packages)" -- "$cur" - fi -} - - function _spack_module_lmod { if $list_options then @@ -822,6 +867,15 @@ function _spack_module_lmod { fi } +function _spack_module_lmod_refresh { + if $list_options + then + compgen -W "-h --help --delete-tree --upstream-modules + -y --yes-to-all" -- "$cur" + else + compgen -W "$(_installed_packages)" -- "$cur" + fi +} function _spack_module_lmod_find { if $list_options @@ -832,6 +886,15 @@ function _spack_module_lmod_find { fi } +function _spack_module_lmod_rm { + if $list_options + then + compgen -W "-h --help -y --yes-to-all" -- "$cur" + else + compgen -W "$(_installed_packages)" -- "$cur" + fi +} + function _spack_module_lmod_loads { if $list_options then @@ -843,16 +906,44 @@ function _spack_module_lmod_loads { } -function _spack_module_lmod_refresh { +function _spack_module_lmod_setdefault { if $list_options then - compgen -W "-h --help --delete-tree -y --yes-to-all" -- "$cur" + compgen -W "-h --help" -- "$cur" else compgen -W "$(_installed_packages)" -- "$cur" fi } -function _spack_module_lmod_rm { +function _spack_module_tcl { + if $list_options + then + compgen -W "-h --help" -- "$cur" + else + compgen -W "refresh find rm loads" -- "$cur" + fi +} + +function _spack_module_tcl_refresh { + if $list_options + then + compgen -W "-h --help --delete-tree --upstream-modules + -y --yes-to-all" -- "$cur" + else + compgen -W "$(_installed_packages)" -- "$cur" + fi +} + +function _spack_module_tcl_find { + if $list_options + then + compgen -W "-h --help --full-path -r --dependencies" -- "$cur" + else + compgen -W "$(_installed_packages)" -- "$cur" + fi +} + +function _spack_module_tcl_rm { if $list_options then compgen -W "-h --help -y --yes-to-all" -- "$cur" @@ -861,10 +952,11 @@ function _spack_module_lmod_rm { fi } -function _spack_module_lmod_setdefault { +function _spack_module_tcl_loads { if $list_options then - compgen -W "-h --help" -- "$cur" + compgen -W "-h --help --input-only -p --prefix -x --exclude + -r --dependencies" -- "$cur" else compgen -W "$(_installed_packages)" -- "$cur" fi @@ -884,7 +976,7 @@ function _spack_pkg { then compgen -W "-h --help" -- "$cur" else - compgen -W "add added diff list removed" -- "$cur" + compgen -W "add list diff added changed removed" -- "$cur" fi } @@ -897,7 +989,7 @@ function _spack_pkg_add { fi } -function _spack_pkg_added { +function _spack_pkg_list { # FIXME: How to list git revisions? if $list_options then @@ -913,7 +1005,7 @@ function _spack_pkg_diff { fi } -function _spack_pkg_list { +function _spack_pkg_added { # FIXME: How to list git revisions? if $list_options then @@ -921,6 +1013,14 @@ function _spack_pkg_list { fi } +function _spack_pkg_changed { + # FIXME: How to list git revisions? + if $list_options + then + compgen -W "-h --help -t --type" -- "$cur" + fi +} + function _spack_pkg_removed { # FIXME: How to list git revisions? if $list_options @@ -956,10 +1056,15 @@ function _spack_reindex { compgen -W "-h --help" -- "$cur" } +function _spack_release_jobs { + compgen -W "-h --help -o --output-file -p --print-summary + --cdash-credentials" -- "$cur" +} + function _spack_remove { if $list_options then - compgen -W "-h --help -a --all -f --force" -- "$cur" + compgen -W "-h --help -a --all -l --list-name -f --force" -- "$cur" else compgen -W "$(_all_packages)" -- "$cur" fi @@ -970,14 +1075,7 @@ function _spack_repo { then compgen -W "-h --help" -- "$cur" else - compgen -W "add create list remove rm" -- "$cur" - fi -} - -function _spack_repo_add { - if $list_options - then - compgen -W "-h --help --scope" -- "$cur" + compgen -W "create list add remove rm" -- "$cur" fi } @@ -992,6 +1090,13 @@ function _spack_repo_list { compgen -W "-h --help --scope" -- "$cur" } +function _spack_repo_add { + if $list_options + then + compgen -W "-h --help --scope" -- "$cur" + fi +} + function _spack_repo_remove { if $list_options then @@ -1051,7 +1156,7 @@ function _spack_spec { if $list_options then compgen -W "-h --help -l --long -L --very-long -I --install-status - -j --json -y --yaml -c --cover -N --namespaces + -y --yaml -j --json -c --cover -N --namespaces -t --types" -- "$cur" else compgen -W "$(_all_packages)" -- "$cur" @@ -1092,34 +1197,37 @@ function _spack_unload { then compgen -W "-h --help" -- "$cur" else - compgen -W "$(_installed_packages)" + compgen -W "$(_installed_packages)" -- "$cur" fi } -function _spack_unuse { +function _spack_upload_s3 { if $list_options then compgen -W "-h --help" -- "$cur" else - compgen -W "$(_installed_packages)" + compgen -W "spec index" -- "$cur" fi } +function _spack_upload_s3_spec { + compgen -W "-h --help -s --spec -y --spec-yaml -b --base-dir + -e --endpoint-url" -- "$cur" +} + +function _spack_upload_s3_index { + compgen -W "-h --help -e --endpoint-url" -- "$cur" +} + function _spack_url { if $list_options then compgen -W "-h --help" -- "$cur" else - compgen -W "list parse stats summary" -- "$cur" + compgen -W "parse list summary stats" -- "$cur" fi } -function _spack_url_list { - compgen -W "-h --help -c --color -e --extrapolation - -n --incorrect-name -N --correct-name - -v --incorrect-version -V --correct-version" -- "$cur" -} - function _spack_url_parse { if $list_options then @@ -1127,20 +1235,27 @@ function _spack_url_parse { fi } -function _spack_url_stats { - compgen -W "-h --help" -- "$cur" +function _spack_url_list { + compgen -W "-h --help -c --color -e --extrapolation + -n --incorrect-name -N --correct-name + -v --incorrect-version -V --correct-version" -- "$cur" } function _spack_url_summary { compgen -W "-h --help" -- "$cur" } -function _spack_use { +function _spack_url_stats { + compgen -W "-h --help" -- "$cur" +} + +function _spack_verify { if $list_options then - compgen -W "-h --help -r --dependencies" -- "$cur" + compgen -W "-h --help -l --local -j --json -a --all -s --specs + -f --files" -- "$cur" else - compgen -W "$(_installed_packages)" -- "$cur" + compgen -W "$(_all_packages)" -- "$cur" fi } @@ -1159,8 +1274,16 @@ function _spack_view { compgen -W "-h --help -v --verbose -e --exclude -d --dependencies" -- "$cur" else - compgen -W "add check hard hardlink remove rm soft - statlink status symlink" -- "$cur" + compgen -W "symlink add soft hardlink hard remove rm statlink + status check" -- "$cur" + fi +} + +function _spack_view_symlink { + if $list_options + then + compgen -W "-h --help --projection-file + -i --ignore-conflicts" -- "$cur" fi } @@ -1169,23 +1292,24 @@ function _spack_view_add { _spack_view_symlink } -function _spack_view_check { - # Alias for `spack view statlink` - _spack_view_statlink -} - -function _spack_view_hard { - # Alias for `spack view hardlink` - _spack_view_hardlink +function _spack_view_soft { + # Alias for `spack view symlink` + _spack_view_symlink } function _spack_view_hardlink { if $list_options then - compgen -W "-h --help -i --ignore-conflicts" -- "$cur" + compgen -W "-h --help --projection-file + -i --ignore-conflicts" -- "$cur" fi } +function _spack_view_hard { + # Alias for `spack view hardlink` + _spack_view_hardlink +} + function _spack_view_remove { if $list_options then @@ -1198,11 +1322,6 @@ function _spack_view_rm { _spack_view_remove } -function _spack_view_soft { - # Alias for `spack view symlink` - _spack_view_symlink -} - function _spack_view_statlink { if $list_options then @@ -1215,11 +1334,9 @@ function _spack_view_status { _spack_view_statlink } -function _spack_view_symlink { - if $list_options - then - compgen -W "-h --help -i --ignore-conflicts" -- "$cur" - fi +function _spack_view_check { + # Alias for `spack view statlink` + _spack_view_statlink } # Helper functions for subcommands @@ -1264,6 +1381,10 @@ function _environments { spack env list } +function _keys { + spack gpg list +} + # Testing functions function _test_vars { From 4da8f7fceff7c9f829ed99823c288abf6b81db48 Mon Sep 17 00:00:00 2001 From: Sajid Ali <30510036+s-sajid-ali@users.noreply.github.com> Date: Thu, 2 Jan 2020 15:30:11 -0600 Subject: [PATCH 4/8] RHEL8 bugfix for module_cmd (#14349) --- lib/spack/spack/util/module_cmd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/util/module_cmd.py b/lib/spack/spack/util/module_cmd.py index 4ff6b0de431..d2036707691 100644 --- a/lib/spack/spack/util/module_cmd.py +++ b/lib/spack/spack/util/module_cmd.py @@ -9,6 +9,7 @@ """ import subprocess import os +import sys import json import re @@ -31,7 +32,7 @@ def module(*args): if args[0] in module_change_commands: # Do the module manipulation, then output the environment in JSON # and read the JSON back in the parent process to update os.environ - module_cmd += ' >/dev/null; python -c %s' % py_cmd + module_cmd += ' >/dev/null;' + sys.executable + ' -c %s' % py_cmd module_p = subprocess.Popen(module_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, From f9f28e8fbaaf6bbc3fa72050bfce34d6fad25173 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 16 Jan 2020 15:46:18 -0600 Subject: [PATCH 5/8] Fix use of sys.executable for module/env commands (#14496) * Fix use of sys.executable for module/env commands * Fix unit tests * More consistent quotation, less duplication * Fix import syntax --- lib/spack/spack/util/environment.py | 5 +++-- lib/spack/spack/util/module_cmd.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index f7dc728e7cb..83b350d1c79 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -17,6 +17,7 @@ import llnl.util.tty as tty import spack.util.executable as executable +from spack.util.module_cmd import py_cmd from llnl.util.lang import dedupe @@ -918,8 +919,8 @@ def _source_single_file(file_and_args, environment): source_file.extend(x for x in file_and_args) source_file = ' '.join(source_file) - dump_cmd = 'import os, json; print(json.dumps(dict(os.environ)))' - dump_environment = sys.executable + ' -c "{0}"'.format(dump_cmd) + dump_environment = 'PYTHONHOME="{0}" "{1}" -c "{2}"'.format( + sys.prefix, sys.executable, py_cmd) # Try to source the file source_file_arguments = ' '.join([ diff --git a/lib/spack/spack/util/module_cmd.py b/lib/spack/spack/util/module_cmd.py index d2036707691..1781e050321 100644 --- a/lib/spack/spack/util/module_cmd.py +++ b/lib/spack/spack/util/module_cmd.py @@ -18,7 +18,7 @@ # This list is not exhaustive. Currently we only use load and unload # If we need another option that changes the environment, add it here. module_change_commands = ['load', 'swap', 'unload', 'purge', 'use', 'unuse'] -py_cmd = "'import os;import json;print(json.dumps(dict(os.environ)))'" +py_cmd = 'import os; import json; print(json.dumps(dict(os.environ)))' # This is just to enable testing. I hate it but we can't find a better way _test_mode = False @@ -32,7 +32,8 @@ def module(*args): if args[0] in module_change_commands: # Do the module manipulation, then output the environment in JSON # and read the JSON back in the parent process to update os.environ - module_cmd += ' >/dev/null;' + sys.executable + ' -c %s' % py_cmd + module_cmd += ' > /dev/null; PYTHONHOME="{0}" "{1}" -c "{2}"'.format( + sys.prefix, sys.executable, py_cmd) module_p = subprocess.Popen(module_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, From 010f9451c9d27fb1e8b73802876123573c872d17 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Sat, 25 Jan 2020 01:49:45 +0100 Subject: [PATCH 6/8] bugfix: make `_source_single_file` work in venvs (#14569) Using `sys.executable` to run Python in a sub-shell doesn't always work in a virtual environment as the `sys.executable` Python is not necessarily compatible with any loaded spack/other virtual environment. - revert use of sys.executable to print out subshell environment (#14496) - try instead to use an available python, then if there *is not* one, use `sys.executable` - this addresses RHEL8 (where there is no `python` and `PYTHONHOME` issue in a simpler way --- lib/spack/spack/util/environment.py | 11 ++++++++--- lib/spack/spack/util/module_cmd.py | 5 ++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index 83b350d1c79..bfd6300ec8f 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -17,7 +17,6 @@ import llnl.util.tty as tty import spack.util.executable as executable -from spack.util.module_cmd import py_cmd from llnl.util.lang import dedupe @@ -919,8 +918,14 @@ def _source_single_file(file_and_args, environment): source_file.extend(x for x in file_and_args) source_file = ' '.join(source_file) - dump_environment = 'PYTHONHOME="{0}" "{1}" -c "{2}"'.format( - sys.prefix, sys.executable, py_cmd) + # If the environment contains 'python' use it, if not + # go with sys.executable. Below we just need a working + # Python interpreter, not necessarily sys.executable. + python_cmd = executable.which('python3', 'python', 'python2') + python_cmd = python_cmd.name if python_cmd else sys.executable + + dump_cmd = 'import os, json; print(json.dumps(dict(os.environ)))' + dump_environment = python_cmd + ' -c "{0}"'.format(dump_cmd) # Try to source the file source_file_arguments = ' '.join([ diff --git a/lib/spack/spack/util/module_cmd.py b/lib/spack/spack/util/module_cmd.py index 1781e050321..d2036707691 100644 --- a/lib/spack/spack/util/module_cmd.py +++ b/lib/spack/spack/util/module_cmd.py @@ -18,7 +18,7 @@ # This list is not exhaustive. Currently we only use load and unload # If we need another option that changes the environment, add it here. module_change_commands = ['load', 'swap', 'unload', 'purge', 'use', 'unuse'] -py_cmd = 'import os; import json; print(json.dumps(dict(os.environ)))' +py_cmd = "'import os;import json;print(json.dumps(dict(os.environ)))'" # This is just to enable testing. I hate it but we can't find a better way _test_mode = False @@ -32,8 +32,7 @@ def module(*args): if args[0] in module_change_commands: # Do the module manipulation, then output the environment in JSON # and read the JSON back in the parent process to update os.environ - module_cmd += ' > /dev/null; PYTHONHOME="{0}" "{1}" -c "{2}"'.format( - sys.prefix, sys.executable, py_cmd) + module_cmd += ' >/dev/null;' + sys.executable + ' -c %s' % py_cmd module_p = subprocess.Popen(module_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, From a8d5c6ccf22cba113e60efbd6f0d2142f61bce14 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 7 Feb 2020 16:51:44 -0600 Subject: [PATCH 7/8] version bump: 0.13.4 --- lib/spack/spack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 00744096613..953ec89f9f4 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -5,7 +5,7 @@ #: major, minor, patch version for Spack, in a tuple -spack_version_info = (0, 13, 3) +spack_version_info = (0, 13, 4) #: String containing Spack version joined with .'s spack_version = '.'.join(str(v) for v in spack_version_info) From 0311b63e0bf9c88cbfc9ddf470fcd1f674f24073 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 7 Feb 2020 16:52:07 -0600 Subject: [PATCH 8/8] update CHANGELOG.md for 0.13.4 --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b2a1cf9f01..fef5553c8fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# v0.13.4 (2020-02-07) + +This release contains several bugfixes: + +* bugfixes for invoking python in various environments (#14349, #14496, #14569) +* brought tab completion up to date (#14392) +* bugfix for removing extensions from views in order (#12961) +* bugfix for nondeterministic hashing for specs with externals (#14390) + # v0.13.3 (2019-12-23) This release contains more major performance improvements for Spack