zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

env: consolidate most of spack env status into spack find

Browse Source 
- `spack env status` used to show install status; consolidate that into
  `spack find`.

- `spack env status` will still print out whether there is an active
  environment
This commit is contained in:
Todd Gamblin 2018-11-02 12:43:02 -07:00
parent 26a55ff749
commit efad7ac81b
7 changed files with 259 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
lib/spack/spack/cmd/__init__.py
View File

@ -175,7 +175,8 @@ def disambiguate_spec(spec):
def gray_hash(spec, length):
return colorize('@K{%s}' % spec.dag_hash(length))
h = spec.dag_hash(length) if spec.concrete else '-' * length
return colorize('@K{%s}' % h)
def display_specs(specs, args=None, **kwargs):
@ -209,7 +210,9 @@ def display_specs(specs, args=None, **kwargs):
show_flags (bool): Show compiler flags with specs
variants (bool): Show variants with specs
indent (int): indent each line this much
decorators (dict): dictionary mappng specs to decorators
header_callback (function): called at start of arch/compiler sections
all_headers (bool): show headers even when arch/compiler aren't defined
"""
def get_arg(name, default=None):
"""Prefer kwargs, then args, then default."""
@ -220,12 +223,17 @@ def get_arg(name, default=None):
else:
return default
mode = get_arg('mode', 'short')
hashes = get_arg('long', False)
namespace = get_arg('namespace', False)
flags = get_arg('show_flags', False)
mode = get_arg('mode', 'short')
hashes = get_arg('long', False)
namespace = get_arg('namespace', False)
flags = get_arg('show_flags', False)
full_compiler = get_arg('show_full_compiler', False)
variants = get_arg('variants', False)
variants = get_arg('variants', False)
all_headers = get_arg('all_headers', False)
decorator = get_arg('decorator', None)
if decorator is None:
decorator = lambda s, f: f
indent = get_arg('indent', 0)
ispace = indent * ' '
@ -235,7 +243,7 @@ def get_arg(name, default=None):
hashes = True
hlen = None
nfmt = '.' if namespace else '_'
nfmt = '{fullpackage}' if namespace else '{package}'
ffmt = ''
if full_compiler or flags:
ffmt += '$%'
@ -247,35 +255,46 @@ def get_arg(name, default=None):
# Make a dict with specs keyed by architecture and compiler.
index = index_by(specs, ('architecture', 'compiler'))
transform = {'package': decorator, 'fullpackage': decorator}
# Traverse the index and print out each package
for i, (architecture, compiler) in enumerate(sorted(index)):
if i > 0:
print()
header = "%s{%s} / %s{%s}" % (spack.spec.architecture_color,
architecture, spack.spec.compiler_color,
compiler)
header = "%s{%s} / %s{%s}" % (
spack.spec.architecture_color,
architecture if architecture else 'no arch',
spack.spec.compiler_color,
compiler if compiler else 'no compiler')
# Sometimes we want to display specs that are not yet concretized.
# If they don't have a compiler / architecture attached to them,
# then skip the header
if architecture is not None or compiler is not None:
if all_headers or (architecture is not None or compiler is not None):
sys.stdout.write(ispace)
tty.hline(colorize(header), char='-')
specs = index[(architecture, compiler)]
specs.sort()
abbreviated = [s.cformat(format_string) for s in specs]
if mode == 'paths':
# Print one spec per line along with prefix path
abbreviated = [s.cformat(format_string, transform=transform)
for s in specs]
width = max(len(s) for s in abbreviated)
width += 2
format = " %%-%ds%%s" % width
for abbrv, spec in zip(abbreviated, specs):
prefix = gray_hash(spec, hlen) if hashes else ''
print(ispace + prefix + (format % (abbrv, spec.prefix)))
# optional hash prefix for paths
h = gray_hash(spec, hlen) if hashes else ''
# only show prefix for concrete specs
prefix = spec.prefix if spec.concrete else ''
# print it all out at once
fmt = "%%s%%s %%-%ds%%s" % width
print(fmt % (ispace, h, abbrv, prefix))
elif mode == 'deps':
for spec in specs:
@ -285,24 +304,25 @@ def get_arg(name, default=None):
prefix=(lambda s: gray_hash(s, hlen)) if hashes else None))
elif mode == 'short':
# Print columns of output if not printing flags
def fmt(s):
string = ""
if hashes:
string += gray_hash(s, hlen) + ' '
string += s.cformat(
'$%s$@%s' % (nfmt, vfmt), transform=transform)
return string
if not flags and not full_compiler:
def fmt(s):
string = ""
if hashes:
string += gray_hash(s, hlen) + ' '
string += s.cformat('$-%s$@%s' % (nfmt, vfmt))
return string
# Print columns of output if not printing flags
colify((fmt(s) for s in specs), indent=indent)
# Print one entry per line if including flags
else:
# Print one entry per line if including flags
for spec in specs:
# Print the hash if necessary
hsh = gray_hash(spec, hlen) + ' ' if hashes else ''
print(ispace + hsh + spec.cformat(format_string))
print(ispace + hsh + spec.cformat(
format_string, transform=transform))
else:
raise ValueError(

23
lib/spack/spack/cmd/env.py
View File

@ -271,26 +271,19 @@ def env_list(args):
# env status
#
def env_status_setup_parser(subparser):
"""get install status of specs in an environment"""
subparser.add_argument(
'env', nargs='?', help='name of environment to show status for')
arguments.add_common_arguments(
subparser,
['recurse_dependencies', 'long', 'very_long'])
"""print whether there is an active environment"""
def env_status(args):
env = ev.get_env(args, 'env status', required=False)
if not env:
if env:
if env.path == os.getcwd():
tty.msg('Using %s in current directory: %s'
% (ev.manifest_name, env.path))
else:
tty.msg('In environment %s' % env.name)
else:
tty.msg('No active environment')
return
# TODO: option to show packages w/ multiple instances?
env.status(
sys.stdout, recurse_dependencies=args.recurse_dependencies,
hashes=args.long or args.very_long,
hashlen=None if args.very_long else 7,
install_status=True)
#

70
lib/spack/spack/cmd/find.py
View File

@ -2,15 +2,17 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import sys
from __future__ import print_function
import llnl.util.tty as tty
import llnl.util.tty.color as color
import llnl.util.lang
import spack.environment as ev
import spack.repo
import spack.cmd.common.arguments as arguments
from spack.cmd import display_specs
from spack.util.string import plural
description = "list and search installed packages"
section = "basic"
@ -40,6 +42,9 @@ def setup_parser(subparser):
arguments.add_common_arguments(
subparser, ['long', 'very_long', 'tags'])
subparser.add_argument('-c', '--show-concretized',
action='store_true',
help='show concretized specs in an environment')
subparser.add_argument('-f', '--show-flags',
action='store_true',
dest='show_flags',
@ -116,12 +121,42 @@ def query_arguments(args):
return q_args
def setup_env(env):
"""Create a function for decorating specs when in an environment."""
def strip_build(seq):
return set(s.copy(deps=('link', 'run')) for s in seq)
added = set(strip_build(env.added_specs()))
roots = set(strip_build(env.roots()))
removed = set(strip_build(env.removed_specs()))
def decorator(spec, fmt):
# add +/-/* to show added/removed/root specs
if spec in roots:
return color.colorize('@*{%s}' % fmt)
elif spec in removed:
return color.colorize('@K{%s}' % fmt)
else:
return '%s' % fmt
return decorator, added, roots, removed
def find(parser, args):
q_args = query_arguments(args)
query_specs = args.specs(**q_args)
results = args.specs(**q_args)
decorator = lambda s, f: f
added = set()
removed = set()
env = ev.get_env(args, 'find', required=False)
if env:
decorator, added, roots, removed = setup_env(env)
# Exit early if no package matches the constraint
if not query_specs and args.constraint:
if not results and args.constraint:
msg = "No package matches the query: {0}"
msg = msg.format(' '.join(args.constraint))
tty.msg(msg)
@ -130,10 +165,29 @@ def find(parser, args):
# If tags have been specified on the command line, filter by tags
if args.tags:
packages_with_tags = spack.repo.path.packages_with_tags(*args.tags)
query_specs = [x for x in query_specs if x.name in packages_with_tags]
results = [x for x in results if x.name in packages_with_tags]
# Display the result
if sys.stdout.isatty():
tty.msg("%d installed packages." % len(query_specs))
if env:
tty.msg('In environment %s' % env.name)
display_specs(query_specs, args)
print()
tty.msg('Root specs')
if not env.user_specs:
print('none')
else:
display_specs(
env.user_specs, args,
decorator=lambda s, f: color.colorize('@*{%s}' % f))
print()
if args.show_concretized:
tty.msg('Concretized roots')
display_specs(
env.specs_by_hash.values(), args, decorator=decorator)
print()
tty.msg("%s" % plural(len(results), 'installed package'))
display_specs(results, args, decorator=decorator, all_headers=True)

94
lib/spack/spack/environment.py
View File

@ -13,7 +13,6 @@
import llnl.util.filesystem as fs
import llnl.util.tty as tty
import llnl.util.tty.color as color
import spack.error
import spack.repo
@ -152,7 +151,7 @@ def get_env(args, cmd_name, required=True):
"""
# try arguments
env = args.env
env = getattr(args, 'env', None)
# try a manifest file in the current directory
if not env:
@ -356,6 +355,11 @@ def clear(self):
self._repo = None # RepoPath for this env (memoized)
self._previous_active = None # previously active environment
@property
def internal(self):
"""Whether this environment is managed by Spack."""
return self.path.startswith(env_path)
@property
def name(self):
"""Human-readable representation of the environment.
@ -363,7 +367,7 @@ def name(self):
This is the path for directory environments, and just the name
for named environments.
"""
if self.path.startswith(env_path):
if self.internal:
return os.path.basename(self.path)
else:
return self.path
@ -625,46 +629,6 @@ def install_all(self, args=None):
os.remove(build_log_link)
os.symlink(spec.package.build_log_path, build_log_link)
def status(self, stream, **kwargs):
"""List the specs in an environment."""
tty.msg('In environment %s' % self.name)
concretized = [(spec, self.specs_by_hash[h])
for spec, h in zip(self.concretized_user_specs,
self.concretized_order)]
added = [s for s in self.user_specs
if s not in self.concretized_user_specs]
removed = [(s, c) for s, c in concretized if s not in self.user_specs]
current = [(s, c) for s, c in concretized if s in self.user_specs]
def write_kind(s):
color.cwrite('@c{%s}\n' % color.cescape(s), stream)
def write_user_spec(s, c):
color.cwrite('@%s{----} %s\n' % (c, color.cescape(s)), stream)
if added:
write_kind('added:')
for s in added:
write_user_spec(s, 'g')
if current:
if added:
stream.write('\n')
write_kind('concrete:')
for s, c in current:
write_user_spec(s, 'K')
stream.write(c.tree(**kwargs))
if removed:
if added or current:
stream.write('\n')
write_kind('removed:')
for s, c in removed:
write_user_spec(s, 'r')
stream.write(c.tree(**kwargs))
def all_specs_by_hash(self):
"""Map of hashes to spec for all specs in this environment."""
hashes = {}
@ -682,6 +646,50 @@ def all_hashes(self):
"""Return all specs, even those a user spec would shadow."""
return list(self.all_specs_by_hash().keys())
def roots(self):
"""Specs explicitly requested by the user *in this environment*.
Yields both added and installed specs that have user specs in
`spack.yaml`.
"""
concretized = dict(self.concretized_specs())
for spec in self.user_specs:
concrete = concretized.get(spec)
yield concrete if concrete else spec
def added_specs(self):
"""Specs that are not yet installed.
Yields the user spec for non-concretized specs, and the concrete
spec for already concretized but not yet installed specs.
"""
concretized = dict(self.concretized_specs())
for spec in self.user_specs:
concrete = concretized.get(spec)
if not concrete:
yield spec
elif not concrete.package.installed:
yield concrete
def concretized_specs(self):
"""Tuples of (user spec, concrete spec) for all concrete specs."""
for s, h in zip(self.concretized_user_specs, self.concretized_order):
yield (s, self.specs_by_hash[h])
def removed_specs(self):
"""Tuples of (user spec, concrete spec) for all specs that will be
removed on nexg concretize."""
needed = set()
for s, c in self.concretized_specs():
if s in self.user_specs:
for d in c.traverse():
needed.add(d)
for s, c in self.concretized_specs():
for d in c.traverse():
if d not in needed:
yield d
def _get_environment_specs(self, recurse_dependencies=True):
"""Returns the specs of all the packages in an environment.

2
lib/spack/spack/modules/common.py
View File

@ -546,7 +546,7 @@ def environment_modifications(self):
# tokens uppercase.
transform = {}
for token in _valid_tokens:
transform[token] = str.upper
transform[token] = lambda spec, string: str.upper(string)
for x in env:
# Ensure all the tokens are valid in this context

63
lib/spack/spack/spec.py
View File

@ -2906,7 +2906,7 @@ def _cmp_node(self):
"""Comparison key for just *this node* and not its deps."""
return (self.name,
self.namespace,
self.versions,
tuple(self.versions),
self.variants,
self.architecture,
self.compiler,
@ -2964,6 +2964,7 @@ def format(self, format_string='$_$@$%@+$+$=', **kwargs):
You can also use full-string versions, which elide the prefixes::
${PACKAGE} Package name
${FULLPACKAGE} Full package name (with namespace)
${VERSION} Version
${COMPILER} Full compiler string
${COMPILERNAME} Compiler name
@ -2995,11 +2996,9 @@ def format(self, format_string='$_$@$%@+$+$=', **kwargs):
Args:
format_string (str): string containing the format to be expanded
**kwargs (dict): the following list of keywords is supported
- color (bool): True if returned string is colored
- transform (dict): maps full-string formats to a callable \
Keyword Args:
color (bool): True if returned string is colored
transform (dict): maps full-string formats to a callable \
that accepts a string and returns another one
Examples:
@ -3019,16 +3018,18 @@ def format(self, format_string='$_$@$%@+$+$=', **kwargs):
color = kwargs.get('color', False)
# Dictionary of transformations for named tokens
token_transforms = {}
token_transforms.update(kwargs.get('transform', {}))
token_transforms = dict(
(k.upper(), v) for k, v in kwargs.get('transform', {}).items())
length = len(format_string)
out = StringIO()
named = escape = compiler = False
named_str = fmt = ''
def write(s, c):
f = color_formats[c] + cescape(s) + '@.'
def write(s, c=None):
f = cescape(s)
if c is not None:
f = color_formats[c] + f + '@.'
cwrite(f, stream=out, color=color)
iterator = enumerate(format_string)
@ -3048,7 +3049,8 @@ def write(s, c):
name = self.name if self.name else ''
out.write(fmt % name)
elif c == '.':
out.write(fmt % self.fullname)
name = self.fullname if self.fullname else ''
out.write(fmt % name)
elif c == '@':
if self.versions and self.versions != _any_version:
write(fmt % (c + str(self.versions)), c)
@ -3103,60 +3105,63 @@ def write(s, c):
#
# The default behavior is to leave the string unchanged
# (`lambda x: x` is the identity function)
token_transform = token_transforms.get(named_str, lambda x: x)
transform = token_transforms.get(named_str, lambda s, x: x)
if named_str == 'PACKAGE':
name = self.name if self.name else ''
write(fmt % token_transform(name), '@')
if named_str == 'VERSION':
write(fmt % transform(self, name))
elif named_str == 'FULLPACKAGE':
name = self.fullname if self.fullname else ''
write(fmt % transform(self, name))
elif named_str == 'VERSION':
if self.versions and self.versions != _any_version:
write(fmt % token_transform(str(self.versions)), '@')
write(fmt % transform(self, str(self.versions)), '@')
elif named_str == 'COMPILER':
if self.compiler:
write(fmt % token_transform(self.compiler), '%')
write(fmt % transform(self, self.compiler), '%')
elif named_str == 'COMPILERNAME':
if self.compiler:
write(fmt % token_transform(self.compiler.name), '%')
write(fmt % transform(self, self.compiler.name), '%')
elif named_str in ['COMPILERVER', 'COMPILERVERSION']:
if self.compiler:
write(
fmt % token_transform(self.compiler.versions),
fmt % transform(self, self.compiler.versions),
'%'
)
elif named_str == 'COMPILERFLAGS':
if self.compiler:
write(
fmt % token_transform(str(self.compiler_flags)),
fmt % transform(self, str(self.compiler_flags)),
'%'
)
elif named_str == 'OPTIONS':
if self.variants:
write(fmt % token_transform(str(self.variants)), '+')
write(fmt % transform(self, str(self.variants)), '+')
elif named_str in ["ARCHITECTURE", "PLATFORM", "TARGET", "OS"]:
if self.architecture and str(self.architecture):
if named_str == "ARCHITECTURE":
write(
fmt % token_transform(str(self.architecture)),
fmt % transform(self, str(self.architecture)),
'='
)
elif named_str == "PLATFORM":
platform = str(self.architecture.platform)
write(fmt % token_transform(platform), '=')
write(fmt % transform(self, platform), '=')
elif named_str == "OS":
operating_sys = str(self.architecture.platform_os)
write(fmt % token_transform(operating_sys), '=')
write(fmt % transform(self, operating_sys), '=')
elif named_str == "TARGET":
target = str(self.architecture.target)
write(fmt % token_transform(target), '=')
write(fmt % transform(self, target), '=')
elif named_str == 'SHA1':
if self.dependencies:
out.write(fmt % token_transform(str(self.dag_hash(7))))
out.write(fmt % transform(self, str(self.dag_hash(7))))
elif named_str == 'SPACK_ROOT':
out.write(fmt % token_transform(spack.paths.prefix))
out.write(fmt % transform(self, spack.paths.prefix))
elif named_str == 'SPACK_INSTALL':
out.write(fmt % token_transform(spack.store.root))
out.write(fmt % transform(self, spack.store.root))
elif named_str == 'PREFIX':
out.write(fmt % token_transform(self.prefix))
out.write(fmt % transform(self, self.prefix))
elif named_str.startswith('HASH'):
if named_str.startswith('HASH:'):
_, hashlen = named_str.split(':')
@ -3165,7 +3170,7 @@ def write(s, c):
hashlen = None
out.write(fmt % (self.dag_hash(hashlen)))
elif named_str == 'NAMESPACE':
out.write(fmt % token_transform(self.namespace))
out.write(fmt % transform(self.namespace))
named = False

79
lib/spack/spack/test/cmd/env.py
View File

@ -21,13 +21,14 @@
pytestmark = pytest.mark.usefixtures(
'mutable_mock_env_path', 'config', 'mutable_mock_packages')
env = SpackCommand('env')
install = SpackCommand('install')
add = SpackCommand('add')
remove = SpackCommand('remove')
concretize = SpackCommand('concretize')
stage = SpackCommand('stage')
uninstall = SpackCommand('uninstall')
env = SpackCommand('env')
install = SpackCommand('install')
add = SpackCommand('add')
remove = SpackCommand('remove')
concretize = SpackCommand('concretize')
stage = SpackCommand('stage')
uninstall = SpackCommand('uninstall')
find = SpackCommand('find')
def test_add():
@ -156,32 +157,62 @@ def test_remove_after_concretize():
def test_remove_command():
env('create', 'test')
assert 'test' in env('list')
with ev.read('test'):
add('mpileaks')
assert 'mpileaks' in env('status', 'test')
assert 'mpileaks' in find()
assert 'mpileaks@' not in find()
assert 'mpileaks@' not in find('--show-concretized')
with ev.read('test'):
remove('mpileaks')
assert 'mpileaks' not in env('status', 'test')
assert 'mpileaks' not in find()
assert 'mpileaks@' not in find()
assert 'mpileaks@' not in find('--show-concretized')
with ev.read('test'):
add('mpileaks')
assert 'mpileaks' in env('status', 'test')
assert 'mpileaks' in find()
assert 'mpileaks@' not in find()
assert 'mpileaks@' not in find('--show-concretized')
with ev.read('test'):
concretize()
assert 'mpileaks' in find()
assert 'mpileaks@' not in find()
assert 'mpileaks@' in find('--show-concretized')
with ev.read('test'):
remove('mpileaks')
assert 'mpileaks' not in find()
# removed but still in last concretized specs
assert 'mpileaks@' in find('--show-concretized')
with ev.read('test'):
concretize()
assert 'mpileaks' not in find()
assert 'mpileaks@' not in find()
# now the lockfile is regenerated and it's gone.
assert 'mpileaks@' not in find('--show-concretized')
def test_environment_status():
e = ev.create('test')
e.add('mpileaks')
e.concretize()
e.add('python')
mock_stream = StringIO()
e.status(mock_stream)
list_content = mock_stream.getvalue()
assert 'mpileaks' in list_content
assert 'python' in list_content
mpileaks_spec = e.specs_by_hash[e.concretized_order[0]]
assert mpileaks_spec.format() in list_content
def test_environment_status(capfd, tmpdir):
with capfd.disabled():
with tmpdir.as_cwd():
assert 'No active environment' in env('status')
with ev.create('test'):
assert 'In environment test' in env('status')
with ev.Environment('local_dir'):
assert os.path.join(os.getcwd(), 'local_dir') in env('status')
e = ev.Environment('myproject')
e.write()
with tmpdir.join('myproject').as_cwd():
with e:
assert 'in current directory' in env('status')
def test_to_lockfile_dict():
@ -309,8 +340,8 @@ def test_init_with_file_and_remove(tmpdir):
out = env('list')
assert 'test' in out
out = env('status', 'test')
assert 'mpileaks' in out
with ev.read('test'):
assert 'mpileaks' in find()
env('remove', '-y', 'test')