env: consolidate most of spack env status into spack find

- `spack env status` used to show install status; consolidate that into
  `spack find`.

- `spack env status` will still print out whether there is an active
  environment

lib/spack/spack/cmd/__init__.py

@@ -175,7 +175,8 @@ def disambiguate_spec(spec):
 
 
 def gray_hash(spec, length):
-    return colorize('@K{%s}' % spec.dag_hash(length))
+    h = spec.dag_hash(length) if spec.concrete else '-' * length
+    return colorize('@K{%s}' % h)
 
 
 def display_specs(specs, args=None, **kwargs):
@@ -209,7 +210,9 @@ def display_specs(specs, args=None, **kwargs):
         show_flags (bool): Show compiler flags with specs
         variants (bool): Show variants with specs
         indent (int): indent each line this much
-
+        decorators (dict): dictionary mappng specs to decorators
+        header_callback (function): called at start of arch/compiler sections
+        all_headers (bool): show headers even when arch/compiler aren't defined
     """
     def get_arg(name, default=None):
         """Prefer kwargs, then args, then default."""
@@ -226,6 +229,11 @@ def get_arg(name, default=None):
     flags         = get_arg('show_flags', False)
     full_compiler = get_arg('show_full_compiler', False)
     variants      = get_arg('variants', False)
+    all_headers   = get_arg('all_headers', False)
+
+    decorator     = get_arg('decorator', None)
+    if decorator is None:
+        decorator = lambda s, f: f
 
     indent = get_arg('indent', 0)
     ispace = indent * ' '
@@ -235,7 +243,7 @@ def get_arg(name, default=None):
         hashes = True
         hlen = None
 
-    nfmt = '.' if namespace else '_'
+    nfmt = '{fullpackage}' if namespace else '{package}'
     ffmt = ''
     if full_compiler or flags:
         ffmt += '$%'
@@ -247,35 +255,46 @@ def get_arg(name, default=None):
 
     # Make a dict with specs keyed by architecture and compiler.
     index = index_by(specs, ('architecture', 'compiler'))
+    transform = {'package': decorator, 'fullpackage': decorator}
 
     # Traverse the index and print out each package
     for i, (architecture, compiler) in enumerate(sorted(index)):
         if i > 0:
             print()
 
-        header = "%s{%s} / %s{%s}" % (spack.spec.architecture_color,
+        header = "%s{%s} / %s{%s}" % (
-                                      architecture, spack.spec.compiler_color,
+            spack.spec.architecture_color,
-                                      compiler)
+            architecture if architecture else 'no arch',
+            spack.spec.compiler_color,
+            compiler if compiler else 'no compiler')
+
         # Sometimes we want to display specs that are not yet concretized.
         # If they don't have a compiler / architecture attached to them,
         # then skip the header
-        if architecture is not None or compiler is not None:
+        if all_headers or (architecture is not None or compiler is not None):
             sys.stdout.write(ispace)
             tty.hline(colorize(header), char='-')
 
         specs = index[(architecture, compiler)]
         specs.sort()
 
-        abbreviated = [s.cformat(format_string) for s in specs]
         if mode == 'paths':
             # Print one spec per line along with prefix path
+            abbreviated = [s.cformat(format_string, transform=transform)
+                           for s in specs]
             width = max(len(s) for s in abbreviated)
             width += 2
-            format = "    %%-%ds%%s" % width
 
             for abbrv, spec in zip(abbreviated, specs):
-                prefix = gray_hash(spec, hlen) if hashes else ''
+                # optional hash prefix for paths
-                print(ispace + prefix + (format % (abbrv, spec.prefix)))
+                h = gray_hash(spec, hlen) if hashes else ''
+
+                # only show prefix for concrete specs
+                prefix = spec.prefix if spec.concrete else ''
+
+                # print it all out at once
+                fmt = "%%s%%s    %%-%ds%%s" % width
+                print(fmt % (ispace, h, abbrv, prefix))
 
         elif mode == 'deps':
             for spec in specs:
@@ -285,24 +304,25 @@ def get_arg(name, default=None):
                     prefix=(lambda s: gray_hash(s, hlen)) if hashes else None))
 
         elif mode == 'short':
-            # Print columns of output if not printing flags
-            if not flags and not full_compiler:
-
             def fmt(s):
                 string = ""
                 if hashes:
                     string += gray_hash(s, hlen) + ' '
-                    string += s.cformat('$-%s$@%s' % (nfmt, vfmt))
+                string += s.cformat(
-
+                    '$%s$@%s' % (nfmt, vfmt), transform=transform)
                 return string
 
+            if not flags and not full_compiler:
+                # Print columns of output if not printing flags
                 colify((fmt(s) for s in specs), indent=indent)
-            # Print one entry per line if including flags
+
             else:
+                # Print one entry per line if including flags
                 for spec in specs:
                     # Print the hash if necessary
                     hsh = gray_hash(spec, hlen) + ' ' if hashes else ''
-                    print(ispace + hsh + spec.cformat(format_string))
+                    print(ispace + hsh + spec.cformat(
+                        format_string, transform=transform))
 
         else:
             raise ValueError(

lib/spack/spack/cmd/env.py

@@ -271,26 +271,19 @@ def env_list(args):
 # env status
 #
 def env_status_setup_parser(subparser):
-    """get install status of specs in an environment"""
+    """print whether there is an active environment"""
-    subparser.add_argument(
-        'env', nargs='?', help='name of environment to show status for')
-    arguments.add_common_arguments(
-        subparser,
-        ['recurse_dependencies', 'long', 'very_long'])
 
 
 def env_status(args):
     env = ev.get_env(args, 'env status', required=False)
-    if not env:
+    if env:
+        if env.path == os.getcwd():
+            tty.msg('Using %s in current directory: %s'
+                    % (ev.manifest_name, env.path))
+        else:
+            tty.msg('In environment %s' % env.name)
+    else:
         tty.msg('No active environment')
-        return
-
-    # TODO: option to show packages w/ multiple instances?
-    env.status(
-        sys.stdout, recurse_dependencies=args.recurse_dependencies,
-        hashes=args.long or args.very_long,
-        hashlen=None if args.very_long else 7,
-        install_status=True)
 
 
 #

lib/spack/spack/cmd/find.py

@@ -2,15 +2,17 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
+from __future__ import print_function
-import sys
 
 import llnl.util.tty as tty
+import llnl.util.tty.color as color
 import llnl.util.lang
 
+import spack.environment as ev
 import spack.repo
 import spack.cmd.common.arguments as arguments
 from spack.cmd import display_specs
+from spack.util.string import plural
 
 description = "list and search installed packages"
 section = "basic"
@@ -40,6 +42,9 @@ def setup_parser(subparser):
     arguments.add_common_arguments(
         subparser, ['long', 'very_long', 'tags'])
 
+    subparser.add_argument('-c', '--show-concretized',
+                           action='store_true',
+                           help='show concretized specs in an environment')
     subparser.add_argument('-f', '--show-flags',
                            action='store_true',
                            dest='show_flags',
@@ -116,12 +121,42 @@ def query_arguments(args):
     return q_args
 
 
+def setup_env(env):
+    """Create a function for decorating specs when in an environment."""
+
+    def strip_build(seq):
+        return set(s.copy(deps=('link', 'run')) for s in seq)
+
+    added = set(strip_build(env.added_specs()))
+    roots = set(strip_build(env.roots()))
+    removed = set(strip_build(env.removed_specs()))
+
+    def decorator(spec, fmt):
+        # add +/-/* to show added/removed/root specs
+        if spec in roots:
+            return color.colorize('@*{%s}' % fmt)
+        elif spec in removed:
+            return color.colorize('@K{%s}' % fmt)
+        else:
+            return '%s' % fmt
+
+    return decorator, added, roots, removed
+
+
 def find(parser, args):
     q_args = query_arguments(args)
-    query_specs = args.specs(**q_args)
+    results = args.specs(**q_args)
+
+    decorator = lambda s, f: f
+    added = set()
+    removed = set()
+
+    env = ev.get_env(args, 'find', required=False)
+    if env:
+        decorator, added, roots, removed = setup_env(env)
 
     # Exit early if no package matches the constraint
-    if not query_specs and args.constraint:
+    if not results and args.constraint:
         msg = "No package matches the query: {0}"
         msg = msg.format(' '.join(args.constraint))
         tty.msg(msg)
@@ -130,10 +165,29 @@ def find(parser, args):
     # If tags have been specified on the command line, filter by tags
     if args.tags:
         packages_with_tags = spack.repo.path.packages_with_tags(*args.tags)
-        query_specs = [x for x in query_specs if x.name in packages_with_tags]
+        results = [x for x in results if x.name in packages_with_tags]
 
     # Display the result
-    if sys.stdout.isatty():
+    if env:
-        tty.msg("%d installed packages." % len(query_specs))
+        tty.msg('In environment %s' % env.name)
 
-    display_specs(query_specs, args)
+        print()
+        tty.msg('Root specs')
+
+        if not env.user_specs:
+            print('none')
+        else:
+            display_specs(
+                env.user_specs, args,
+                decorator=lambda s, f: color.colorize('@*{%s}' % f))
+        print()
+
+        if args.show_concretized:
+            tty.msg('Concretized roots')
+            display_specs(
+                env.specs_by_hash.values(), args, decorator=decorator)
+            print()
+
+    tty.msg("%s" % plural(len(results), 'installed package'))
+
+    display_specs(results, args, decorator=decorator, all_headers=True)

lib/spack/spack/environment.py

@@ -13,7 +13,6 @@
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
-import llnl.util.tty.color as color
 
 import spack.error
 import spack.repo
@@ -152,7 +151,7 @@ def get_env(args, cmd_name, required=True):
 
     """
     # try arguments
-    env = args.env
+    env = getattr(args, 'env', None)
 
     # try a manifest file in the current directory
     if not env:
@@ -356,6 +355,11 @@ def clear(self):
         self._repo = None                 # RepoPath for this env (memoized)
         self._previous_active = None      # previously active environment
 
+    @property
+    def internal(self):
+        """Whether this environment is managed by Spack."""
+        return self.path.startswith(env_path)
+
     @property
     def name(self):
         """Human-readable representation of the environment.
@@ -363,7 +367,7 @@ def name(self):
         This is the path for directory environments, and just the name
         for named environments.
         """
-        if self.path.startswith(env_path):
+        if self.internal:
             return os.path.basename(self.path)
         else:
             return self.path
@@ -625,46 +629,6 @@ def install_all(self, args=None):
                     os.remove(build_log_link)
                 os.symlink(spec.package.build_log_path, build_log_link)
 
-    def status(self, stream, **kwargs):
-        """List the specs in an environment."""
-        tty.msg('In environment %s' % self.name)
-
-        concretized = [(spec, self.specs_by_hash[h])
-                       for spec, h in zip(self.concretized_user_specs,
-                                          self.concretized_order)]
-
-        added = [s for s in self.user_specs
-                 if s not in self.concretized_user_specs]
-        removed = [(s, c) for s, c in concretized if s not in self.user_specs]
-        current = [(s, c) for s, c in concretized if s in self.user_specs]
-
-        def write_kind(s):
-            color.cwrite('@c{%s}\n' % color.cescape(s), stream)
-
-        def write_user_spec(s, c):
-            color.cwrite('@%s{----} %s\n' % (c, color.cescape(s)), stream)
-
-        if added:
-            write_kind('added:')
-        for s in added:
-            write_user_spec(s, 'g')
-
-        if current:
-            if added:
-                stream.write('\n')
-            write_kind('concrete:')
-        for s, c in current:
-            write_user_spec(s, 'K')
-            stream.write(c.tree(**kwargs))
-
-        if removed:
-            if added or current:
-                stream.write('\n')
-            write_kind('removed:')
-        for s, c in removed:
-            write_user_spec(s, 'r')
-            stream.write(c.tree(**kwargs))
-
     def all_specs_by_hash(self):
         """Map of hashes to spec for all specs in this environment."""
         hashes = {}
@@ -682,6 +646,50 @@ def all_hashes(self):
         """Return all specs, even those a user spec would shadow."""
         return list(self.all_specs_by_hash().keys())
 
+    def roots(self):
+        """Specs explicitly requested by the user *in this environment*.
+
+        Yields both added and installed specs that have user specs in
+        `spack.yaml`.
+        """
+        concretized = dict(self.concretized_specs())
+        for spec in self.user_specs:
+            concrete = concretized.get(spec)
+            yield concrete if concrete else spec
+
+    def added_specs(self):
+        """Specs that are not yet installed.
+
+        Yields the user spec for non-concretized specs, and the concrete
+        spec for already concretized but not yet installed specs.
+        """
+        concretized = dict(self.concretized_specs())
+        for spec in self.user_specs:
+            concrete = concretized.get(spec)
+            if not concrete:
+                yield spec
+            elif not concrete.package.installed:
+                yield concrete
+
+    def concretized_specs(self):
+        """Tuples of (user spec, concrete spec) for all concrete specs."""
+        for s, h in zip(self.concretized_user_specs, self.concretized_order):
+            yield (s, self.specs_by_hash[h])
+
+    def removed_specs(self):
+        """Tuples of (user spec, concrete spec) for all specs that will be
+           removed on nexg concretize."""
+        needed = set()
+        for s, c in self.concretized_specs():
+            if s in self.user_specs:
+                for d in c.traverse():
+                    needed.add(d)
+
+        for s, c in self.concretized_specs():
+            for d in c.traverse():
+                if d not in needed:
+                    yield d
+
     def _get_environment_specs(self, recurse_dependencies=True):
         """Returns the specs of all the packages in an environment.
 

lib/spack/spack/modules/common.py

@@ -546,7 +546,7 @@ def environment_modifications(self):
         # tokens uppercase.
         transform = {}
         for token in _valid_tokens:
-            transform[token] = str.upper
+            transform[token] = lambda spec, string: str.upper(string)
 
         for x in env:
             # Ensure all the tokens are valid in this context

lib/spack/spack/spec.py

@@ -2906,7 +2906,7 @@ def _cmp_node(self):
         """Comparison key for just *this node* and not its deps."""
         return (self.name,
                 self.namespace,
-                self.versions,
+                tuple(self.versions),
                 self.variants,
                 self.architecture,
                 self.compiler,
@@ -2964,6 +2964,7 @@ def format(self, format_string='$_$@$%@+$+$=', **kwargs):
         You can also use full-string versions, which elide the prefixes::
 
             ${PACKAGE}       Package name
+            ${FULLPACKAGE}   Full package name (with namespace)
             ${VERSION}       Version
             ${COMPILER}      Full compiler string
             ${COMPILERNAME}  Compiler name
@@ -2995,11 +2996,9 @@ def format(self, format_string='$_$@$%@+$+$=', **kwargs):
         Args:
             format_string (str): string containing the format to be expanded
 
-            **kwargs (dict): the following list of keywords is supported
+        Keyword Args:
-
+            color (bool): True if returned string is colored
-                - color (bool): True if returned string is colored
+            transform (dict): maps full-string formats to a callable \
-
-                - transform (dict): maps full-string formats to a callable \
                 that accepts a string and returns another one
 
         Examples:
@@ -3019,16 +3018,18 @@ def format(self, format_string='$_$@$%@+$+$=', **kwargs):
         color = kwargs.get('color', False)
 
         # Dictionary of transformations for named tokens
-        token_transforms = {}
+        token_transforms = dict(
-        token_transforms.update(kwargs.get('transform', {}))
+            (k.upper(), v) for k, v in kwargs.get('transform', {}).items())
 
         length = len(format_string)
         out = StringIO()
         named = escape = compiler = False
         named_str = fmt = ''
 
-        def write(s, c):
+        def write(s, c=None):
-            f = color_formats[c] + cescape(s) + '@.'
+            f = cescape(s)
+            if c is not None:
+                f = color_formats[c] + f + '@.'
             cwrite(f, stream=out, color=color)
 
         iterator = enumerate(format_string)
@@ -3048,7 +3049,8 @@ def write(s, c):
                     name = self.name if self.name else ''
                     out.write(fmt % name)
                 elif c == '.':
-                    out.write(fmt % self.fullname)
+                    name = self.fullname if self.fullname else ''
+                    out.write(fmt % name)
                 elif c == '@':
                     if self.versions and self.versions != _any_version:
                         write(fmt % (c + str(self.versions)), c)
@@ -3103,60 +3105,63 @@ def write(s, c):
                 #
                 # The default behavior is to leave the string unchanged
                 # (`lambda x: x` is the identity function)
-                token_transform = token_transforms.get(named_str, lambda x: x)
+                transform = token_transforms.get(named_str, lambda s, x: x)
 
                 if named_str == 'PACKAGE':
                     name = self.name if self.name else ''
-                    write(fmt % token_transform(name), '@')
+                    write(fmt % transform(self, name))
-                if named_str == 'VERSION':
+                elif named_str == 'FULLPACKAGE':
+                    name = self.fullname if self.fullname else ''
+                    write(fmt % transform(self, name))
+                elif named_str == 'VERSION':
                     if self.versions and self.versions != _any_version:
-                        write(fmt % token_transform(str(self.versions)), '@')
+                        write(fmt % transform(self, str(self.versions)), '@')
                 elif named_str == 'COMPILER':
                     if self.compiler:
-                        write(fmt % token_transform(self.compiler), '%')
+                        write(fmt % transform(self, self.compiler), '%')
                 elif named_str == 'COMPILERNAME':
                     if self.compiler:
-                        write(fmt % token_transform(self.compiler.name), '%')
+                        write(fmt % transform(self, self.compiler.name), '%')
                 elif named_str in ['COMPILERVER', 'COMPILERVERSION']:
                     if self.compiler:
                         write(
-                            fmt % token_transform(self.compiler.versions),
+                            fmt % transform(self, self.compiler.versions),
                             '%'
                         )
                 elif named_str == 'COMPILERFLAGS':
                     if self.compiler:
                         write(
-                            fmt % token_transform(str(self.compiler_flags)),
+                            fmt % transform(self, str(self.compiler_flags)),
                             '%'
                         )
                 elif named_str == 'OPTIONS':
                     if self.variants:
-                        write(fmt % token_transform(str(self.variants)), '+')
+                        write(fmt % transform(self, str(self.variants)), '+')
                 elif named_str in ["ARCHITECTURE", "PLATFORM", "TARGET", "OS"]:
                     if self.architecture and str(self.architecture):
                         if named_str == "ARCHITECTURE":
                             write(
-                                fmt % token_transform(str(self.architecture)),
+                                fmt % transform(self, str(self.architecture)),
                                 '='
                             )
                         elif named_str == "PLATFORM":
                             platform = str(self.architecture.platform)
-                            write(fmt % token_transform(platform), '=')
+                            write(fmt % transform(self, platform), '=')
                         elif named_str == "OS":
                             operating_sys = str(self.architecture.platform_os)
-                            write(fmt % token_transform(operating_sys), '=')
+                            write(fmt % transform(self, operating_sys), '=')
                         elif named_str == "TARGET":
                             target = str(self.architecture.target)
-                            write(fmt % token_transform(target), '=')
+                            write(fmt % transform(self, target), '=')
                 elif named_str == 'SHA1':
                     if self.dependencies:
-                        out.write(fmt % token_transform(str(self.dag_hash(7))))
+                        out.write(fmt % transform(self, str(self.dag_hash(7))))
                 elif named_str == 'SPACK_ROOT':
-                    out.write(fmt % token_transform(spack.paths.prefix))
+                    out.write(fmt % transform(self, spack.paths.prefix))
                 elif named_str == 'SPACK_INSTALL':
-                    out.write(fmt % token_transform(spack.store.root))
+                    out.write(fmt % transform(self, spack.store.root))
                 elif named_str == 'PREFIX':
-                    out.write(fmt % token_transform(self.prefix))
+                    out.write(fmt % transform(self, self.prefix))
                 elif named_str.startswith('HASH'):
                     if named_str.startswith('HASH:'):
                         _, hashlen = named_str.split(':')
@@ -3165,7 +3170,7 @@ def write(s, c):
                         hashlen = None
                     out.write(fmt % (self.dag_hash(hashlen)))
                 elif named_str == 'NAMESPACE':
-                    out.write(fmt % token_transform(self.namespace))
+                    out.write(fmt % transform(self.namespace))
 
                 named = False
 

lib/spack/spack/test/cmd/env.py

@@ -28,6 +28,7 @@
 concretize = SpackCommand('concretize')
 stage      = SpackCommand('stage')
 uninstall  = SpackCommand('uninstall')
+find       = SpackCommand('find')
 
 
 def test_add():
@@ -156,32 +157,62 @@ def test_remove_after_concretize():
 
 def test_remove_command():
     env('create', 'test')
+    assert 'test' in env('list')
 
     with ev.read('test'):
         add('mpileaks')
-    assert 'mpileaks' in env('status', 'test')
+        assert 'mpileaks' in find()
+        assert 'mpileaks@' not in find()
+        assert 'mpileaks@' not in find('--show-concretized')
 
     with ev.read('test'):
         remove('mpileaks')
-    assert 'mpileaks' not in env('status', 'test')
+        assert 'mpileaks' not in find()
+        assert 'mpileaks@' not in find()
+        assert 'mpileaks@' not in find('--show-concretized')
 
     with ev.read('test'):
         add('mpileaks')
-    assert 'mpileaks' in env('status', 'test')
+        assert 'mpileaks' in find()
+        assert 'mpileaks@' not in find()
+        assert 'mpileaks@' not in find('--show-concretized')
+
+    with ev.read('test'):
+        concretize()
+        assert 'mpileaks' in find()
+        assert 'mpileaks@' not in find()
+        assert 'mpileaks@' in find('--show-concretized')
+
+    with ev.read('test'):
+        remove('mpileaks')
+        assert 'mpileaks' not in find()
+        # removed but still in last concretized specs
+        assert 'mpileaks@' in find('--show-concretized')
+
+    with ev.read('test'):
+        concretize()
+        assert 'mpileaks' not in find()
+        assert 'mpileaks@' not in find()
+        # now the lockfile is regenerated and it's gone.
+        assert 'mpileaks@' not in find('--show-concretized')
 
 
-def test_environment_status():
+def test_environment_status(capfd, tmpdir):
-    e = ev.create('test')
+    with capfd.disabled():
-    e.add('mpileaks')
+        with tmpdir.as_cwd():
-    e.concretize()
+            assert 'No active environment' in env('status')
-    e.add('python')
+
-    mock_stream = StringIO()
+            with ev.create('test'):
-    e.status(mock_stream)
+                assert 'In environment test' in env('status')
-    list_content = mock_stream.getvalue()
+
-    assert 'mpileaks' in list_content
+            with ev.Environment('local_dir'):
-    assert 'python' in list_content
+                assert os.path.join(os.getcwd(), 'local_dir') in env('status')
-    mpileaks_spec = e.specs_by_hash[e.concretized_order[0]]
+
-    assert mpileaks_spec.format() in list_content
+                e = ev.Environment('myproject')
+                e.write()
+                with tmpdir.join('myproject').as_cwd():
+                    with e:
+                        assert 'in current directory' in env('status')
 
 
 def test_to_lockfile_dict():
@@ -309,8 +340,8 @@ def test_init_with_file_and_remove(tmpdir):
     out = env('list')
     assert 'test' in out
 
-    out = env('status', 'test')
+    with ev.read('test'):
-    assert 'mpileaks' in out
+        assert 'mpileaks' in find()
 
     env('remove', '-y', 'test')