Merge branch 'features/better-find' into develop

This commit is contained in:
Todd Gamblin 2014-12-02 22:55:11 -08:00
commit 652b761894
7 changed files with 207 additions and 137 deletions

View File

@ -68,6 +68,12 @@ def index_by(objects, *funcs):
index1 = index_by(list_of_specs, 'arch', 'compiler') index1 = index_by(list_of_specs, 'arch', 'compiler')
index2 = index_by(list_of_specs, 'compiler') index2 = index_by(list_of_specs, 'compiler')
You can also index by tuples by passing tuples:
index1 = index_by(list_of_specs, ('arch', 'compiler'))
Keys in the resulting dict will look like ('gcc', 'bgqos_0').
""" """
if not funcs: if not funcs:
return objects return objects
@ -75,6 +81,8 @@ def index_by(objects, *funcs):
f = funcs[0] f = funcs[0]
if isinstance(f, basestring): if isinstance(f, basestring):
f = lambda x: getattr(x, funcs[0]) f = lambda x: getattr(x, funcs[0])
elif isinstance(f, tuple):
f = lambda x: tuple(getattr(x, p) for p in funcs[0])
result = {} result = {}
for o in objects: for o in objects:

View File

@ -25,6 +25,9 @@
import sys import sys
import os import os
import textwrap import textwrap
import fcntl
import termios
import struct
from StringIO import StringIO from StringIO import StringIO
from llnl.util.tty.color import * from llnl.util.tty.color import *
@ -142,20 +145,18 @@ def get_yes_or_no(prompt, **kwargs):
def hline(label=None, **kwargs): def hline(label=None, **kwargs):
"""Draw an optionally colored or labeled horizontal line. """Draw a labeled horizontal line.
Options: Options:
char Char to draw the line with. Default '-' char Char to draw the line with. Default '-'
color Color of the label. Default is no color.
max_width Maximum width of the line. Default is 64 chars. max_width Maximum width of the line. Default is 64 chars.
See tty.color for possible color formats.
""" """
char = kwargs.get('char', '-') char = kwargs.pop('char', '-')
color = kwargs.get('color', '') max_width = kwargs.pop('max_width', 64)
max_width = kwargs.get('max_width', 64) if kwargs:
raise TypeError("'%s' is an invalid keyword argument for this function."
% next(kwargs.iterkeys()))
cols, rows = terminal_size() rows, cols = terminal_size()
if not cols: if not cols:
cols = max_width cols = max_width
else: else:
@ -163,37 +164,34 @@ def hline(label=None, **kwargs):
cols = min(max_width, cols) cols = min(max_width, cols)
label = str(label) label = str(label)
prefix = char * 2 + " " + label + " " prefix = char * 2 + " "
suffix = (cols - len(prefix)) * char suffix = " " + (cols - len(prefix) - clen(label)) * char
out = StringIO() out = StringIO()
if color:
prefix = char * 2 + " " + color + cescape(label) + "@. "
cwrite(prefix, stream=out, color=True)
else:
out.write(prefix) out.write(prefix)
out.write(label)
out.write(suffix) out.write(suffix)
print out.getvalue() print out.getvalue()
def terminal_size(): def terminal_size():
"""Gets the dimensions of the console: cols, rows.""" """Gets the dimensions of the console: (rows, cols)."""
def ioctl_GWINSZ(fd): def ioctl_GWINSZ(fd):
try: try:
cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) rc = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
except: except:
return return
return cr return rc
cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) rc = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
if not cr: if not rc:
try: try:
fd = os.open(os.ctermid(), os.O_RDONLY) fd = os.open(os.ctermid(), os.O_RDONLY)
cr = ioctl_GWINSZ(fd) rc = ioctl_GWINSZ(fd)
os.close(fd) os.close(fd)
except: except:
pass pass
if not cr: if not rc:
cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) rc = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80))
return int(cr[1]), int(cr[0]) return int(rc[0]), int(rc[1])

View File

@ -22,16 +22,9 @@
# along with this program; if not, write to the Free Software Foundation, # along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
############################################################################## ##############################################################################
# colify """
# By Todd Gamblin, tgamblin@llnl.gov Routines for printing columnar output. See colify() for more information.
# """
# Takes a list of items as input and finds a good columnization of them,
# similar to how gnu ls does. You can pipe output to this script and
# get a tight display for it. This supports both uniform-width and
# variable-width (tighter) columns.
#
# Run colify -h for more information.
#
import os import os
import sys import sys
import fcntl import fcntl
@ -40,6 +33,7 @@
from StringIO import StringIO from StringIO import StringIO
from llnl.util.tty import terminal_size from llnl.util.tty import terminal_size
from llnl.util.tty.color import clen
class ColumnConfig: class ColumnConfig:
@ -47,32 +41,52 @@ def __init__(self, cols):
self.cols = cols self.cols = cols
self.line_length = 0 self.line_length = 0
self.valid = True self.valid = True
self.widths = [0] * cols self.widths = [0] * cols # does not include ansi colors
self.cwidths = [0] * cols # includes ansi colors
def __repr__(self): def __repr__(self):
attrs = [(a,getattr(self, a)) for a in dir(self) if not a.startswith("__")] attrs = [(a,getattr(self, a)) for a in dir(self) if not a.startswith("__")]
return "<Config: %s>" % ", ".join("%s: %r" % a for a in attrs) return "<Config: %s>" % ", ".join("%s: %r" % a for a in attrs)
def config_variable_cols(elts, console_cols, padding): def config_variable_cols(elts, console_width, padding, cols=0):
"""Variable-width column fitting algorithm.
This function determines the most columns that can fit in the
screen width. Unlike uniform fitting, where all columns take
the width of the longest element in the list, each column takes
the width of its own longest element. This packs elements more
efficiently on screen.
If cols is nonzero, force
"""
if cols < 0:
raise ValueError("cols must be non-negative.")
# Get a bound on the most columns we could possibly have. # Get a bound on the most columns we could possibly have.
lengths = [len(elt) for elt in elts] # 'clen' ignores length of ansi color sequences.
max_cols = max(1, console_cols / (min(lengths) + padding)) lengths = [clen(e) for e in elts]
clengths = [len(e) for e in elts]
max_cols = max(1, console_width / (min(lengths) + padding))
max_cols = min(len(elts), max_cols) max_cols = min(len(elts), max_cols)
configs = [ColumnConfig(c) for c in xrange(1, max_cols+1)] # Range of column counts to try. If forced, use the supplied value.
for elt, length in enumerate(lengths): col_range = [cols] if cols else xrange(1, max_cols+1)
for i, conf in enumerate(configs):
if conf.valid:
col = elt / ((len(elts) + i) / (i + 1))
padded = length
if col < i:
padded += padding
if conf.widths[col] < padded: # Determine the most columns possible for the console width.
conf.line_length += padded - conf.widths[col] configs = [ColumnConfig(c) for c in col_range]
conf.widths[col] = padded for i, length in enumerate(lengths):
conf.valid = (conf.line_length < console_cols) for conf in configs:
if conf.valid:
col = i / ((len(elts) + conf.cols - 1) / conf.cols)
p = padding if col < (conf.cols - 1) else 0
if conf.widths[col] < (length + p):
conf.line_length += length + p - conf.widths[col]
conf.widths[col] = length + p
conf.cwidths[col] = clengths[i] + p
conf.valid = (conf.line_length < console_width)
try: try:
config = next(conf for conf in reversed(configs) if conf.valid) config = next(conf for conf in reversed(configs) if conf.valid)
@ -85,57 +99,98 @@ def config_variable_cols(elts, console_cols, padding):
return config return config
def config_uniform_cols(elts, console_cols, padding): def config_uniform_cols(elts, console_width, padding, cols=0):
max_len = max(len(elt) for elt in elts) + padding """Uniform-width column fitting algorithm.
cols = max(1, console_cols / max_len)
Determines the longest element in the list, and determines how
many columns of that width will fit on screen. Returns a
corresponding column config.
"""
if cols < 0:
raise ValueError("cols must be non-negative.")
# 'clen' ignores length of ansi color sequences.
max_len = max(clen(e) for e in elts) + padding
max_clen = max(len(e) for e in elts) + padding
if cols == 0:
cols = max(1, console_width / max_len)
cols = min(len(elts), cols) cols = min(len(elts), cols)
config = ColumnConfig(cols) config = ColumnConfig(cols)
config.widths = [max_len] * cols config.widths = [max_len] * cols
config.cwidths = [max_clen] * cols
return config return config
def isatty(ostream):
force = os.environ.get('COLIFY_TTY', 'false').lower() != 'false'
return force or ostream.isatty()
def colify(elts, **options): def colify(elts, **options):
"""Takes a list of elements as input and finds a good columnization
of them, similar to how gnu ls does. This supports both
uniform-width and variable-width (tighter) columns.
If elts is not a list of strings, each element is first conveted
using str().
Keyword arguments:
output=<stream> A file object to write to. Default is sys.stdout.
indent=<int> Optionally indent all columns by some number of spaces.
padding=<int> Spaces between columns. Default is 2.
width=<int> Width of the output. Default is 80 if tty is not detected.
cols=<int> Force number of columns. Default is to size to terminal,
or single-column if no tty
tty=<bool> Whether to attempt to write to a tty. Default is to
autodetect a tty. Set to False to force single-column output.
method=<string> Method to use to fit columns. Options are variable or uniform.
Variable-width columns are tighter, uniform columns are all the
same width and fit less data on the screen.
len=<func> Function to use for calculating string length.
Useful for ignoring ansi color. Default is 'len'.
"""
# Get keyword arguments or set defaults # Get keyword arguments or set defaults
output = options.get("output", sys.stdout) cols = options.pop("cols", 0)
indent = options.get("indent", 0) output = options.pop("output", sys.stdout)
padding = options.get("padding", 2) indent = options.pop("indent", 0)
tty = options.get('tty', None) padding = options.pop("padding", 2)
tty = options.pop('tty', None)
method = options.pop("method", "variable")
console_cols = options.pop("width", None)
if options:
raise TypeError("'%s' is an invalid keyword argument for this function."
% next(options.iterkeys()))
# elts needs to be an array of strings so we can count the elements # elts needs to be an array of strings so we can count the elements
elts = [str(elt) for elt in elts] elts = [str(elt) for elt in elts]
if not elts: if not elts:
return (0, ()) return (0, ())
# Use only one column if not a tty.
if not tty: if not tty:
if tty is False or not isatty(output): if tty is False or not output.isatty():
for elt in elts: cols = 1
output.write("%s\n" % elt)
maxlen = max(len(str(s)) for s in elts) # Specify the number of character columns to use.
return (1, (maxlen,))
console_cols = options.get("cols", None)
if not console_cols: if not console_cols:
console_cols, console_rows = terminal_size() console_rows, console_cols = terminal_size()
elif type(console_cols) != int: elif type(console_cols) != int:
raise ValueError("Number of columns must be an int") raise ValueError("Number of columns must be an int")
console_cols = max(1, console_cols - indent) console_cols = max(1, console_cols - indent)
method = options.get("method", "variable") # Choose a method. Variable-width colums vs uniform-width.
if method == "variable": if method == "variable":
config = config_variable_cols(elts, console_cols, padding) config = config_variable_cols(elts, console_cols, padding, cols)
elif method == "uniform": elif method == "uniform":
config = config_uniform_cols(elts, console_cols, padding) config = config_uniform_cols(elts, console_cols, padding, cols)
else: else:
raise ValueError("method must be one of: " + allowed_methods) raise ValueError("method must be one of: " + allowed_methods)
cols = config.cols cols = config.cols
formats = ["%%-%ds" % width for width in config.widths[:-1]] formats = ["%%-%ds" % width for width in config.cwidths[:-1]]
formats.append("%s") # last column has no trailing space formats.append("%s") # last column has no trailing space
rows = (len(elts) + cols - 1) / cols rows = (len(elts) + cols - 1) / cols
@ -155,6 +210,25 @@ def colify(elts, **options):
return (config.cols, tuple(config.widths)) return (config.cols, tuple(config.widths))
def colify_table(table, **options):
if table is None:
raise TypeError("Can't call colify_table on NoneType")
elif not table or not table[0]:
raise ValueError("Table is empty in colify_table!")
columns = len(table[0])
def transpose():
for i in xrange(columns):
for row in table:
yield row[i]
if 'cols' in options:
raise ValueError("Cannot override columsn in colify_table.")
options['cols'] = columns
colify(transpose(), **options)
def colified(elts, **options): def colified(elts, **options):
"""Invokes the colify() function but returns the result as a string """Invokes the colify() function but returns the result as a string
instead of writing it to an output string.""" instead of writing it to an output string."""
@ -162,29 +236,3 @@ def colified(elts, **options):
options['output'] = sio options['output'] = sio
colify(elts, **options) colify(elts, **options)
return sio.getvalue() return sio.getvalue()
if __name__ == "__main__":
import optparse
cols, rows = terminal_size()
parser = optparse.OptionParser()
parser.add_option("-u", "--uniform", action="store_true", default=False,
help="Use uniformly sized columns instead of variable-size.")
parser.add_option("-p", "--padding", metavar="PADDING", action="store",
type=int, default=2, help="Spaces to add between columns. Default is 2.")
parser.add_option("-i", "--indent", metavar="SPACES", action="store",
type=int, default=0, help="Indent the output by SPACES. Default is 0.")
parser.add_option("-w", "--width", metavar="COLS", action="store",
type=int, default=cols, help="Indent the output by SPACES. Default is 0.")
options, args = parser.parse_args()
method = "variable"
if options.uniform:
method = "uniform"
if sys.stdin.isatty():
parser.print_help()
sys.exit(1)
else:
colify([line.strip() for line in sys.stdin], method=method, **options.__dict__)

View File

@ -149,6 +149,11 @@ def colorize(string, **kwargs):
return re.sub(color_re, match_to_ansi(color), string) return re.sub(color_re, match_to_ansi(color), string)
def clen(string):
"""Return the length of a string, excluding ansi color sequences."""
return len(re.sub(r'\033[^m]*m', '', string))
def cwrite(string, stream=sys.stdout, color=None): def cwrite(string, stream=sys.stdout, color=None):
"""Replace all color expressions in string with ANSI control """Replace all color expressions in string with ANSI control
codes and write the result to the stream. If color is codes and write the result to the stream. If color is

View File

@ -25,6 +25,7 @@
from external import argparse from external import argparse
import llnl.util.tty as tty import llnl.util.tty as tty
from llnl.util.tty.color import colorize
from llnl.util.tty.colify import colify from llnl.util.tty.colify import colify
from llnl.util.lang import index_by from llnl.util.lang import index_by
@ -96,9 +97,12 @@ def compiler_info(args):
def compiler_list(args): def compiler_list(args):
tty.msg("Available compilers") tty.msg("Available compilers")
index = index_by(spack.compilers.all_compilers(), 'name') index = index_by(spack.compilers.all_compilers(), 'name')
for name, compilers in index.items(): for i, (name, compilers) in enumerate(index.items()):
tty.hline(name, char='-', color=spack.spec.compiler_color) if i >= 1: print
colify(reversed(sorted(compilers)), indent=4)
cname = "%s{%s}" % (spack.spec.compiler_color, name)
tty.hline(colorize(cname), char='-')
colify(reversed(sorted(compilers)))
def compiler(parser, args): def compiler(parser, args):

View File

@ -24,13 +24,14 @@
############################################################################## ##############################################################################
import sys import sys
import collections import collections
import itertools
from external import argparse from external import argparse
from StringIO import StringIO from StringIO import StringIO
import llnl.util.tty as tty import llnl.util.tty as tty
from llnl.util.tty.colify import colify from llnl.util.tty.colify import *
from llnl.util.tty.color import * from llnl.util.tty.color import *
from llnl.util.lang import partition_list, index_by from llnl.util.lang import *
import spack import spack
import spack.spec import spack.spec
@ -39,12 +40,15 @@
def setup_parser(subparser): def setup_parser(subparser):
format_group = subparser.add_mutually_exclusive_group() format_group = subparser.add_mutually_exclusive_group()
format_group.add_argument(
'-l', '--long', action='store_true', dest='long',
help='Show dependency hashes as well as versions.')
format_group.add_argument( format_group.add_argument(
'-p', '--paths', action='store_true', dest='paths', '-p', '--paths', action='store_true', dest='paths',
help='Show paths to package install directories') help='Show paths to package install directories')
format_group.add_argument( format_group.add_argument(
'-l', '--long', action='store_true', dest='full_specs', '-d', '--deps', action='store_true', dest='full_deps',
help='Show full-length specs of installed packages') help='Show full dependency DAG of installed packages')
subparser.add_argument( subparser.add_argument(
'query_specs', nargs=argparse.REMAINDER, 'query_specs', nargs=argparse.REMAINDER,
@ -65,23 +69,29 @@ def find(parser, args):
if not query_specs: if not query_specs:
return return
specs = [s for s in spack.db.installed_package_specs() # Get all the specs the user asked for
if not query_specs or any(s.satisfies(q) for q in query_specs)] if not query_specs:
specs = set(spack.db.installed_package_specs())
else:
results = [set(spack.db.get_installed(qs)) for qs in query_specs]
specs = set.union(*results)
# Make a dict with specs keyed by architecture and compiler. # Make a dict with specs keyed by architecture and compiler.
index = index_by(specs, 'architecture', 'compiler') index = index_by(specs, ('architecture', 'compiler'))
# Traverse the index and print out each package # Traverse the index and print out each package
for architecture in index: for i, (architecture, compiler) in enumerate(sorted(index)):
tty.hline(architecture, char='=', color=spack.spec.architecture_color) if i > 0: print
for compiler in index[architecture]:
tty.hline(compiler, char='-', color=spack.spec.compiler_color)
specs = index[architecture][compiler] header = "%s{%s} / %s{%s}" % (
spack.spec.architecture_color, architecture,
spack.spec.compiler_color, compiler)
tty.hline(colorize(header), char='-')
specs = index[(architecture,compiler)]
specs.sort() specs.sort()
abbreviated = [s.format('$_$@$+$#', color=True) for s in specs] abbreviated = [s.format('$_$@$+', color=True) for s in specs]
if args.paths: if args.paths:
# Print one spec per line along with prefix path # Print one spec per line along with prefix path
width = max(len(s) for s in abbreviated) width = max(len(s) for s in abbreviated)
@ -91,13 +101,11 @@ def find(parser, args):
for abbrv, spec in zip(abbreviated, specs): for abbrv, spec in zip(abbreviated, specs):
print format % (abbrv, spec.prefix) print format % (abbrv, spec.prefix)
elif args.full_specs: elif args.full_deps:
for spec in specs: for spec in specs:
print spec.tree(indent=4, format='$_$@$+', color=True), print spec.tree(indent=4, format='$_$@$+', color=True),
else: else:
max_len = max([len(s.name) for s in specs]) fmt = '$-_$@$+'
max_len += 4 if args.long:
fmt += '$#'
for spec in specs: colify(s.format(fmt, color=True) for s in specs)
format = '$-' + str(max_len) + '_$@$+$#'
print " " + spec.format(format, color=True)

View File

@ -61,5 +61,4 @@ def match(p, f):
indent=0 indent=0
if sys.stdout.isatty(): if sys.stdout.isatty():
tty.msg("%d packages." % len(sorted_packages)) tty.msg("%d packages." % len(sorted_packages))
indent=2
colify(sorted_packages, indent=indent) colify(sorted_packages, indent=indent)