zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://github.com/LLNL/spack into features/install_with_phases

Browse Source 
Conflicts:
	lib/spack/llnl/util/tty/log.py
	lib/spack/spack/__init__.py
	lib/spack/spack/cmd/install.py
	lib/spack/spack/cmd/setup.py
	lib/spack/spack/package.py
	var/spack/repos/builtin/packages/blitz/package.py
	var/spack/repos/builtin/packages/gmp/package.py
	var/spack/repos/builtin/packages/qhull/package.py
	var/spack/repos/builtin/packages/szip/package.py
This commit is contained in:
alalazo
2016-08-11 08:55:20 +02:00
parent b92deda41b f59653ac2c
commit b4b9ebe7d7
770 changed files with 14235 additions and 4355 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
lib/spack/llnl/util/filesystem.py
View File

@@ -29,8 +29,9 @@
import stat
import errno
import getpass
from contextlib import contextmanager, closing
from contextlib import contextmanager
import subprocess
import fileinput
import llnl.util.tty as tty
@@ -85,13 +86,14 @@ def groupid_to_group(x):
if ignore_absent and not os.path.exists(filename):
continue
shutil.copy(filename, backup_filename)
# Create backup file. Don't overwrite an existing backup
# file in case this file is being filtered multiple times.
if not os.path.exists(backup_filename):
shutil.copy(filename, backup_filename)
try:
with closing(open(backup_filename)) as infile:
with closing(open(filename, 'w')) as outfile:
for line in infile:
foo = re.sub(regex, repl, line)
outfile.write(foo)
for line in fileinput.input(filename, inplace=True):
print(re.sub(regex, repl, line.rstrip('\n')))
except:
# clean up the original file on failure.
shutil.move(backup_filename, filename)
@@ -104,6 +106,7 @@ def groupid_to_group(x):
class FileFilter(object):
"""Convenience class for calling filter_file a lot."""
def __init__(self, *filenames):
self.filenames = filenames
@@ -189,7 +192,7 @@ def install(src, dest):
def install_tree(src, dest, **kwargs):
"""Manually install a file to a particular location."""
"""Manually install a directory tree to a particular location."""
tty.debug("Installing %s to %s" % (src, dest))
shutil.copytree(src, dest, **kwargs)
@@ -353,7 +356,8 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
# When follow_nonexisting isn't set, don't descend into dirs
# in source that do not exist in dest
if follow_nonexisting or os.path.exists(dest_child):
tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs) # NOQA: ignore=E501
tuples = traverse_tree(
source_root, dest_root, rel_child, **kwargs)
for t in tuples:
yield t
@@ -420,14 +424,20 @@ def fix_darwin_install_name(path):
libs = glob.glob(join_path(path, "*.dylib"))
for lib in libs:
# fix install name first:
subprocess.Popen(["install_name_tool", "-id", lib, lib], stdout=subprocess.PIPE).communicate()[0] # NOQA: ignore=E501
long_deps = subprocess.Popen(["otool", "-L", lib], stdout=subprocess.PIPE).communicate()[0].split('\n') # NOQA: ignore=E501
subprocess.Popen(
["install_name_tool", "-id", lib, lib],
stdout=subprocess.PIPE).communicate()[0]
long_deps = subprocess.Popen(
["otool", "-L", lib],
stdout=subprocess.PIPE).communicate()[0].split('\n')
deps = [dep.partition(' ')[0][1::] for dep in long_deps[2:-1]]
# fix all dependencies:
for dep in deps:
for loc in libs:
if dep == os.path.basename(loc):
subprocess.Popen(["install_name_tool", "-change", dep, loc, lib], stdout=subprocess.PIPE).communicate()[0] # NOQA: ignore=E501
subprocess.Popen(
["install_name_tool", "-change", dep, loc, lib],
stdout=subprocess.PIPE).communicate()[0]
break

77
lib/spack/llnl/util/lang.py
View File

@@ -24,7 +24,6 @@
##############################################################################
import os
import re
import sys
import functools
import collections
import inspect
@@ -39,14 +38,15 @@ def index_by(objects, *funcs):
Values are used as keys. For example, suppose you have four
objects with attributes that look like this:
a = Spec(name="boost", compiler="gcc", arch="bgqos_0")
b = Spec(name="mrnet", compiler="intel", arch="chaos_5_x86_64_ib")
c = Spec(name="libelf", compiler="xlc", arch="bgqos_0")
d = Spec(name="libdwarf", compiler="intel", arch="chaos_5_x86_64_ib")
a = Spec(name="boost", compiler="gcc", arch="bgqos_0")
b = Spec(name="mrnet", compiler="intel", arch="chaos_5_x86_64_ib")
c = Spec(name="libelf", compiler="xlc", arch="bgqos_0")
d = Spec(name="libdwarf", compiler="intel", arch="chaos_5_x86_64_ib")
list_of_specs = [a,b,c,d]
index1 = index_by(list_of_specs, lambda s: s.arch, lambda s: s.compiler)
index2 = index_by(list_of_specs, lambda s: s.compiler)
list_of_specs = [a,b,c,d]
index1 = index_by(list_of_specs, lambda s: s.arch,
lambda s: s.compiler)
index2 = index_by(list_of_specs, lambda s: s.compiler)
``index1'' now has two levels of dicts, with lists at the
leaves, like this:
@@ -137,7 +137,7 @@ def get_calling_module_name():
finally:
del stack
if not '__module__' in caller_locals:
if '__module__' not in caller_locals:
raise RuntimeError("Must invoke get_calling_module_name() "
"from inside a class definition!")
@@ -173,11 +173,11 @@ def has_method(cls, name):
class memoized(object):
"""Decorator that caches the results of a function, storing them
in an attribute of that function."""
def __init__(self, func):
self.func = func
self.cache = {}
def __call__(self, *args):
if not isinstance(args, collections.Hashable):
# Not hashable, so just call the function.
@@ -187,12 +187,10 @@ def __call__(self, *args):
self.cache[args] = self.func(*args)
return self.cache[args]
def __get__(self, obj, objtype):
"""Support instance methods."""
return functools.partial(self.__call__, obj)
def clear(self):
"""Expunge cache so that self.func will be called again."""
self.cache.clear()
@@ -237,13 +235,21 @@ def setter(name, value):
if not has_method(cls, '_cmp_key'):
raise TypeError("'%s' doesn't define _cmp_key()." % cls.__name__)
setter('__eq__', lambda s,o: (s is o) or (o is not None and s._cmp_key() == o._cmp_key()))
setter('__lt__', lambda s,o: o is not None and s._cmp_key() < o._cmp_key())
setter('__le__', lambda s,o: o is not None and s._cmp_key() <= o._cmp_key())
setter('__eq__',
lambda s, o:
(s is o) or (o is not None and s._cmp_key() == o._cmp_key()))
setter('__lt__',
lambda s, o: o is not None and s._cmp_key() < o._cmp_key())
setter('__le__',
lambda s, o: o is not None and s._cmp_key() <= o._cmp_key())
setter('__ne__', lambda s,o: (s is not o) and (o is None or s._cmp_key() != o._cmp_key()))
setter('__gt__', lambda s,o: o is None or s._cmp_key() > o._cmp_key())
setter('__ge__', lambda s,o: o is None or s._cmp_key() >= o._cmp_key())
setter('__ne__',
lambda s, o:
(s is not o) and (o is None or s._cmp_key() != o._cmp_key()))
setter('__gt__',
lambda s, o: o is None or s._cmp_key() > o._cmp_key())
setter('__ge__',
lambda s, o: o is None or s._cmp_key() >= o._cmp_key())
setter('__hash__', lambda self: hash(self._cmp_key()))
@@ -254,10 +260,10 @@ def setter(name, value):
class HashableMap(dict):
"""This is a hashable, comparable dictionary. Hash is performed on
a tuple of the values in the dictionary."""
def _cmp_key(self):
return tuple(sorted(self.values()))
def copy(self):
"""Type-agnostic clone method. Preserves subclass type."""
# Construct a new dict of my type
@@ -336,24 +342,39 @@ def match(string):
return match
def DictWrapper(dictionary):
"""Returns a class that wraps a dictionary and enables it to be used
like an object."""
class wrapper(object):
def __getattr__(self, name): return dictionary[name]
def __setattr__(self, name, value): dictionary[name] = value
def setdefault(self, *args): return dictionary.setdefault(*args)
def get(self, *args): return dictionary.get(*args)
def keys(self): return dictionary.keys()
def values(self): return dictionary.values()
def items(self): return dictionary.items()
def __iter__(self): return iter(dictionary)
def __getattr__(self, name):
return dictionary[name]
def __setattr__(self, name, value):
dictionary[name] = value
def setdefault(self, *args):
return dictionary.setdefault(*args)
def get(self, *args):
return dictionary.get(*args)
def keys(self):
return dictionary.keys()
def values(self):
return dictionary.values()
def items(self):
return dictionary.items()
def __iter__(self):
return iter(dictionary)
return wrapper()
class RequiredAttributeError(ValueError):
def __init__(self, message):
super(RequiredAttributeError, self).__init__(message)

10
lib/spack/llnl/util/link_tree.py
View File

@@ -23,12 +23,13 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""LinkTree class for setting up trees of symbolic links."""
__all__ = ['LinkTree']
import os
import shutil
from llnl.util.filesystem import *
__all__ = ['LinkTree']
empty_file_name = '.spack-empty'
@@ -43,13 +44,13 @@ class LinkTree(object):
modified.
"""
def __init__(self, source_root):
if not os.path.exists(source_root):
raise IOError("No such file or directory: '%s'", source_root)
self._root = source_root
def find_conflict(self, dest_root, **kwargs):
"""Returns the first file in dest that conflicts with src"""
kwargs['follow_nonexisting'] = False
@@ -61,9 +62,9 @@ def find_conflict(self, dest_root, **kwargs):
return dest
return None
def merge(self, dest_root, **kwargs):
"""Link all files in src into dest, creating directories if necessary."""
"""Link all files in src into dest, creating directories
if necessary."""
kwargs['order'] = 'pre'
for src, dest in traverse_tree(self._root, dest_root, **kwargs):
if os.path.isdir(src):
@@ -83,7 +84,6 @@ def merge(self, dest_root, **kwargs):
assert(not os.path.exists(dest))
os.symlink(src, dest)
def unmerge(self, dest_root, **kwargs):
"""Unlink all files in dest that exist in src.

92
lib/spack/llnl/util/lock.py
View File

@@ -28,6 +28,9 @@
import time
import socket
__all__ = ['Lock', 'LockTransaction', 'WriteTransaction', 'ReadTransaction',
'LockError']
# Default timeout in seconds, after which locks will raise exceptions.
_default_timeout = 60
@@ -36,13 +39,21 @@
class Lock(object):
def __init__(self,file_path):
"""This is an implementation of a filesystem lock using Python's lockf.
In Python, `lockf` actually calls `fcntl`, so this should work with any
filesystem implementation that supports locking through the fcntl calls.
This includes distributed filesystems like Lustre (when flock is enabled)
and recent NFS versions.
"""
def __init__(self, file_path):
self._file_path = file_path
self._fd = None
self._reads = 0
self._writes = 0
def _lock(self, op, timeout):
"""This takes a lock using POSIX locks (``fnctl.lockf``).
@@ -63,7 +74,9 @@ def _lock(self, op, timeout):
fcntl.lockf(self._fd, op | fcntl.LOCK_NB)
if op == fcntl.LOCK_EX:
os.write(self._fd, "pid=%s,host=%s" % (os.getpid(), socket.getfqdn()))
os.write(
self._fd,
"pid=%s,host=%s" % (os.getpid(), socket.getfqdn()))
return
except IOError as error:
@@ -75,7 +88,6 @@ def _lock(self, op, timeout):
raise LockError("Timed out waiting for lock.")
def _unlock(self):
"""Releases a lock using POSIX locks (``fcntl.lockf``)
@@ -83,11 +95,10 @@ def _unlock(self):
be masquerading as write locks, but this removes either.
"""
fcntl.lockf(self._fd,fcntl.LOCK_UN)
fcntl.lockf(self._fd, fcntl.LOCK_UN)
os.close(self._fd)
self._fd = None
def acquire_read(self, timeout=_default_timeout):
"""Acquires a recursive, shared lock for reading.
@@ -107,7 +118,6 @@ def acquire_read(self, timeout=_default_timeout):
self._reads += 1
return False
def acquire_write(self, timeout=_default_timeout):
"""Acquires a recursive, exclusive lock for writing.
@@ -127,7 +137,6 @@ def acquire_write(self, timeout=_default_timeout):
self._writes += 1
return False
def release_read(self):
"""Releases a read lock.
@@ -148,7 +157,6 @@ def release_read(self):
self._reads -= 1
return False
def release_write(self):
"""Releases a write lock.
@@ -170,6 +178,70 @@ def release_write(self):
return False
class LockTransaction(object):
"""Simple nested transaction context manager that uses a file lock.
This class can trigger actions when the lock is acquired for the
first time and released for the last.
If the acquire_fn returns a value, it is used as the return value for
__enter__, allowing it to be passed as the `as` argument of a `with`
statement.
If acquire_fn returns a context manager, *its* `__enter__` function will be
called in `__enter__` after acquire_fn, and its `__exit__` funciton will be
called before `release_fn` in `__exit__`, allowing you to nest a context
manager to be used along with the lock.
Timeout for lock is customizable.
"""
def __init__(self, lock, acquire_fn=None, release_fn=None,
timeout=_default_timeout):
self._lock = lock
self._timeout = timeout
self._acquire_fn = acquire_fn
self._release_fn = release_fn
self._as = None
def __enter__(self):
if self._enter() and self._acquire_fn:
self._as = self._acquire_fn()
if hasattr(self._as, '__enter__'):
return self._as.__enter__()
else:
return self._as
def __exit__(self, type, value, traceback):
suppress = False
if self._exit():
if self._as and hasattr(self._as, '__exit__'):
if self._as.__exit__(type, value, traceback):
suppress = True
if self._release_fn:
if self._release_fn(type, value, traceback):
suppress = True
return suppress
class ReadTransaction(LockTransaction):
def _enter(self):
return self._lock.acquire_read(self._timeout)
def _exit(self):
return self._lock.release_read()
class WriteTransaction(LockTransaction):
def _enter(self):
return self._lock.acquire_write(self._timeout)
def _exit(self):
return self._lock.release_write()
class LockError(Exception):
"""Raised when an attempt to acquire a lock times out."""
pass

12
lib/spack/llnl/util/tty/__init__.py
View File

@@ -36,6 +36,7 @@
_verbose = False
indent = " "
def is_verbose():
return _verbose
@@ -148,7 +149,8 @@ def get_yes_or_no(prompt, **kwargs):
elif default_value is False:
prompt += ' [y/N] '
else:
raise ValueError("default for get_yes_no() must be True, False, or None.")
raise ValueError(
"default for get_yes_no() must be True, False, or None.")
result = None
while result is None:
@@ -174,8 +176,9 @@ def hline(label=None, **kwargs):
char = kwargs.pop('char', '-')
max_width = kwargs.pop('max_width', 64)
if kwargs:
raise TypeError("'%s' is an invalid keyword argument for this function."
% next(kwargs.iterkeys()))
raise TypeError(
"'%s' is an invalid keyword argument for this function."
% next(kwargs.iterkeys()))
rows, cols = terminal_size()
if not cols:
@@ -200,7 +203,8 @@ def terminal_size():
"""Gets the dimensions of the console: (rows, cols)."""
def ioctl_GWINSZ(fd):
try:
rc = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
rc = struct.unpack('hh', fcntl.ioctl(
fd, termios.TIOCGWINSZ, '1234'))
except:
return
return rc

32
lib/spack/llnl/util/tty/colify.py
View File

@@ -27,15 +27,14 @@
"""
import os
import sys
import fcntl
import termios
import struct
from StringIO import StringIO
from llnl.util.tty import terminal_size
from llnl.util.tty.color import clen, cextra
class ColumnConfig:
def __init__(self, cols):
self.cols = cols
self.line_length = 0
@@ -43,7 +42,8 @@ def __init__(self, cols):
self.widths = [0] * cols # does not include ansi colors
def __repr__(self):
attrs = [(a,getattr(self, a)) for a in dir(self) if not a.startswith("__")]
attrs = [(a, getattr(self, a))
for a in dir(self) if not a.startswith("__")]
return "<Config: %s>" % ", ".join("%s: %r" % a for a in attrs)
@@ -68,7 +68,7 @@ def config_variable_cols(elts, console_width, padding, cols=0):
max_cols = min(len(elts), max_cols)
# Range of column counts to try. If forced, use the supplied value.
col_range = [cols] if cols else xrange(1, max_cols+1)
col_range = [cols] if cols else xrange(1, max_cols + 1)
# Determine the most columns possible for the console width.
configs = [ColumnConfig(c) for c in col_range]
@@ -106,7 +106,6 @@ def config_uniform_cols(elts, console_width, padding, cols=0):
# 'clen' ignores length of ansi color sequences.
max_len = max(clen(e) for e in elts) + padding
max_clen = max(len(e) for e in elts) + padding
if cols == 0:
cols = max(1, console_width / max_len)
cols = min(len(elts), cols)
@@ -130,17 +129,19 @@ def colify(elts, **options):
output=<stream> A file object to write to. Default is sys.stdout.
indent=<int> Optionally indent all columns by some number of spaces.
padding=<int> Spaces between columns. Default is 2.
width=<int> Width of the output. Default is 80 if tty is not detected.
width=<int> Width of the output. Default is 80 if tty not detected.
cols=<int> Force number of columns. Default is to size to terminal,
or single-column if no tty
tty=<bool> Whether to attempt to write to a tty. Default is to
autodetect a tty. Set to False to force single-column output.
autodetect a tty. Set to False to force
single-column output.
method=<string> Method to use to fit columns. Options are variable or uniform.
Variable-width columns are tighter, uniform columns are all the
same width and fit less data on the screen.
method=<string> Method to use to fit columns. Options are variable or
uniform. Variable-width columns are tighter, uniform
columns are all the same width and fit less data on
the screen.
"""
# Get keyword arguments or set defaults
cols = options.pop("cols", 0)
@@ -152,8 +153,9 @@ def colify(elts, **options):
console_cols = options.pop("width", None)
if options:
raise TypeError("'%s' is an invalid keyword argument for this function."
% next(options.iterkeys()))
raise TypeError(
"'%s' is an invalid keyword argument for this function."
% next(options.iterkeys()))
# elts needs to be an array of strings so we can count the elements
elts = [str(elt) for elt in elts]
@@ -167,7 +169,8 @@ def colify(elts, **options):
r, c = env_size.split('x')
console_rows, console_cols = int(r), int(c)
tty = True
except: pass
except:
pass
# Use only one column if not a tty.
if not tty:
@@ -228,6 +231,7 @@ def colify_table(table, **options):
raise ValueError("Table is empty in colify_table!")
columns = len(table[0])
def transpose():
for i in xrange(columns):
for row in table:

29
lib/spack/llnl/util/tty/color.py
View File

@@ -75,25 +75,27 @@
import re
import sys
class ColorParseError(Exception):
"""Raised when a color format fails to parse."""
def __init__(self, message):
super(ColorParseError, self).__init__(message)
# Text styles for ansi codes
styles = {'*' : '1', # bold
'_' : '4', # underline
None : '0' } # plain
styles = {'*': '1', # bold
'_': '4', # underline
None: '0'} # plain
# Dim and bright ansi colors
colors = {'k' : 30, 'K' : 90, # black
'r' : 31, 'R' : 91, # red
'g' : 32, 'G' : 92, # green
'y' : 33, 'Y' : 93, # yellow
'b' : 34, 'B' : 94, # blue
'm' : 35, 'M' : 95, # magenta
'c' : 36, 'C' : 96, # cyan
'w' : 37, 'W' : 97 } # white
colors = {'k': 30, 'K': 90, # black
'r': 31, 'R': 91, # red
'g': 32, 'G': 92, # green
'y': 33, 'Y': 93, # yellow
'b': 34, 'B': 94, # blue
'm': 35, 'M': 95, # magenta
'c': 36, 'C': 96, # cyan
'w': 37, 'W': 97} # white
# Regex to be used for color formatting
color_re = r'@(?:@|\.|([*_])?([a-zA-Z])?(?:{((?:[^}]|}})*)})?)'
@@ -104,6 +106,7 @@ def __init__(self, message):
class match_to_ansi(object):
def __init__(self, color=True):
self.color = color
@@ -179,12 +182,14 @@ def cprint(string, stream=sys.stdout, color=None):
"""Same as cwrite, but writes a trailing newline to the stream."""
cwrite(string + "\n", stream, color)
def cescape(string):
"""Replace all @ with @@ in the string provided."""
return str(string).replace('@', '@@')
class ColorStream(object):
def __init__(self, stream, color=None):
self._stream = stream
self._color = color
@@ -196,7 +201,7 @@ def write(self, string, **kwargs):
color = self._color
if self._color is None:
if raw:
color=True
color = True
else:
color = self._stream.isatty() or _force_color
raw_write(colorize(string, color=color))

6
lib/spack/llnl/util/tty/log.py
View File

@@ -37,6 +37,7 @@
# Use this to strip escape sequences
_escape = re.compile(r'\x1b[^m]*m|\x1b\[?1034h')
def _strip(line):
"""Strip color and control characters from a line."""
return _escape.sub('', line)
@@ -59,10 +60,10 @@ class keyboard_input(object):
When the with block completes, this will restore settings before
canonical and echo were disabled.
"""
def __init__(self, stream):
self.stream = stream
def __enter__(self):
self.old_cfg = None
@@ -87,10 +88,9 @@ def __enter__(self):
# Apply new settings for terminal
termios.tcsetattr(fd, termios.TCSADRAIN, self.new_cfg)
except Exception, e:
except Exception:
pass # Some OS's do not support termios, so ignore.
def __exit__(self, exc_type, exception, traceback):
# If termios was avaialble, restore old settings after the
# with block