zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://github.com/LLNL/spack into features/install_with_phases_rebase

Browse Source 
Conflicts:
	lib/spack/spack/build_environment.py
	lib/spack/spack/cmd/install.py
	lib/spack/spack/cmd/setup.py
	lib/spack/spack/package.py
	var/spack/repos/builtin/packages/gmp/package.py
	var/spack/repos/builtin/packages/hdf5/package.py
This commit is contained in:
alalazo
2016-10-11 16:28:08 +02:00
parent 7a26c60dbd 4d939802ae
commit ab995df777
186 changed files with 5440 additions and 1639 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
lib/spack/llnl/util/filesystem.py
View File

@@ -39,15 +39,34 @@
import llnl.util.tty as tty
from llnl.util.lang import dedupe
__all__ = ['set_install_permissions', 'install', 'install_tree',
'traverse_tree',
'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp',
'force_remove', 'join_path', 'ancestor', 'can_access',
'filter_file',
'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink',
'set_executable', 'copy_mode', 'unset_executable_mode',
'remove_dead_links', 'remove_linked_tree',
'fix_darwin_install_name', 'find_libraries', 'LibraryList']
__all__ = [
'FileFilter',
'LibraryList',
'ancestor',
'can_access',
'change_sed_delimiter',
'copy_mode',
'expand_user',
'filter_file',
'find_libraries',
'fix_darwin_install_name',
'force_remove',
'force_symlink',
'install',
'install_tree',
'is_exe',
'join_path',
'mkdirp',
'remove_dead_links',
'remove_if_dead_link',
'remove_linked_tree',
'set_executable',
'set_install_permissions',
'touch',
'touchp',
'traverse_tree',
'unset_executable_mode',
'working_dir']
def filter_file(regex, repl, *filenames, **kwargs):
@@ -388,10 +407,20 @@ def remove_dead_links(root):
"""
for file in os.listdir(root):
path = join_path(root, file)
if os.path.islink(path):
real_path = os.path.realpath(path)
if not os.path.exists(real_path):
os.unlink(path)
remove_if_dead_link(path)
def remove_if_dead_link(path):
"""
Removes the argument if it is a dead link, does nothing otherwise
Args:
path: the potential dead link
"""
if os.path.islink(path):
real_path = os.path.realpath(path)
if not os.path.exists(real_path):
os.unlink(path)
def remove_linked_tree(path):

151
lib/spack/llnl/util/lock.py
View File

@@ -28,9 +28,13 @@
import time
import socket
import llnl.util.tty as tty
__all__ = ['Lock', 'LockTransaction', 'WriteTransaction', 'ReadTransaction',
'LockError']
# Default timeout in seconds, after which locks will raise exceptions.
_default_timeout = 60
@@ -41,51 +45,86 @@
class Lock(object):
"""This is an implementation of a filesystem lock using Python's lockf.
In Python, `lockf` actually calls `fcntl`, so this should work with any
filesystem implementation that supports locking through the fcntl calls.
This includes distributed filesystems like Lustre (when flock is enabled)
and recent NFS versions.
In Python, `lockf` actually calls `fcntl`, so this should work with
any filesystem implementation that supports locking through the fcntl
calls. This includes distributed filesystems like Lustre (when flock
is enabled) and recent NFS versions.
"""
def __init__(self, file_path):
self._file_path = file_path
self._fd = None
def __init__(self, path, start=0, length=0):
"""Construct a new lock on the file at ``path``.
By default, the lock applies to the whole file. Optionally,
caller can specify a byte range beginning ``start`` bytes from
the start of the file and extending ``length`` bytes from there.
This exposes a subset of fcntl locking functionality. It does
not currently expose the ``whence`` parameter -- ``whence`` is
always os.SEEK_SET and ``start`` is always evaluated from the
beginning of the file.
"""
self.path = path
self._file = None
self._reads = 0
self._writes = 0
def _lock(self, op, timeout):
# byte range parameters
self._start = start
self._length = length
# PID and host of lock holder
self.pid = self.old_pid = None
self.host = self.old_host = None
def _lock(self, op, timeout=_default_timeout):
"""This takes a lock using POSIX locks (``fnctl.lockf``).
The lock is implemented as a spin lock using a nonblocking
call to lockf().
The lock is implemented as a spin lock using a nonblocking call
to lockf().
On acquiring an exclusive lock, the lock writes this process's
pid and host to the lock file, in case the holding process
needs to be killed later.
pid and host to the lock file, in case the holding process needs
to be killed later.
If the lock times out, it raises a ``LockError``.
"""
start_time = time.time()
while (time.time() - start_time) < timeout:
try:
# If this is already open read-only and we want to
# upgrade to an exclusive write lock, close first.
if self._fd is not None:
flags = fcntl.fcntl(self._fd, fcntl.F_GETFL)
if op == fcntl.LOCK_EX and flags | os.O_RDONLY:
os.close(self._fd)
self._fd = None
if self._fd is None:
mode = os.O_RDWR if op == fcntl.LOCK_EX else os.O_RDONLY
self._fd = os.open(self._file_path, mode)
fcntl.lockf(self._fd, op | fcntl.LOCK_NB)
# If we could write the file, we'd have opened it 'r+'.
# Raise an error when we attempt to upgrade to a write lock.
if op == fcntl.LOCK_EX:
os.write(
self._fd,
"pid=%s,host=%s" % (os.getpid(), socket.getfqdn()))
if self._file and self._file.mode == 'r':
raise LockError(
"Can't take exclusive lock on read-only file: %s"
% self.path)
# Create file and parent directories if they don't exist.
if self._file is None:
self._ensure_parent_directory()
# Prefer to open 'r+' to allow upgrading to write
# lock later if possible. Open read-only if we can't
# write the lock file at all.
os_mode, fd_mode = (os.O_RDWR | os.O_CREAT), 'r+'
if os.path.exists(self.path) and not os.access(
self.path, os.W_OK):
os_mode, fd_mode = os.O_RDONLY, 'r'
fd = os.open(self.path, os_mode)
self._file = os.fdopen(fd, fd_mode)
# Try to get the lock (will raise if not available.)
fcntl.lockf(self._file, op | fcntl.LOCK_NB,
self._length, self._start, os.SEEK_SET)
# All locks read the owner PID and host
self._read_lock_data()
# Exclusive locks write their PID/host
if op == fcntl.LOCK_EX:
self._write_lock_data()
return
except IOError as error:
@@ -97,6 +136,40 @@ def _lock(self, op, timeout):
raise LockError("Timed out waiting for lock.")
def _ensure_parent_directory(self):
parent = os.path.dirname(self.path)
try:
os.makedirs(parent)
return True
except OSError as e:
# makedirs can fail when diretory already exists.
if not (e.errno == errno.EEXIST and os.path.isdir(parent) or
e.errno == errno.EISDIR):
raise
def _read_lock_data(self):
"""Read PID and host data out of the file if it is there."""
line = self._file.read()
if line:
pid, host = line.strip().split(',')
_, _, self.pid = pid.rpartition('=')
_, _, self.host = host.rpartition('=')
def _write_lock_data(self):
"""Write PID and host data to the file, recording old values."""
self.old_pid = self.pid
self.old_host = self.host
self.pid = os.getpid()
self.host = socket.getfqdn()
# write pid, host to disk to sync over FS
self._file.seek(0)
self._file.write("pid=%s,host=%s" % (self.pid, self.host))
self._file.truncate()
self._file.flush()
os.fsync(self._file.fileno())
def _unlock(self):
"""Releases a lock using POSIX locks (``fcntl.lockf``)
@@ -104,9 +177,10 @@ def _unlock(self):
be masquerading as write locks, but this removes either.
"""
fcntl.lockf(self._fd, fcntl.LOCK_UN)
os.close(self._fd)
self._fd = None
fcntl.lockf(self._file, fcntl.LOCK_UN,
self._length, self._start, os.SEEK_SET)
self._file.close()
self._file = None
def acquire_read(self, timeout=_default_timeout):
"""Acquires a recursive, shared lock for reading.
@@ -120,7 +194,9 @@ def acquire_read(self, timeout=_default_timeout):
"""
if self._reads == 0 and self._writes == 0:
self._lock(fcntl.LOCK_SH, timeout) # can raise LockError.
tty.debug('READ LOCK: {0.path}[{0._start}:{0._length}] [Acquiring]'
.format(self))
self._lock(fcntl.LOCK_SH, timeout=timeout) # can raise LockError.
self._reads += 1
return True
else:
@@ -139,7 +215,10 @@ def acquire_write(self, timeout=_default_timeout):
"""
if self._writes == 0:
self._lock(fcntl.LOCK_EX, timeout) # can raise LockError.
tty.debug(
'WRITE LOCK: {0.path}[{0._start}:{0._length}] [Acquiring]'
.format(self))
self._lock(fcntl.LOCK_EX, timeout=timeout) # can raise LockError.
self._writes += 1
return True
else:
@@ -159,6 +238,8 @@ def release_read(self):
assert self._reads > 0
if self._reads == 1 and self._writes == 0:
tty.debug('READ LOCK: {0.path}[{0._start}:{0._length}] [Released]'
.format(self))
self._unlock() # can raise LockError.
self._reads -= 1
return True
@@ -179,6 +260,8 @@ def release_write(self):
assert self._writes > 0
if self._writes == 1 and self._reads == 0:
tty.debug('WRITE LOCK: {0.path}[{0._start}:{0._length}] [Released]'
.format(self))
self._unlock() # can raise LockError.
self._writes -= 1
return True