zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

"spack commands --update-completion"

Browse Source
This commit is contained in:
John Parent
2022-01-24 14:35:44 -05:00
committed by Peter Scheibel
parent e4d4a5193f
commit cf1349ba35
105 changed files with 691 additions and 1065 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
lib/spack/llnl/util/filesystem.py
View File

@@ -36,17 +36,6 @@
import win32security
is_windows = _platform == 'win32'
if not is_windows:
import grp
import pwd
if sys.version_info >= (3, 3):
from collections.abc import Sequence # novm
else:
from collections import Sequence
__all__ = [
'FileFilter',
'FileList',
@@ -908,7 +897,7 @@ def ancestor(dir, n=1):
parent = os.path.abspath(dir)
for i in range(n):
parent = os.path.dirname(parent)
return parent.replace("\\", "/")
return parent
@system_path_filter

59
lib/spack/llnl/util/lock.py
View File

@@ -4,9 +4,9 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import errno
import fcntl
import os
import socket
import sys
import time
from datetime import datetime
from typing import Dict, Tuple # novm
@@ -15,6 +15,10 @@
import spack.util.string
if sys.platform != 'win32':
import fcntl
__all__ = [
'Lock',
'LockDowngradeError',
@@ -29,8 +33,6 @@
'CantCreateLockError'
]
#: Mapping of supported locks to description
lock_type = {fcntl.LOCK_SH: 'read', fcntl.LOCK_EX: 'write'}
#: A useful replacement for functions that should return True when not provided
#: for example.
@@ -166,6 +168,29 @@ def _attempts_str(wait_time, nattempts):
return ' after {0:0.2f}s and {1}'.format(wait_time, attempts)
class LockType(object):
READ = 0
WRITE = 1
@staticmethod
def to_str(tid):
ret = "READ"
if tid == LockType.WRITE:
ret = "WRITE"
return ret
@staticmethod
def to_module(tid):
lock = fcntl.LOCK_SH
if tid == LockType.WRITE:
lock = fcntl.LOCK_EX
return lock
@staticmethod
def is_valid(op):
return op == LockType.READ \
or op == LockType.WRITE
class Lock(object):
"""This is an implementation of a filesystem lock using Python's lockf.
@@ -276,9 +301,10 @@ def _lock(self, op, timeout=None):
successfully acquired, the total wait time and the number of attempts
is returned.
"""
assert op in lock_type
assert LockType.is_valid(op)
op_str = LockType.to_str(op)
self._log_acquiring('{0} LOCK'.format(lock_type[op].upper()))
self._log_acquiring('{0} LOCK'.format(op_str))
timeout = timeout or self.default_timeout
# Create file and parent directories if they don't exist.
@@ -286,13 +312,13 @@ def _lock(self, op, timeout=None):
self._ensure_parent_directory()
self._file = file_tracker.get_fh(self.path)
if op == fcntl.LOCK_EX and self._file.mode == 'r':
if LockType.to_module(op) == fcntl.LOCK_EX and self._file.mode == 'r':
# Attempt to upgrade to write lock w/a read-only file.
# If the file were writable, we'd have opened it 'r+'
raise LockROFileError(self.path)
self._log_debug("{0} locking [{1}:{2}]: timeout {3} sec"
.format(lock_type[op], self._start, self._length,
.format(op_str.lower(), self._start, self._length,
timeout))
poll_intervals = iter(Lock._poll_interval_generator())
@@ -313,17 +339,16 @@ def _lock(self, op, timeout=None):
return total_wait_time, num_attempts
raise LockTimeoutError("Timed out waiting for a {0} lock."
.format(lock_type[op]))
.format(op_str.lower()))
def _poll_lock(self, op):
"""Attempt to acquire the lock in a non-blocking manner. Return whether
the locking attempt succeeds
"""
assert op in lock_type
module_op = LockType.to_module(op)
try:
# Try to get the lock (will raise if not available.)
fcntl.lockf(self._file, op | fcntl.LOCK_NB,
fcntl.lockf(self._file, module_op | fcntl.LOCK_NB,
self._length, self._start, os.SEEK_SET)
# help for debugging distributed locking
@@ -331,11 +356,11 @@ def _poll_lock(self, op):
# All locks read the owner PID and host
self._read_log_debug_data()
self._log_debug('{0} locked {1} [{2}:{3}] (owner={4})'
.format(lock_type[op], self.path,
.format(LockType.to_str(op), self.path,
self._start, self._length, self.pid))
# Exclusive locks write their PID/host
if op == fcntl.LOCK_EX:
if module_op == fcntl.LOCK_EX:
self._write_log_debug_data()
return True
@@ -420,7 +445,7 @@ def acquire_read(self, timeout=None):
if self._reads == 0 and self._writes == 0:
# can raise LockError.
wait_time, nattempts = self._lock(fcntl.LOCK_SH, timeout=timeout)
wait_time, nattempts = self._lock(LockType.READ, timeout=timeout)
self._reads += 1
# Log if acquired, which includes counts when verbose
self._log_acquired('READ LOCK', wait_time, nattempts)
@@ -445,7 +470,7 @@ def acquire_write(self, timeout=None):
if self._writes == 0:
# can raise LockError.
wait_time, nattempts = self._lock(fcntl.LOCK_EX, timeout=timeout)
wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout)
self._writes += 1
# Log if acquired, which includes counts when verbose
self._log_acquired('WRITE LOCK', wait_time, nattempts)
@@ -489,7 +514,7 @@ def downgrade_write_to_read(self, timeout=None):
if self._writes == 1 and self._reads == 0:
self._log_downgrading()
# can raise LockError.
wait_time, nattempts = self._lock(fcntl.LOCK_SH, timeout=timeout)
wait_time, nattempts = self._lock(LockType.READ, timeout=timeout)
self._reads = 1
self._writes = 0
self._log_downgraded(wait_time, nattempts)
@@ -508,7 +533,7 @@ def upgrade_read_to_write(self, timeout=None):
if self._reads == 1 and self._writes == 0:
self._log_upgrading()
# can raise LockError.
wait_time, nattempts = self._lock(fcntl.LOCK_EX, timeout=timeout)
wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout)
self._reads = 0
self._writes = 1
self._log_upgraded(wait_time, nattempts)

41
lib/spack/llnl/util/symlink.py
View File

@@ -9,9 +9,11 @@
from os.path import exists, join
from sys import platform as _platform
is_windows = _platform == 'win32'
from llnl.util import lang
__win32_can_symlink__ = None
from spack.util.executable import Executable
is_windows = _platform == 'win32'
def symlink(real_path, link_path):
@@ -51,21 +53,20 @@ def _win32_junction(path, link):
path = os.path.join(parent, path)
path = os.path.abspath(path)
command = "mklink"
default_args = [link, path]
if os.path.isdir(path):
# try using a junction
command = 'mklink /J "%s" "%s"' % (link, path)
default_args.insert(0, '/J')
else:
# try using a hard link
command = 'mklink /H "%s" "%s"' % (link, path)
default_args.insert(0, '/H')
_cmd(command)
Executable(command)(*default_args)
@lang.memoized
def _win32_can_symlink():
global __win32_can_symlink__
if __win32_can_symlink__ is not None:
return __win32_can_symlink__
tempdir = tempfile.mkdtemp()
dpath = join(tempdir, 'dpath')
@@ -92,9 +93,7 @@ def _win32_can_symlink():
# Cleanup the test directory
shutil.rmtree(tempdir)
__win32_can_symlink__ = can_symlink_directories and can_symlink_files
return __win32_can_symlink__
return can_symlink_directories and can_symlink_files
def _win32_is_junction(path):
@@ -119,21 +118,3 @@ def _win32_is_junction(path):
bool(res & FILE_ATTRIBUTE_REPARSE_POINT)
return False
# Based on https://github.com/Erotemic/ubelt/blob/master/ubelt/util_cmd.py
def _cmd(command):
import subprocess
# Create a new process to execute the command
def make_proc():
# delay the creation of the process until we validate all args
proc = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True,
universal_newlines=True, cwd=None, env=None)
return proc
proc = make_proc()
(out, err) = proc.communicate()
if proc.wait() != 0:
raise OSError(str(err))

5
lib/spack/llnl/util/tty/__init__.py
View File

@@ -394,7 +394,8 @@ def ioctl_gwinsz(fd):
return int(rc[0]), int(rc[1])
else:
# return shutil.get_terminal_size()
# TODO: find python 2 compatible module to get terminal size
if sys.version_info[0] < 3:
raise RuntimeError("""Terminal size not obtainable on Windows with a
Python version older than 3""")
rc = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80))
return int(rc[0]), int(rc[1])

3
lib/spack/llnl/util/tty/log.py
View File

@@ -938,8 +938,7 @@ def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo,
# write_fd to terminate the reading loop, so we close the file descriptor
# here. Forking is the process spawning method everywhere except Mac OS
# for Python >= 3.8 and on Windows
if sys.version_info < (3, 8) \
or sys.platform not in ['darwin', 'cygwin']:
if sys.version_info < (3, 8) or sys.platform != 'darwin':
os.close(write_fd)
# Use line buffering (3rd param = 1) since Python 3 has a bug

1
lib/spack/llnl/util/tty/pty.py
View File

@@ -11,6 +11,7 @@
things like timeouts in ``ProcessController.wait()``, which are set to
get tests done quickly, not to avoid high CPU usage.
Note: The functionality in this module is unsupported on Windows
"""
from __future__ import print_function