Refactor utils into separate modules.
This commit is contained in:
Todd Gamblin
@@ -2,9 +2,9 @@
|
||||
import platform as py_platform
|
||||
|
||||
import spack
|
||||
import error as serr
|
||||
from version import Version
|
||||
from util import memoized
|
||||
import spack.error as serr
|
||||
from spack.version import Version
|
||||
from spack.util.lang import memoized
|
||||
|
||||
|
||||
class InvalidSysTypeError(serr.SpackError):
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import spack
|
||||
import spack.packages as packages
|
||||
import spack.test
|
||||
from spack.util import list_modules
|
||||
from spack.util.lang import list_modules
|
||||
from spack.colify import colify
|
||||
from pprint import pprint
|
||||
|
||||
|
||||
@@ -4,8 +4,7 @@
|
||||
|
||||
import spack
|
||||
import spack.compilers.gcc
|
||||
from spack.util import list_modules, memoized
|
||||
|
||||
from spack.util.lang import memoized, list_modules
|
||||
|
||||
@memoized
|
||||
def supported_compilers():
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import os
|
||||
from version import Version
|
||||
from util import *
|
||||
import arch
|
||||
from directory_layout import DefaultDirectoryLayout
|
||||
|
||||
import spack.arch as arch
|
||||
from spack.version import Version
|
||||
from spack.util.filesystem import *
|
||||
from spack.util.executable import *
|
||||
from spack.directory_layout import DefaultDirectoryLayout
|
||||
|
||||
# This lives in $prefix/lib/spac/spack/__file__
|
||||
prefix = ancestor(__file__, 4)
|
||||
|
||||
@@ -24,11 +24,11 @@
|
||||
import validate
|
||||
import url
|
||||
|
||||
|
||||
from spec import Compiler
|
||||
from version import *
|
||||
from multi_function import platform
|
||||
from stage import Stage
|
||||
from spack.util.lang import memoized, list_modules
|
||||
|
||||
|
||||
class Package(object):
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
import spack
|
||||
import spack.error
|
||||
import spack.spec
|
||||
from spack.util import *
|
||||
from spack.util.filesystem import new_path
|
||||
import spack.arch as arch
|
||||
|
||||
# Valid package names can contain '-' but can't start with it.
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
import re
|
||||
|
||||
import spack.error
|
||||
import spack.util
|
||||
import spack.util.filesystem as fs
|
||||
from spack.version import Version
|
||||
|
||||
#
|
||||
@@ -61,9 +61,9 @@ def parse_version_string_with_indices(path):
|
||||
if os.path.isdir(path):
|
||||
stem = os.path.basename(path)
|
||||
elif re.search(r'((?:sourceforge.net|sf.net)/.*)/download$', path):
|
||||
stem = spack.util.stem(os.path.dirname(path))
|
||||
stem = fs.stem(os.path.dirname(path))
|
||||
else:
|
||||
stem = spack.util.stem(path)
|
||||
stem = fs.stem(path)
|
||||
|
||||
version_types = [
|
||||
# GitHub tarballs, e.g. v1.2.3
|
||||
|
||||
@@ -1,209 +0,0 @@
|
||||
import os
|
||||
import re
|
||||
import errno
|
||||
import shutil
|
||||
import subprocess
|
||||
import multiprocessing
|
||||
from itertools import product
|
||||
import functools
|
||||
from contextlib import closing, contextmanager
|
||||
|
||||
import tty
|
||||
|
||||
# Supported archvie extensions.
|
||||
PRE_EXTS = ["tar"]
|
||||
EXTS = ["gz", "bz2", "xz", "Z", "zip", "tgz"]
|
||||
|
||||
# Add EXTS last so that .tar.gz is matched *before* tar.gz
|
||||
ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS
|
||||
|
||||
|
||||
def memoized(obj):
|
||||
"""Decorator that caches the results of a function, storing them
|
||||
in an attribute of that function."""
|
||||
cache = obj.cache = {}
|
||||
@functools.wraps(obj)
|
||||
def memoizer(*args, **kwargs):
|
||||
if args not in cache:
|
||||
cache[args] = obj(*args, **kwargs)
|
||||
return cache[args]
|
||||
return memoizer
|
||||
|
||||
|
||||
def install(src, dest):
|
||||
tty.info("Installing %s to %s" % (src, dest))
|
||||
shutil.copy(src, dest)
|
||||
|
||||
|
||||
def list_modules(directory):
|
||||
"""Lists all of the modules, excluding __init__.py, in
|
||||
a particular directory."""
|
||||
for name in os.listdir(directory):
|
||||
if name == '__init__.py':
|
||||
continue
|
||||
|
||||
path = new_path(directory, name)
|
||||
if os.path.isdir(path):
|
||||
init_py = new_path(path, '__init__.py')
|
||||
if os.path.isfile(init_py):
|
||||
yield name
|
||||
|
||||
elif name.endswith('.py'):
|
||||
yield re.sub('.py$', '', name)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def working_dir(dirname):
|
||||
orig_dir = os.getcwd()
|
||||
os.chdir(dirname)
|
||||
yield
|
||||
os.chdir(orig_dir)
|
||||
|
||||
|
||||
def mkdirp(*paths):
|
||||
for path in paths:
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
elif not os.path.isdir(path):
|
||||
raise OSError(errno.EEXIST, "File alredy exists", path)
|
||||
|
||||
|
||||
def env_flag(name):
|
||||
if name in os.environ:
|
||||
return os.environ[name].lower() == "true"
|
||||
return False
|
||||
|
||||
|
||||
def path_set(var_name, directories):
|
||||
path_str = ":".join(str(dir) for dir in directories)
|
||||
os.environ[var_name] = path_str
|
||||
|
||||
|
||||
def path_put_first(var_name, directories):
|
||||
"""Puts the provided directories first in the path, adding them
|
||||
if they're not already there.
|
||||
"""
|
||||
path = os.environ.get(var_name, "").split(':')
|
||||
|
||||
for dir in directories:
|
||||
if dir in path:
|
||||
path.remove(dir)
|
||||
|
||||
new_path = tuple(directories) + tuple(path)
|
||||
path_set(var_name, new_path)
|
||||
|
||||
|
||||
def pop_keys(dictionary, *keys):
|
||||
for key in keys:
|
||||
if key in dictionary:
|
||||
dictionary.pop(key)
|
||||
|
||||
|
||||
def remove_items(item_list, *items):
|
||||
for item in items:
|
||||
if item in item_list:
|
||||
item_list.remove(item)
|
||||
|
||||
|
||||
def has_whitespace(string):
|
||||
return re.search(r'\s', string)
|
||||
|
||||
|
||||
def new_path(prefix, *args):
|
||||
path=str(prefix)
|
||||
for elt in args:
|
||||
path = os.path.join(path, str(elt))
|
||||
|
||||
if has_whitespace(path):
|
||||
tty.die("Invalid path: '%s'. Use a path without whitespace." % path)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def ancestor(dir, n=1):
|
||||
"""Get the nth ancestor of a directory."""
|
||||
parent = os.path.abspath(dir)
|
||||
for i in range(n):
|
||||
parent = os.path.dirname(parent)
|
||||
return parent
|
||||
|
||||
|
||||
class Executable(object):
|
||||
"""Class representing a program that can be run on the command line."""
|
||||
def __init__(self, name):
|
||||
self.exe = name.split(' ')
|
||||
|
||||
def add_default_arg(self, arg):
|
||||
self.exe.append(arg)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
"""Run the executable with subprocess.check_output, return output."""
|
||||
return_output = kwargs.get("return_output", False)
|
||||
fail_on_error = kwargs.get("fail_on_error", True)
|
||||
|
||||
quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)]
|
||||
if quoted_args:
|
||||
tty.warn("Quotes in package command arguments can confuse shell scripts like configure.",
|
||||
"The following arguments may cause problems when executed:",
|
||||
str("\n".join([" "+arg for arg in quoted_args])),
|
||||
"Quotes aren't needed because spack doesn't use a shell. Consider removing them")
|
||||
|
||||
cmd = self.exe + list(args)
|
||||
tty.verbose(" ".join(cmd))
|
||||
|
||||
if return_output:
|
||||
return subprocess.check_output(cmd)
|
||||
elif fail_on_error:
|
||||
return subprocess.check_call(cmd)
|
||||
else:
|
||||
return subprocess.call(cmd)
|
||||
|
||||
def __repr__(self):
|
||||
return "<exe: %s>" % self.exe
|
||||
|
||||
|
||||
def which(name, **kwargs):
|
||||
"""Finds an executable in the path like command-line which."""
|
||||
path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep))
|
||||
required = kwargs.get('required', False)
|
||||
|
||||
if not path:
|
||||
path = []
|
||||
|
||||
for dir in path:
|
||||
exe = os.path.join(dir, name)
|
||||
if os.access(exe, os.X_OK):
|
||||
return Executable(exe)
|
||||
|
||||
if required:
|
||||
tty.die("spack requires %s. Make sure it is in your path." % name)
|
||||
return None
|
||||
|
||||
|
||||
def stem(path):
|
||||
"""Get the part of a path that does not include its compressed
|
||||
type extension."""
|
||||
for type in ALLOWED_ARCHIVE_TYPES:
|
||||
suffix = r'\.%s$' % type
|
||||
if re.search(suffix, path):
|
||||
return re.sub(suffix, "", path)
|
||||
return path
|
||||
|
||||
|
||||
def decompressor_for(path):
|
||||
"""Get the appropriate decompressor for a path."""
|
||||
tar = which('tar', required=True)
|
||||
tar.add_default_arg('-xf')
|
||||
return tar
|
||||
|
||||
|
||||
def md5(filename, block_size=2**20):
|
||||
import hashlib
|
||||
md5 = hashlib.md5()
|
||||
with closing(open(filename)) as file:
|
||||
while True:
|
||||
data = file.read(block_size)
|
||||
if not data:
|
||||
break
|
||||
md5.update(data)
|
||||
return md5.hexdigest()
|
||||
|
||||
15
lib/spack/spack/util/compression.py
Normal file
15
lib/spack/spack/util/compression.py
Normal file
@@ -0,0 +1,15 @@
|
||||
from itertools import product
|
||||
|
||||
# Supported archvie extensions.
|
||||
PRE_EXTS = ["tar"]
|
||||
EXTS = ["gz", "bz2", "xz", "Z", "zip", "tgz"]
|
||||
|
||||
# Add EXTS last so that .tar.gz is matched *before* tar.gz
|
||||
ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS
|
||||
|
||||
|
||||
def decompressor_for(path):
|
||||
"""Get the appropriate decompressor for a path."""
|
||||
tar = which('tar', required=True)
|
||||
tar.add_default_arg('-xf')
|
||||
return tar
|
||||
30
lib/spack/spack/util/environment.py
Normal file
30
lib/spack/spack/util/environment.py
Normal file
@@ -0,0 +1,30 @@
|
||||
|
||||
def env_flag(name):
|
||||
if name in os.environ:
|
||||
return os.environ[name].lower() == "true"
|
||||
return False
|
||||
|
||||
|
||||
def path_set(var_name, directories):
|
||||
path_str = ":".join(str(dir) for dir in directories)
|
||||
os.environ[var_name] = path_str
|
||||
|
||||
|
||||
def path_put_first(var_name, directories):
|
||||
"""Puts the provided directories first in the path, adding them
|
||||
if they're not already there.
|
||||
"""
|
||||
path = os.environ.get(var_name, "").split(':')
|
||||
|
||||
for dir in directories:
|
||||
if dir in path:
|
||||
path.remove(dir)
|
||||
|
||||
new_path = tuple(directories) + tuple(path)
|
||||
path_set(var_name, new_path)
|
||||
|
||||
|
||||
def pop_keys(dictionary, *keys):
|
||||
for key in keys:
|
||||
if key in dictionary:
|
||||
dictionary.pop(key)
|
||||
55
lib/spack/spack/util/executable.py
Normal file
55
lib/spack/spack/util/executable.py
Normal file
@@ -0,0 +1,55 @@
|
||||
import os
|
||||
import subprocess
|
||||
import spack.tty as tty
|
||||
|
||||
|
||||
class Executable(object):
|
||||
"""Class representing a program that can be run on the command line."""
|
||||
def __init__(self, name):
|
||||
self.exe = name.split(' ')
|
||||
|
||||
def add_default_arg(self, arg):
|
||||
self.exe.append(arg)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
"""Run the executable with subprocess.check_output, return output."""
|
||||
return_output = kwargs.get("return_output", False)
|
||||
fail_on_error = kwargs.get("fail_on_error", True)
|
||||
|
||||
quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)]
|
||||
if quoted_args:
|
||||
tty.warn("Quotes in package command arguments can confuse shell scripts like configure.",
|
||||
"The following arguments may cause problems when executed:",
|
||||
str("\n".join([" "+arg for arg in quoted_args])),
|
||||
"Quotes aren't needed because spack doesn't use a shell. Consider removing them")
|
||||
|
||||
cmd = self.exe + list(args)
|
||||
tty.verbose(" ".join(cmd))
|
||||
|
||||
if return_output:
|
||||
return subprocess.check_output(cmd)
|
||||
elif fail_on_error:
|
||||
return subprocess.check_call(cmd)
|
||||
else:
|
||||
return subprocess.call(cmd)
|
||||
|
||||
def __repr__(self):
|
||||
return "<exe: %s>" % self.exe
|
||||
|
||||
|
||||
def which(name, **kwargs):
|
||||
"""Finds an executable in the path like command-line which."""
|
||||
path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep))
|
||||
required = kwargs.get('required', False)
|
||||
|
||||
if not path:
|
||||
path = []
|
||||
|
||||
for dir in path:
|
||||
exe = os.path.join(dir, name)
|
||||
if os.access(exe, os.X_OK):
|
||||
return Executable(exe)
|
||||
|
||||
if required:
|
||||
tty.die("spack requires %s. Make sure it is in your path." % name)
|
||||
return None
|
||||
71
lib/spack/spack/util/filesystem.py
Normal file
71
lib/spack/spack/util/filesystem.py
Normal file
@@ -0,0 +1,71 @@
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import errno
|
||||
from contextlib import contextmanager, closing
|
||||
|
||||
import spack.tty as tty
|
||||
from spack.util.compression import ALLOWED_ARCHIVE_TYPES
|
||||
|
||||
def install(src, dest):
|
||||
"""Manually install a file to a particular location."""
|
||||
tty.info("Installing %s to %s" % (src, dest))
|
||||
shutil.copy(src, dest)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def working_dir(dirname):
|
||||
orig_dir = os.getcwd()
|
||||
os.chdir(dirname)
|
||||
yield
|
||||
os.chdir(orig_dir)
|
||||
|
||||
|
||||
def mkdirp(*paths):
|
||||
for path in paths:
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
elif not os.path.isdir(path):
|
||||
raise OSError(errno.EEXIST, "File alredy exists", path)
|
||||
|
||||
|
||||
def new_path(prefix, *args):
|
||||
path=str(prefix)
|
||||
for elt in args:
|
||||
path = os.path.join(path, str(elt))
|
||||
|
||||
if re.search(r'\s', path):
|
||||
tty.die("Invalid path: '%s'. Use a path without whitespace." % path)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def ancestor(dir, n=1):
|
||||
"""Get the nth ancestor of a directory."""
|
||||
parent = os.path.abspath(dir)
|
||||
for i in range(n):
|
||||
parent = os.path.dirname(parent)
|
||||
return parent
|
||||
|
||||
|
||||
def stem(path):
|
||||
"""Get the part of a path that does not include its compressed
|
||||
type extension."""
|
||||
for type in ALLOWED_ARCHIVE_TYPES:
|
||||
suffix = r'\.%s$' % type
|
||||
if re.search(suffix, path):
|
||||
return re.sub(suffix, "", path)
|
||||
return path
|
||||
|
||||
|
||||
def md5(filename, block_size=2**20):
|
||||
"""Computes the md5 hash of a file."""
|
||||
import hashlib
|
||||
md5 = hashlib.md5()
|
||||
with closing(open(filename)) as file:
|
||||
while True:
|
||||
data = file.read(block_size)
|
||||
if not data:
|
||||
break
|
||||
md5.update(data)
|
||||
return md5.hexdigest()
|
||||
32
lib/spack/spack/util/lang.py
Normal file
32
lib/spack/spack/util/lang.py
Normal file
@@ -0,0 +1,32 @@
|
||||
import os
|
||||
import re
|
||||
import functools
|
||||
from spack.util.filesystem import new_path
|
||||
|
||||
def memoized(obj):
|
||||
"""Decorator that caches the results of a function, storing them
|
||||
in an attribute of that function."""
|
||||
cache = obj.cache = {}
|
||||
@functools.wraps(obj)
|
||||
def memoizer(*args, **kwargs):
|
||||
if args not in cache:
|
||||
cache[args] = obj(*args, **kwargs)
|
||||
return cache[args]
|
||||
return memoizer
|
||||
|
||||
|
||||
def list_modules(directory):
|
||||
"""Lists all of the modules, excluding __init__.py, in
|
||||
a particular directory."""
|
||||
for name in os.listdir(directory):
|
||||
if name == '__init__.py':
|
||||
continue
|
||||
|
||||
path = new_path(directory, name)
|
||||
if os.path.isdir(path):
|
||||
init_py = new_path(path, '__init__.py')
|
||||
if os.path.isfile(init_py):
|
||||
yield name
|
||||
|
||||
elif name.endswith('.py'):
|
||||
yield re.sub('.py$', '', name)
|
||||
@@ -1,7 +1,8 @@
|
||||
import tty
|
||||
from util import ALLOWED_ARCHIVE_TYPES
|
||||
from urlparse import urlparse
|
||||
|
||||
from spack.util.compression import ALLOWED_ARCHIVE_TYPES
|
||||
|
||||
ALLOWED_SCHEMES = ["http", "https", "ftp"]
|
||||
|
||||
def url(url_string):
|
||||
|
||||
Reference in New Issue
Block a user