zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge from develop

Browse Source
This commit is contained in:
Peter Scheibel
2016-03-31 12:56:34 -07:00
parent a0c42a3fd1 d0c8245a58
commit 9eb314fcaa
115 changed files with 3976 additions and 852 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
lib/spack/docs/packaging_guide.rst
View File

@@ -1844,6 +1844,20 @@ dedicated process.
.. _prefix-objects:
Failing the build
----------------------
Sometimes you don't want a package to successfully install unless some
condition is true. You can explicitly cause the build to fail from
``install()`` by raising an ``InstallError``, for example:
.. code-block:: python
if spec.architecture.startswith('darwin'):
raise InstallError('This package does not build on Mac OS X!')
Prefix objects
----------------------

231
lib/spack/env/cc vendored
View File

@@ -65,7 +65,7 @@ function die {
}
for param in $parameters; do
if [ -z "${!param}" ]; then
if [[ -z ${!param} ]]; then
die "Spack compiler must be run from spack! Input $param was missing!"
fi
done
@@ -78,12 +78,17 @@ done
# 'command' is set based on the input command to $SPACK_[CC|CXX|F77|F90]
#
# 'mode' is set to one of:
# cpp preprocess
# cc compile
# as assemble
# ld link
# ccld compile & link
# cpp preprocessor
# vcheck version check
#
# Depending on the mode, we may or may not add extra rpaths.
# This variable controls whether they are added.
add_rpaths=true
command=$(basename "$0")
case "$command" in
cc|c89|c99|gcc|clang|icc|pgcc|xlc)
@@ -107,13 +112,26 @@ case "$command" in
;;
ld)
mode=ld
# Darwin's linker has a -r argument that merges object files
# together. It doesn't work with -rpath.
if [[ $OSTYPE = darwin* ]]; then
for arg in "$@"; do
if [ "$arg" = -r ]; then
add_rpaths=false
break
fi
done
fi
;;
*)
die "Unkown compiler: $command"
;;
esac
# If any of the arguments below is present then the mode is vcheck. In vcheck mode nothing is added in terms of extra search paths or libraries
# If any of the arguments below is present then the mode is vcheck. In
# vcheck mode nothing is added in terms of extra search paths or
# libraries
if [ -z "$mode" ]; then
for arg in "$@"; do
if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then
@@ -124,13 +142,15 @@ if [ -z "$mode" ]; then
fi
# Finish setting up the mode.
if [ -z "$mode" ]; then
mode=ccld
for arg in "$@"; do
if [ "$arg" = -E ]; then
mode=cpp
break
elif [ "$arg" = -S ]; then
mode=as
break
elif [ "$arg" = -c ]; then
mode=cc
break
@@ -146,168 +166,56 @@ fi
# Check that at least one of the real commands was actually selected,
# otherwise we don't know what to execute.
if [ -z "$command" ]; then
if [[ -z $command ]]; then
die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs."
fi
# Save original command for debug logging
input_command="$@"
if [ "$mode" == vcheck ] ; then
exec ${command} "$@"
fi
#
# Now do real parsing of the command line args, trying hard to keep
# non-rpath linker arguments in the proper order w.r.t. other command
# line arguments. This is important for things like groups.
#
includes=()
libraries=()
libs=()
rpaths=()
other_args=()
while [ -n "$1" ]; do
case "$1" in
-I*)
arg="${1#-I}"
if [ -z "$arg" ]; then shift; arg="$1"; fi
includes+=("$arg")
;;
-L*)
arg="${1#-L}"
if [ -z "$arg" ]; then shift; arg="$1"; fi
libraries+=("$arg")
;;
-l*)
arg="${1#-l}"
if [ -z "$arg" ]; then shift; arg="$1"; fi
libs+=("$arg")
;;
-Wl,*)
arg="${1#-Wl,}"
# TODO: Handle multiple -Wl, continuations of -Wl,-rpath
if [[ $arg == -rpath=* ]]; then
arg="${arg#-rpath=}"
for rpath in ${arg//,/ }; do
rpaths+=("$rpath")
done
elif [[ $arg == -rpath,* ]]; then
arg="${arg#-rpath,}"
for rpath in ${arg//,/ }; do
rpaths+=("$rpath")
done
elif [[ $arg == -rpath ]]; then
shift; arg="$1"
if [[ $arg != '-Wl,'* ]]; then
die "-Wl,-rpath was not followed by -Wl,*"
fi
arg="${arg#-Wl,}"
for rpath in ${arg//,/ }; do
rpaths+=("$rpath")
done
else
other_args+=("-Wl,$arg")
fi
;;
-Xlinker)
shift; arg="$1";
if [[ $arg = -rpath=* ]]; then
rpaths+=("${arg#-rpath=}")
elif [[ $arg = -rpath ]]; then
shift; arg="$1"
if [[ $arg != -Xlinker ]]; then
die "-Xlinker -rpath was not followed by -Xlinker <arg>"
fi
shift; arg="$1"
rpaths+=("$arg")
else
other_args+=("-Xlinker")
other_args+=("$arg")
fi
;;
*)
other_args+=("$1")
;;
esac
shift
done
# Dump parsed values for unit testing if asked for
if [ -n "$SPACK_TEST_COMMAND" ]; then
IFS=$'\n'
case "$SPACK_TEST_COMMAND" in
dump-includes) echo "${includes[*]}";;
dump-libraries) echo "${libraries[*]}";;
dump-libs) echo "${libs[*]}";;
dump-rpaths) echo "${rpaths[*]}";;
dump-other-args) echo "${other_args[*]}";;
dump-all)
echo "INCLUDES:"
echo "${includes[*]}"
echo
echo "LIBRARIES:"
echo "${libraries[*]}"
echo
echo "LIBS:"
echo "${libs[*]}"
echo
echo "RPATHS:"
echo "${rpaths[*]}"
echo
echo "ARGS:"
echo "${other_args[*]}"
;;
*)
echo "ERROR: Unknown test command"
exit 1 ;;
esac
exit
fi
# Save original command for debug logging
input_command="$@"
args=("$@")
# Read spack dependencies from the path environment variable
IFS=':' read -ra deps <<< "$SPACK_DEPENDENCIES"
for dep in "${deps[@]}"; do
if [ -d "$dep/include" ]; then
includes+=("$dep/include")
# Prepend include directories
if [[ -d $dep/include ]]; then
if [[ $mode = cpp || $mode = cc || $mode = as || $mode = ccld ]]; then
args=("-I$dep/include" "${args[@]}")
fi
fi
if [ -d "$dep/lib" ]; then
libraries+=("$dep/lib")
rpaths+=("$dep/lib")
# Prepend lib and RPATH directories
if [[ -d $dep/lib ]]; then
if [[ $mode = ccld ]]; then
$add_rpaths && args=("-Wl,-rpath,$dep/lib" "${args[@]}")
args=("-L$dep/lib" "${args[@]}")
elif [[ $mode = ld ]]; then
$add_rpaths && args=("-rpath" "$dep/lib" "${args[@]}")
args=("-L$dep/lib" "${args[@]}")
fi
fi
if [ -d "$dep/lib64" ]; then
libraries+=("$dep/lib64")
rpaths+=("$dep/lib64")
# Prepend lib64 and RPATH directories
if [[ -d $dep/lib64 ]]; then
if [[ $mode = ccld ]]; then
$add_rpaths && args=("-Wl,-rpath,$dep/lib64" "${args[@]}")
args=("-L$dep/lib64" "${args[@]}")
elif [[ $mode = ld ]]; then
$add_rpaths && args=("-rpath" "$dep/lib64" "${args[@]}")
args=("-L$dep/lib64" "${args[@]}")
fi
fi
done
# Include all -L's and prefix/whatever dirs in rpath
for dir in "${libraries[@]}"; do
[[ dir = $SPACK_INSTALL* ]] && rpaths+=("$dir")
done
rpaths+=("$SPACK_PREFIX/lib")
rpaths+=("$SPACK_PREFIX/lib64")
# Put the arguments together
args=()
for dir in "${includes[@]}"; do args+=("-I$dir"); done
args+=("${other_args[@]}")
for dir in "${libraries[@]}"; do args+=("-L$dir"); done
for lib in "${libs[@]}"; do args+=("-l$lib"); done
if [ "$mode" = ccld ]; then
for dir in "${rpaths[@]}"; do
args+=("-Wl,-rpath")
args+=("-Wl,$dir");
done
elif [ "$mode" = ld ]; then
for dir in "${rpaths[@]}"; do
args+=("-rpath")
args+=("$dir");
done
if [[ $mode = ccld ]]; then
$add_rpaths && args=("-Wl,-rpath,$SPACK_PREFIX/lib" "-Wl,-rpath,$SPACK_PREFIX/lib64" "${args[@]}")
elif [[ $mode = ld ]]; then
$add_rpaths && args=("-rpath" "$SPACK_PREFIX/lib" "-rpath" "$SPACK_PREFIX/lib64" "${args[@]}")
fi
#
@@ -323,34 +231,37 @@ unset DYLD_LIBRARY_PATH
#
IFS=':' read -ra env_path <<< "$PATH"
IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH"
spack_env_dirs+=(".")
spack_env_dirs+=("" ".")
PATH=""
for dir in "${env_path[@]}"; do
remove=""
for rm_dir in "${spack_env_dirs[@]}"; do
if [ "$dir" = "$rm_dir" ]; then remove=True; fi
if [[ $dir = $rm_dir ]]; then remove=True; fi
done
if [ -z "$remove" ]; then
if [ -z "$PATH" ]; then
PATH="$dir"
else
PATH="$PATH:$dir"
fi
if [[ -z $remove ]]; then
PATH="${PATH:+$PATH:}$dir"
fi
done
export PATH
full_command=("$command")
full_command+=("${args[@]}")
full_command=("$command" "${args[@]}")
# In test command mode, write out full command for Spack tests.
if [[ $SPACK_TEST_COMMAND = dump-args ]]; then
echo "${full_command[@]}"
exit
elif [[ -n $SPACK_TEST_COMMAND ]]; then
die "ERROR: Unknown test command"
fi
#
# Write the input and output commands to debug logs if it's asked for.
#
if [ "$SPACK_DEBUG" = "TRUE" ]; then
if [[ $SPACK_DEBUG = TRUE ]]; then
input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.in.log"
output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.out.log"
echo "$input_command" >> $input_log
echo "$mode ${full_command[@]}" >> $output_log
echo "[$mode] $command $input_command" >> $input_log
echo "[$mode] ${full_command[@]}" >> $output_log
fi
exec "${full_command[@]}"

46
lib/spack/llnl/util/filesystem.py
View File

@@ -27,9 +27,11 @@
'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file',
'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink',
'set_executable', 'copy_mode', 'unset_executable_mode',
'remove_dead_links', 'remove_linked_tree']
'remove_dead_links', 'remove_linked_tree', 'find_library_path',
'fix_darwin_install_name']
import os
import glob
import sys
import re
import shutil
@@ -38,6 +40,7 @@
import getpass
from contextlib import contextmanager, closing
from tempfile import NamedTemporaryFile
import subprocess
import llnl.util.tty as tty
from spack.util.compression import ALLOWED_ARCHIVE_TYPES
@@ -392,3 +395,44 @@ def remove_linked_tree(path):
os.unlink(path)
else:
shutil.rmtree(path, True)
def fix_darwin_install_name(path):
"""
Fix install name of dynamic libraries on Darwin to have full path.
There are two parts of this task:
(i) use install_name('-id',...) to change install name of a single lib;
(ii) use install_name('-change',...) to change the cross linking between libs.
The function assumes that all libraries are in one folder and currently won't
follow subfolders.
Args:
path: directory in which .dylib files are alocated
"""
libs = glob.glob(join_path(path,"*.dylib"))
for lib in libs:
# fix install name first:
subprocess.Popen(["install_name_tool", "-id",lib,lib], stdout=subprocess.PIPE).communicate()[0]
long_deps = subprocess.Popen(["otool", "-L",lib], stdout=subprocess.PIPE).communicate()[0].split('\n')
deps = [dep.partition(' ')[0][1::] for dep in long_deps[2:-1]]
# fix all dependencies:
for dep in deps:
for loc in libs:
if dep == os.path.basename(loc):
subprocess.Popen(["install_name_tool", "-change",dep,loc,lib], stdout=subprocess.PIPE).communicate()[0]
break
def find_library_path(libname, *paths):
"""Searches for a file called <libname> in each path.
Return:
directory where the library was found, if found. None otherwise.
"""
for path in paths:
library = join_path(path, libname)
if os.path.exists(library):
return path
return None

6
lib/spack/llnl/util/lang.py
View File

@@ -117,7 +117,8 @@ def caller_locals():
scope. Yes, this is some black magic, and yes it's useful
for implementing things like depends_on and provides.
"""
stack = inspect.stack()
# Passing zero here skips line context for speed.
stack = inspect.stack(0)
try:
return stack[2][0].f_locals
finally:
@@ -128,7 +129,8 @@ def get_calling_module_name():
"""Make sure that the caller is a class definition, and return the
enclosing module's name.
"""
stack = inspect.stack()
# Passing zero here skips line context for speed.
stack = inspect.stack(0)
try:
# Make sure locals contain __module__
caller_locals = stack[2][0].f_locals

8
lib/spack/spack/__init__.py
View File

@@ -193,5 +193,9 @@
from spack.util.executable import *
__all__ += spack.util.executable.__all__
from spack.package import install_dependency_symlinks, flatten_dependencies, DependencyConflictError
__all__ += ['install_dependency_symlinks', 'flatten_dependencies', 'DependencyConflictError']
from spack.package import \
install_dependency_symlinks, flatten_dependencies, DependencyConflictError, \
InstallError, ExternalPackageError
__all__ += [
'install_dependency_symlinks', 'flatten_dependencies', 'DependencyConflictError',
'InstallError', 'ExternalPackageError']

200
lib/spack/spack/build_environment.py
View File

@@ -3,7 +3,7 @@
build environment. All of this is set up by package.py just before
install() is called.
There are two parts to the bulid environment:
There are two parts to the build environment:
1. Python build environment (i.e. install() method)
@@ -13,7 +13,7 @@
the package's module scope. Ths allows package writers to call
them all directly in Package.install() without writing 'self.'
everywhere. No, this isn't Pythonic. Yes, it makes the code more
readable and more like the shell script from whcih someone is
readable and more like the shell script from which someone is
likely porting.
2. Build execution environment
@@ -27,17 +27,18 @@
Skimming this module is a nice way to get acquainted with the types of
calls you can make from within the install() function.
"""
import os
import sys
import shutil
import multiprocessing
import os
import platform
from llnl.util.filesystem import *
import shutil
import sys
import spack
import spack.compilers as compilers
from spack.util.executable import Executable, which
import llnl.util.tty as tty
from llnl.util.filesystem import *
from spack.environment import EnvironmentModifications, validate
from spack.util.environment import *
from spack.util.executable import Executable, which
#
# This can be set by the user to globally disable parallel builds.
@@ -58,6 +59,11 @@
SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR'
# Platform-specific library suffix.
dso_suffix = 'dylib' if sys.platform == 'darwin' else 'so'
class MakeExecutable(Executable):
"""Special callable executable object for make so the user can
specify parallel or not on a per-invocation basis. Using
@@ -83,85 +89,88 @@ def __call__(self, *args, **kwargs):
return super(MakeExecutable, self).__call__(*args, **kwargs)
def set_compiler_environment_variables(pkg):
assert(pkg.spec.concrete)
compiler = pkg.compiler
def set_compiler_environment_variables(pkg, env):
assert pkg.spec.concrete
# Set compiler variables used by CMake and autotools
assert all(key in pkg.compiler.link_paths
for key in ('cc', 'cxx', 'f77', 'fc'))
assert all(key in pkg.compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc'))
# Populate an object with the list of environment modifications
# and return it
# TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc.
link_dir = spack.build_env_path
os.environ['CC'] = join_path(link_dir, pkg.compiler.link_paths['cc'])
os.environ['CXX'] = join_path(link_dir, pkg.compiler.link_paths['cxx'])
os.environ['F77'] = join_path(link_dir, pkg.compiler.link_paths['f77'])
os.environ['FC'] = join_path(link_dir, pkg.compiler.link_paths['fc'])
env.set('CC', join_path(link_dir, pkg.compiler.link_paths['cc']))
env.set('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx']))
env.set('F77', join_path(link_dir, pkg.compiler.link_paths['f77']))
env.set('FC', join_path(link_dir, pkg.compiler.link_paths['fc']))
# Set SPACK compiler variables so that our wrapper knows what to call
compiler = pkg.compiler
if compiler.cc:
os.environ['SPACK_CC'] = compiler.cc
env.set('SPACK_CC', compiler.cc)
if compiler.cxx:
os.environ['SPACK_CXX'] = compiler.cxx
env.set('SPACK_CXX', compiler.cxx)
if compiler.f77:
os.environ['SPACK_F77'] = compiler.f77
env.set('SPACK_F77', compiler.f77)
if compiler.fc:
os.environ['SPACK_FC'] = compiler.fc
env.set('SPACK_FC', compiler.fc)
os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler)
env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler))
return env
def set_build_environment_variables(pkg):
"""This ensures a clean install environment when we build packages.
def set_build_environment_variables(pkg, env):
"""
This ensures a clean install environment when we build packages
"""
# Add spack build environment path with compiler wrappers first in
# the path. We add both spack.env_path, which includes default
# wrappers (cc, c++, f77, f90), AND a subdirectory containing
# compiler-specific symlinks. The latter ensures that builds that
# are sensitive to the *name* of the compiler see the right name
# when we're building wtih the wrappers.
# when we're building with the wrappers.
#
# Conflicts on case-insensitive systems (like "CC" and "cc") are
# handled by putting one in the <build_env_path>/case-insensitive
# directory. Add that to the path too.
env_paths = []
def add_env_path(path):
env_paths.append(path)
ci = join_path(path, 'case-insensitive')
if os.path.isdir(ci): env_paths.append(ci)
add_env_path(spack.build_env_path)
add_env_path(join_path(spack.build_env_path, pkg.compiler.name))
for item in [spack.build_env_path, join_path(spack.build_env_path, pkg.compiler.name)]:
env_paths.append(item)
ci = join_path(item, 'case-insensitive')
if os.path.isdir(ci):
env_paths.append(ci)
path_put_first("PATH", env_paths)
path_set(SPACK_ENV_PATH, env_paths)
for item in reversed(env_paths):
env.prepend_path('PATH', item)
env.set_path(SPACK_ENV_PATH, env_paths)
# Prefixes of all of the package's dependencies go in
# SPACK_DEPENDENCIES
# Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES
dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)]
path_set(SPACK_DEPENDENCIES, dep_prefixes)
env.set_path(SPACK_DEPENDENCIES, dep_prefixes)
env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Add dependencies to CMAKE_PREFIX_PATH
# Install prefix
os.environ[SPACK_PREFIX] = pkg.prefix
env.set(SPACK_PREFIX, pkg.prefix)
# Install root prefix
os.environ[SPACK_INSTALL] = spack.install_path
env.set(SPACK_INSTALL, spack.install_path)
# Remove these vars from the environment during build because they
# can affect how some packages find libraries. We want to make
# sure that builds never pull in unintended external dependencies.
pop_keys(os.environ, "LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH")
env.unset('LD_LIBRARY_PATH')
env.unset('LD_RUN_PATH')
env.unset('DYLD_LIBRARY_PATH')
# Add bin directories from dependencies to the PATH for the build.
bin_dirs = ['%s/bin' % prefix for prefix in dep_prefixes]
path_put_first('PATH', [bin for bin in bin_dirs if os.path.isdir(bin)])
bin_dirs = reversed(filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes]))
for item in bin_dirs:
env.prepend_path('PATH', item)
# Working directory for the spack command itself, for debug logs.
if spack.debug:
os.environ[SPACK_DEBUG] = "TRUE"
os.environ[SPACK_SHORT_SPEC] = pkg.spec.short_spec
os.environ[SPACK_DEBUG_LOG_DIR] = spack.spack_working_dir
# Add dependencies to CMAKE_PREFIX_PATH
path_set("CMAKE_PREFIX_PATH", dep_prefixes)
env.set(SPACK_DEBUG, 'TRUE')
env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec)
env.set(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir)
# Add any pkgconfig directories to PKG_CONFIG_PATH
pkg_config_dirs = []
@@ -170,10 +179,12 @@ def add_env_path(path):
pcdir = join_path(p, libdir, 'pkgconfig')
if os.path.isdir(pcdir):
pkg_config_dirs.append(pcdir)
path_set("PKG_CONFIG_PATH", pkg_config_dirs)
env.set_path('PKG_CONFIG_PATH', pkg_config_dirs)
return env
def set_module_variables_for_package(pkg, m):
def set_module_variables_for_package(pkg, module):
"""Populate the module scope of install() with some useful functions.
This makes things easier for package writers.
"""
@@ -183,6 +194,8 @@ def set_module_variables_for_package(pkg, m):
jobs = 1
elif pkg.make_jobs:
jobs = pkg.make_jobs
m = module
m.make_jobs = jobs
# TODO: make these build deps that can be installed if not found.
@@ -217,7 +230,7 @@ def set_module_variables_for_package(pkg, m):
m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc'])
m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx'])
m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77'])
m.spack_f90 = join_path(link_dir, pkg.compiler.link_paths['fc'])
m.spack_fc = join_path(link_dir, pkg.compiler.link_paths['fc'])
# Emulate some shell commands for convenience
m.pwd = os.getcwd
@@ -238,6 +251,9 @@ def set_module_variables_for_package(pkg, m):
# a Prefix object.
m.prefix = pkg.prefix
# Platform-specific library suffix.
m.dso_suffix = dso_suffix
def get_rpaths(pkg):
"""Get a list of all the rpaths for a package."""
@@ -262,24 +278,63 @@ def parent_class_modules(cls):
return result
def setup_module_variables_for_dag(pkg):
"""Set module-scope variables for all packages in the DAG."""
for spec in pkg.spec.traverse(order='post'):
# If a user makes their own package repo, e.g.
# spack.repos.mystuff.libelf.Libelf, and they inherit from
# an existing class like spack.repos.original.libelf.Libelf,
# then set the module variables for both classes so the
# parent class can still use them if it gets called.
spkg = spec.package
modules = parent_class_modules(spkg.__class__)
for mod in modules:
set_module_variables_for_package(spkg, mod)
set_module_variables_for_package(spkg, spkg.module)
def setup_package(pkg):
"""Execute all environment setup routines."""
set_compiler_environment_variables(pkg)
set_build_environment_variables(pkg)
spack_env = EnvironmentModifications()
run_env = EnvironmentModifications()
# If a user makes their own package repo, e.g.
# spack.repos.mystuff.libelf.Libelf, and they inherit from
# an existing class like spack.repos.original.libelf.Libelf,
# then set the module variables for both classes so the
# parent class can still use them if it gets called.
modules = parent_class_modules(pkg.__class__)
for mod in modules:
set_module_variables_for_package(pkg, mod)
# Before proceeding, ensure that specs and packages are consistent
#
# This is a confusing behavior due to how packages are
# constructed. `setup_dependent_package` may set attributes on
# specs in the DAG for use by other packages' install
# method. However, spec.package will look up a package via
# spack.repo, which defensively copies specs into packages. This
# code ensures that all packages in the DAG have pieces of the
# same spec object at build time.
#
# This is safe for the build process, b/c the build process is a
# throwaway environment, but it is kind of dirty.
#
# TODO: Think about how to avoid this fix and do something cleaner.
for s in pkg.spec.traverse(): s.package.spec = s
# Allow dependencies to set up environment as well.
for dep_spec in pkg.spec.traverse(root=False):
dep_spec.package.setup_dependent_environment(
pkg.module, dep_spec, pkg.spec)
set_compiler_environment_variables(pkg, spack_env)
set_build_environment_variables(pkg, spack_env)
setup_module_variables_for_dag(pkg)
# Allow dependencies to modify the module
spec = pkg.spec
for dependency_spec in spec.traverse(root=False):
dpkg = dependency_spec.package
dpkg.setup_dependent_package(pkg.module, spec)
# Allow dependencies to set up environment as well
for dependency_spec in spec.traverse(root=False):
dpkg = dependency_spec.package
dpkg.setup_dependent_environment(spack_env, run_env, spec)
# Allow the package to apply some settings.
pkg.setup_environment(spack_env, run_env)
# Make sure nothing's strange about the Spack environment.
validate(spack_env, tty.warn)
spack_env.apply_modifications()
def fork(pkg, function):
@@ -296,23 +351,23 @@ def child_fun():
# do stuff
build_env.fork(pkg, child_fun)
Forked processes are run with the build environemnt set up by
Forked processes are run with the build environment set up by
spack.build_environment. This allows package authors to have
full control over the environment, etc. without offecting
full control over the environment, etc. without affecting
other builds that might be executed in the same spack call.
If something goes wrong, the child process is expected toprint
If something goes wrong, the child process is expected to print
the error and the parent process will exit with error as
well. If things go well, the child exits and the parent
carries on.
"""
try:
pid = os.fork()
except OSError, e:
except OSError as e:
raise InstallError("Unable to fork build process: %s" % e)
if pid == 0:
# Give the child process the package's build environemnt.
# Give the child process the package's build environment.
setup_package(pkg)
try:
@@ -323,7 +378,7 @@ def child_fun():
# which interferes with unit tests.
os._exit(0)
except spack.error.SpackError, e:
except spack.error.SpackError as e:
e.die()
except:
@@ -338,8 +393,7 @@ def child_fun():
# message. Just make the parent exit with an error code.
pid, returncode = os.waitpid(pid, 0)
if returncode != 0:
raise InstallError("Installation process had nonzero exit code."
.format(str(returncode)))
raise InstallError("Installation process had nonzero exit code.".format(str(returncode)))
class InstallError(spack.error.SpackError):

4
lib/spack/spack/cmd/info.py
View File

@@ -52,7 +52,7 @@ def print_text_info(pkg):
print "Safe versions: "
if not pkg.versions:
print("None")
print(" None")
else:
pad = padder(pkg.versions, 4)
for v in reversed(sorted(pkg.versions)):
@@ -62,7 +62,7 @@ def print_text_info(pkg):
print
print "Variants:"
if not pkg.variants:
print "None"
print " None"
else:
pad = padder(pkg.variants, 4)

2
lib/spack/spack/cmd/module.py
View File

@@ -80,7 +80,7 @@ def module_find(mtype, spec_array):
if not os.path.isfile(mod.file_name):
tty.die("No %s module is installed for %s" % (mtype, spec))
print mod.use_name
print(mod.use_name)
def module_refresh():

3
lib/spack/spack/cmd/pkg.py
View File

@@ -77,7 +77,8 @@ def get_git():
def list_packages(rev):
git = get_git()
relpath = spack.packages_path[len(spack.prefix + os.path.sep):] + os.path.sep
pkgpath = os.path.join(spack.packages_path, 'packages')
relpath = pkgpath[len(spack.prefix + os.path.sep):] + os.path.sep
output = git('ls-tree', '--full-tree', '--name-only', rev, relpath,
output=str)
return sorted(line[len(relpath):] for line in output.split('\n') if line)

5
lib/spack/spack/cmd/stage.py
View File

@@ -35,6 +35,9 @@ def setup_parser(subparser):
subparser.add_argument(
'-n', '--no-checksum', action='store_true', dest='no_checksum',
help="Do not check downloaded packages against checksum")
subparser.add_argument(
'-p', '--path', dest='path',
help="Path to stage package, does not add to spack tree")
subparser.add_argument(
'specs', nargs=argparse.REMAINDER, help="specs of packages to stage")
@@ -50,4 +53,6 @@ def stage(parser, args):
specs = spack.cmd.parse_specs(args.specs, concretize=True)
for spec in specs:
package = spack.repo.get(spec)
if args.path:
package.path = args.path
package.do_stage()

23
lib/spack/spack/cmd/uninstall.py
View File

@@ -22,6 +22,7 @@
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from __future__ import print_function
import sys
import argparse
@@ -63,12 +64,12 @@ def uninstall(parser, args):
matching_specs = spack.installed_db.query(spec)
if not args.all and len(matching_specs) > 1:
tty.error("%s matches multiple packages:" % spec)
print
print()
display_specs(matching_specs, long=True)
print
print "You can either:"
print " a) Use a more specific spec, or"
print " b) use spack uninstall -a to uninstall ALL matching specs."
print()
print("You can either:")
print(" a) Use a more specific spec, or")
print(" b) use spack uninstall -a to uninstall ALL matching specs.")
sys.exit(1)
if len(matching_specs) == 0:
@@ -79,7 +80,7 @@ def uninstall(parser, args):
try:
# should work if package is known to spack
pkgs.append(s.package)
except spack.repository.UnknownPackageError, e:
except spack.repository.UnknownPackageError as e:
# The package.py file has gone away -- but still
# want to uninstall.
spack.Package(s).do_uninstall(force=True)
@@ -94,11 +95,11 @@ def num_installed_deps(pkg):
for pkg in pkgs:
try:
pkg.do_uninstall(force=args.force)
except PackageStillNeededError, e:
except PackageStillNeededError as e:
tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True))
print
print "The following packages depend on it:"
print('')
print("The following packages depend on it:")
display_specs(e.dependents, long=True)
print
print "You can use spack uninstall -f to force this action."
print('')
print("You can use spack uninstall -f to force this action.")
sys.exit(1)

8
lib/spack/spack/concretize.py
View File

@@ -159,6 +159,10 @@ def concretize_version(self, spec):
if any(v.satisfies(sv) for sv in spec.versions)],
cmp=cmp_versions)
def prefer_key(v):
return pkg.versions.get(Version(v)).get('preferred', False)
valid_versions.sort(key=prefer_key, reverse=True)
if valid_versions:
spec.versions = ver([valid_versions[0]])
else:
@@ -241,7 +245,7 @@ def concretize_compiler(self, spec):
return False
#Find the another spec that has a compiler, or the root if none do
other_spec = find_spec(spec, lambda(x) : x.compiler)
other_spec = spec if spec.compiler else find_spec(spec, lambda(x) : x.compiler)
if not other_spec:
other_spec = spec.root
other_compiler = other_spec.compiler
@@ -288,7 +292,7 @@ def find_spec(spec, condition):
if condition(spec):
return spec
return None # Nohting matched the condition.
return None # Nothing matched the condition.
def cmp_specs(lhs, rhs):

28
lib/spack/spack/config.py
View File

@@ -237,7 +237,29 @@
'type' : 'object',
'default' : {},
}
},},},},},}
},},},},},},
'modules': {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack module file configuration file schema',
'type': 'object',
'additionalProperties': False,
'patternProperties': {
r'modules:?': {
'type': 'object',
'default': {},
'additionalProperties': False,
'properties': {
'enable': {
'type': 'array',
'default': [],
'items': {
'type': 'string'
}
}
}
},
},
},
}
"""OrderedDict of config scopes keyed by name.
@@ -405,11 +427,11 @@ def _read_config_file(filename, schema):
validate_section(data, schema)
return data
except MarkedYAMLError, e:
except MarkedYAMLError as e:
raise ConfigFileError(
"Error parsing yaml%s: %s" % (str(e.context_mark), e.problem))
except IOError, e:
except IOError as e:
raise ConfigFileError(
"Error reading configuration file %s: %s" % (filename, str(e)))

2
lib/spack/spack/directory_layout.py
View File

@@ -150,7 +150,7 @@ def remove_install_directory(self, spec):
if os.path.exists(path):
try:
shutil.rmtree(path)
except exceptions.OSError, e:
except exceptions.OSError as e:
raise RemoveFailedError(spec, path, e)
path = os.path.dirname(path)

252
lib/spack/spack/environment.py Normal file
View File

@@ -0,0 +1,252 @@
import os
import os.path
import collections
import inspect
class NameModifier(object):
def __init__(self, name, **kwargs):
self.name = name
self.args = {'name': name}
self.args.update(kwargs)
class NameValueModifier(object):
def __init__(self, name, value, **kwargs):
self.name = name
self.value = value
self.args = {'name': name, 'value': value}
self.args.update(kwargs)
class SetEnv(NameValueModifier):
def execute(self):
os.environ[self.name] = str(self.value)
class UnsetEnv(NameModifier):
def execute(self):
os.environ.pop(self.name, None) # Avoid throwing if the variable was not set
class SetPath(NameValueModifier):
def execute(self):
string_path = concatenate_paths(self.value)
os.environ[self.name] = string_path
class AppendPath(NameValueModifier):
def execute(self):
environment_value = os.environ.get(self.name, '')
directories = environment_value.split(':') if environment_value else []
directories.append(os.path.normpath(self.value))
os.environ[self.name] = ':'.join(directories)
class PrependPath(NameValueModifier):
def execute(self):
environment_value = os.environ.get(self.name, '')
directories = environment_value.split(':') if environment_value else []
directories = [os.path.normpath(self.value)] + directories
os.environ[self.name] = ':'.join(directories)
class RemovePath(NameValueModifier):
def execute(self):
environment_value = os.environ.get(self.name, '')
directories = environment_value.split(':') if environment_value else []
directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)]
os.environ[self.name] = ':'.join(directories)
class EnvironmentModifications(object):
"""
Keeps track of requests to modify the current environment.
Each call to a method to modify the environment stores the extra information on the caller in the request:
- 'filename' : filename of the module where the caller is defined
- 'lineno': line number where the request occurred
- 'context' : line of code that issued the request that failed
"""
def __init__(self, other=None):
"""
Initializes a new instance, copying commands from other if it is not None
Args:
other: another instance of EnvironmentModifications from which (optional)
"""
self.env_modifications = []
if other is not None:
self.extend(other)
def __iter__(self):
return iter(self.env_modifications)
def __len__(self):
return len(self.env_modifications)
def extend(self, other):
self._check_other(other)
self.env_modifications.extend(other.env_modifications)
@staticmethod
def _check_other(other):
if not isinstance(other, EnvironmentModifications):
raise TypeError('other must be an instance of EnvironmentModifications')
def _get_outside_caller_attributes(self):
stack = inspect.stack()
try:
_, filename, lineno, _, context, index = stack[2]
context = context[index].strip()
except Exception:
filename, lineno, context = 'unknown file', 'unknown line', 'unknown context'
args = {
'filename': filename,
'lineno': lineno,
'context': context
}
return args
def set(self, name, value, **kwargs):
"""
Stores in the current object a request to set an environment variable
Args:
name: name of the environment variable to be set
value: value of the environment variable
"""
kwargs.update(self._get_outside_caller_attributes())
item = SetEnv(name, value, **kwargs)
self.env_modifications.append(item)
def unset(self, name, **kwargs):
"""
Stores in the current object a request to unset an environment variable
Args:
name: name of the environment variable to be set
"""
kwargs.update(self._get_outside_caller_attributes())
item = UnsetEnv(name, **kwargs)
self.env_modifications.append(item)
def set_path(self, name, elts, **kwargs):
"""
Stores a request to set a path generated from a list.
Args:
name: name o the environment variable to be set.
elts: elements of the path to set.
"""
kwargs.update(self._get_outside_caller_attributes())
item = SetPath(name, elts, **kwargs)
self.env_modifications.append(item)
def append_path(self, name, path, **kwargs):
"""
Stores in the current object a request to append a path to a path list
Args:
name: name of the path list in the environment
path: path to be appended
"""
kwargs.update(self._get_outside_caller_attributes())
item = AppendPath(name, path, **kwargs)
self.env_modifications.append(item)
def prepend_path(self, name, path, **kwargs):
"""
Same as `append_path`, but the path is pre-pended
Args:
name: name of the path list in the environment
path: path to be pre-pended
"""
kwargs.update(self._get_outside_caller_attributes())
item = PrependPath(name, path, **kwargs)
self.env_modifications.append(item)
def remove_path(self, name, path, **kwargs):
"""
Stores in the current object a request to remove a path from a path list
Args:
name: name of the path list in the environment
path: path to be removed
"""
kwargs.update(self._get_outside_caller_attributes())
item = RemovePath(name, path, **kwargs)
self.env_modifications.append(item)
def group_by_name(self):
"""
Returns a dict of the modifications grouped by variable name
Returns:
dict mapping the environment variable name to the modifications to be done on it
"""
modifications = collections.defaultdict(list)
for item in self:
modifications[item.name].append(item)
return modifications
def clear(self):
"""
Clears the current list of modifications
"""
self.env_modifications.clear()
def apply_modifications(self):
"""
Applies the modifications and clears the list
"""
modifications = self.group_by_name()
# Apply the modifications to the environment variables one variable at a time
for name, actions in sorted(modifications.items()):
for x in actions:
x.execute()
def concatenate_paths(paths):
"""
Concatenates an iterable of paths into a string of column separated paths
Args:
paths: iterable of paths
Returns:
string
"""
return ':'.join(str(item) for item in paths)
def set_or_unset_not_first(variable, changes, errstream):
"""
Check if we are going to set or unset something after other modifications have already been requested
"""
indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]]
if indexes:
good = '\t \t{context} at {filename}:{lineno}'
nogood = '\t--->\t{context} at {filename}:{lineno}'
errstream('Suspicious requests to set or unset the variable \'{var}\' found'.format(var=variable))
for ii, item in enumerate(changes):
print_format = nogood if ii in indexes else good
errstream(print_format.format(**item.args))
def validate(env, errstream):
"""
Validates the environment modifications to check for the presence of suspicious patterns. Prompts a warning for
everything that was found
Current checks:
- set or unset variables after other changes on the same variable
Args:
env: list of environment modifications
"""
modifications = env.group_by_name()
for variable, list_of_changes in sorted(modifications.items()):
set_or_unset_not_first(variable, list_of_changes, errstream)

10
lib/spack/spack/fetch_strategy.py
View File

@@ -289,8 +289,14 @@ def reset(self):
if not self.archive_file:
raise NoArchiveFileError("Tried to reset URLFetchStrategy before fetching",
"Failed on reset() for URL %s" % self.url)
if self.stage.source_path:
shutil.rmtree(self.stage.source_path, ignore_errors=True)
# Remove everythigng but the archive from the stage
for filename in os.listdir(self.stage.path):
abspath = os.path.join(self.stage.path, filename)
if abspath != self.archive_file:
shutil.rmtree(abspath, ignore_errors=True)
# Expand the archive again
self.expand()
def __repr__(self):

245
lib/spack/spack/modules.py
View File

@@ -22,14 +22,12 @@
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""This module contains code for creating environment modules, which
can include dotkits, tcl modules, lmod, and others.
"""
This module contains code for creating environment modules, which can include dotkits, tcl modules, lmod, and others.
The various types of modules are installed by post-install hooks and
removed after an uninstall by post-uninstall hooks. This class
consolidates the logic for creating an abstract description of the
information that module systems need. Currently that includes a
number of directories to be appended to paths in the user's environment:
The various types of modules are installed by post-install hooks and removed after an uninstall by post-uninstall hooks.
This class consolidates the logic for creating an abstract description of the information that module systems need.
Currently that includes a number of directories to be appended to paths in the user's environment:
* /bin directories to be appended to PATH
* /lib* directories for LD_LIBRARY_PATH
@@ -37,30 +35,30 @@
* /man* and /share/man* directories for MANPATH
* the package prefix for CMAKE_PREFIX_PATH
This module also includes logic for coming up with unique names for
the module files so that they can be found by the various
shell-support files in $SPACK/share/spack/setup-env.*.
This module also includes logic for coming up with unique names for the module files so that they can be found by the
various shell-support files in $SPACK/share/spack/setup-env.*.
Each hook in hooks/ implements the logic for writing its specific type
of module file.
Each hook in hooks/ implements the logic for writing its specific type of module file.
"""
__all__ = ['EnvModule', 'Dotkit', 'TclModule']
import os
import os.path
import re
import textwrap
import shutil
from glob import glob
import textwrap
import llnl.util.tty as tty
from llnl.util.filesystem import join_path, mkdirp
import spack
import spack.config
from llnl.util.filesystem import join_path, mkdirp
from spack.environment import *
"""Registry of all types of modules. Entries created by EnvModule's
metaclass."""
__all__ = ['EnvModule', 'Dotkit', 'TclModule']
# Registry of all types of modules. Entries created by EnvModule's metaclass
module_types = {}
CONFIGURATION = spack.config.get_config('modules')
def print_help():
"""For use by commands to tell user how to activate shell support."""
@@ -79,75 +77,76 @@ def print_help():
"")
def inspect_path(prefix):
"""
Inspects the prefix of an installation to search for common layouts. Issues a request to modify the environment
accordingly when an item is found.
Args:
prefix: prefix of the installation
Returns:
instance of EnvironmentModifications containing the requested modifications
"""
env = EnvironmentModifications()
# Inspect the prefix to check for the existence of common directories
prefix_inspections = {
'bin': ('PATH',),
'man': ('MANPATH',),
'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'),
'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'),
'include': ('CPATH',)
}
for attribute, variables in prefix_inspections.items():
expected = getattr(prefix, attribute)
if os.path.isdir(expected):
for variable in variables:
env.prepend_path(variable, expected)
# PKGCONFIG
for expected in (join_path(prefix.lib, 'pkgconfig'), join_path(prefix.lib64, 'pkgconfig')):
if os.path.isdir(expected):
env.prepend_path('PKG_CONFIG_PATH', expected)
# CMake related variables
env.prepend_path('CMAKE_PREFIX_PATH', prefix)
return env
class EnvModule(object):
name = 'env_module'
formats = {}
class __metaclass__(type):
def __init__(cls, name, bases, dict):
type.__init__(cls, name, bases, dict)
if cls.name != 'env_module':
if cls.name != 'env_module' and cls.name in CONFIGURATION['enable']:
module_types[cls.name] = cls
def __init__(self, spec=None):
# category in the modules system
# TODO: come up with smarter category names.
self.category = "spack"
# Descriptions for the module system's UI
self.short_description = ""
self.long_description = ""
# dict pathname -> list of directories to be prepended to in
# the module file.
self._paths = None
self.spec = spec
self.pkg = spec.package # Just stored for convenience
# short description default is just the package + version
# packages can provide this optional attribute
self.short_description = spec.format("$_ $@")
if hasattr(self.pkg, 'short_description'):
self.short_description = self.pkg.short_description
# long description is the docstring with reduced whitespace.
self.long_description = None
if self.spec.package.__doc__:
self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__)
@property
def paths(self):
if self._paths is None:
self._paths = {}
def add_path(path_name, directory):
path = self._paths.setdefault(path_name, [])
path.append(directory)
# Add paths if they exist.
for var, directory in [
('PATH', self.spec.prefix.bin),
('MANPATH', self.spec.prefix.man),
('MANPATH', self.spec.prefix.share_man),
('LIBRARY_PATH', self.spec.prefix.lib),
('LIBRARY_PATH', self.spec.prefix.lib64),
('LD_LIBRARY_PATH', self.spec.prefix.lib),
('LD_LIBRARY_PATH', self.spec.prefix.lib64),
('CPATH', self.spec.prefix.include),
('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')),
('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]:
if os.path.isdir(directory):
add_path(var, directory)
# Add python path unless it's an actual python installation
# TODO: is there a better way to do this?
if self.spec.name != 'python':
site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages"))
if site_packages:
add_path('PYTHONPATH', site_packages[0])
if self.spec.package.extends(spack.spec.Spec('ruby')):
add_path('GEM_PATH', self.spec.prefix)
# short description is just the package + version
# TODO: maybe packages can optionally provide it.
self.short_description = self.spec.format("$_ $@")
# long description is the docstring with reduced whitespace.
if self.spec.package.__doc__:
self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__)
return self._paths
def category(self):
# Anything defined at the package level takes precedence
if hasattr(self.pkg, 'category'):
return self.pkg.category
# Extensions
for extendee in self.pkg.extendees:
return '{extendee} extension'.format(extendee=extendee)
# Not very descriptive fallback
return 'spack installed package'
def write(self):
@@ -156,18 +155,41 @@ def write(self):
if not os.path.exists(module_dir):
mkdirp(module_dir)
# If there are no paths, no need for a dotkit.
if not self.paths:
# Environment modifications guessed by inspecting the
# installation prefix
env = inspect_path(self.spec.prefix)
# Let the extendee modify their extensions before asking for
# package-specific modifications
spack_env = EnvironmentModifications()
for item in self.pkg.extendees:
package = self.spec[item].package
package.setup_dependent_package(self.pkg.module, self.spec)
package.setup_dependent_environment(spack_env, env, self.spec)
# Package-specific environment modifications
self.spec.package.setup_environment(spack_env, env)
# TODO : implement site-specific modifications and filters
if not env:
return
with open(self.file_name, 'w') as f:
self._write(f)
self.write_header(f)
for line in self.process_environment_command(env):
f.write(line)
def _write(self, stream):
"""To be implemented by subclasses."""
def write_header(self, stream):
raise NotImplementedError()
def process_environment_command(self, env):
for command in env:
try:
yield self.formats[type(command)].format(**command.args)
except KeyError:
tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command)))
tty.warn('{context} at {filename}:{lineno}'.format(**command.args))
@property
def file_name(self):
@@ -175,14 +197,12 @@ def file_name(self):
where this module lives."""
raise NotImplementedError()
@property
def use_name(self):
"""Subclasses should implement this to return the name the
module command uses to refer to the package."""
raise NotImplementedError()
def remove(self):
mod_file = self.file_name
if os.path.exists(mod_file):
@@ -193,19 +213,23 @@ class Dotkit(EnvModule):
name = 'dotkit'
path = join_path(spack.share_path, "dotkit")
formats = {
PrependPath: 'dk_alter {name} {value}\n',
SetEnv: 'dk_setenv {name} {value}\n'
}
@property
def file_name(self):
return join_path(Dotkit.path, self.spec.architecture,
'%s.dk' % self.use_name)
return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name)
@property
def use_name(self):
return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version,
self.spec.compiler.name,
self.spec.compiler.version,
self.spec.compiler.version,
self.spec.dag_hash())
def _write(self, dk_file):
def write_header(self, dk_file):
# Category
if self.category:
dk_file.write('#c %s\n' % self.category)
@@ -219,50 +243,41 @@ def _write(self, dk_file):
for line in textwrap.wrap(self.long_description, 72):
dk_file.write("#h %s\n" % line)
# Path alterations
for var, dirs in self.paths.items():
for directory in dirs:
dk_file.write("dk_alter %s %s\n" % (var, directory))
# Let CMake find this package.
dk_file.write("dk_alter CMAKE_PREFIX_PATH %s\n" % self.spec.prefix)
class TclModule(EnvModule):
name = 'tcl'
path = join_path(spack.share_path, "modules")
formats = {
PrependPath: 'prepend-path {name} \"{value}\"\n',
AppendPath: 'append-path {name} \"{value}\"\n',
RemovePath: 'remove-path {name} \"{value}\"\n',
SetEnv: 'setenv {name} \"{value}\"\n',
UnsetEnv: 'unsetenv {name}\n'
}
@property
def file_name(self):
return join_path(TclModule.path, self.spec.architecture, self.use_name)
@property
def use_name(self):
return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version,
self.spec.compiler.name,
self.spec.compiler.version,
self.spec.compiler.version,
self.spec.dag_hash())
def _write(self, m_file):
# TODO: cateogry?
m_file.write('#%Module1.0\n')
def write_header(self, module_file):
# TCL Modulefile header
module_file.write('#%Module1.0\n')
# TODO : category ?
# Short description
if self.short_description:
m_file.write('module-whatis \"%s\"\n\n' % self.short_description)
module_file.write('module-whatis \"%s\"\n\n' % self.short_description)
# Long description
if self.long_description:
m_file.write('proc ModulesHelp { } {\n')
doc = re.sub(r'"', '\"', self.long_description)
m_file.write("puts stderr \"%s\"\n" % doc)
m_file.write('}\n\n')
# Path alterations
for var, dirs in self.paths.items():
for directory in dirs:
m_file.write("prepend-path %s \"%s\"\n" % (var, directory))
m_file.write("prepend-path CMAKE_PREFIX_PATH \"%s\"\n" % self.spec.prefix)
module_file.write('proc ModulesHelp { } {\n')
for line in textwrap.wrap(self.long_description, 72):
module_file.write("puts stderr \"%s\"\n" % line)
module_file.write('}\n\n')

181
lib/spack/spack/package.py
View File

@@ -34,40 +34,34 @@
README.
"""
import os
import errno
import re
import shutil
import time
import itertools
import subprocess
import platform as py_platform
import multiprocessing
from urlparse import urlparse, urljoin
import textwrap
from StringIO import StringIO
import time
import glob
import llnl.util.tty as tty
from llnl.util.tty.log import log_output
from llnl.util.link_tree import LinkTree
from llnl.util.filesystem import *
from llnl.util.lang import *
import spack
import spack.error
import spack.compilers
import spack.mirror
import spack.hooks
import spack.directives
import spack.repository
import spack.build_environment
import spack.compilers
import spack.directives
import spack.error
import spack.fetch_strategy as fs
import spack.hooks
import spack.mirror
import spack.repository
import spack.url
import spack.util.web
import spack.fetch_strategy as fs
from spack.version import *
from StringIO import StringIO
from llnl.util.filesystem import *
from llnl.util.lang import *
from llnl.util.link_tree import LinkTree
from llnl.util.tty.log import log_output
from spack.stage import Stage, ResourceStage, StageComposite
from spack.util.compression import allowed_archive, extension
from spack.util.executable import ProcessError
from spack.util.compression import allowed_archive
from spack.util.environment import dump_environment
from spack.util.executable import ProcessError
from spack.version import *
from urlparse import urlparse
"""Allowed URL schemes for spack packages."""
_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
@@ -341,6 +335,9 @@ def __init__(self, spec):
if '.' in self.name:
self.name = self.name[self.name.rindex('.') + 1:]
# Allow custom staging paths for packages
self.path=None
# Sanity check attributes required by Spack directives.
spack.directives.ensure_dicts(type(self))
@@ -459,7 +456,8 @@ def _make_resource_stage(self, root_stage, fetcher, resource):
resource_stage_folder = self._resource_stage(resource)
resource_mirror = join_path(self.name, os.path.basename(fetcher.url))
stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource,
name=resource_stage_folder, mirror_path=resource_mirror)
name=resource_stage_folder, mirror_path=resource_mirror,
path=self.path)
return stage
def _make_root_stage(self, fetcher):
@@ -469,7 +467,7 @@ def _make_root_stage(self, fetcher):
s = self.spec
stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
# Build the composite stage
stage = Stage(fetcher, mirror_path=mp, name=stage_name)
stage = Stage(fetcher, mirror_path=mp, name=stage_name, path=self.path)
return stage
def _make_stage(self):
@@ -942,6 +940,9 @@ def build_process():
install(env_path, env_install_path)
dump_packages(self.spec, packages_dir)
# Run post install hooks before build stage is removed.
spack.hooks.post_install(self)
# Stop timer.
self._total_time = time.time() - start_time
build_time = self._total_time - self._fetch_time
@@ -970,9 +971,6 @@ def build_process():
# the database, so that we don't need to re-read from file.
spack.installed_db.add(self.spec, self.prefix)
# Once everything else is done, run post install hooks
spack.hooks.post_install(self)
def sanity_check_prefix(self):
"""This function checks whether install succeeded."""
@@ -1018,38 +1016,127 @@ def module(self):
return __import__(self.__class__.__module__,
fromlist=[self.__class__.__name__])
def setup_environment(self, spack_env, run_env):
"""Set up the compile and runtime environemnts for a package.
def setup_dependent_environment(self, module, spec, dependent_spec):
"""Called before the install() method of dependents.
`spack_env` and `run_env` are `EnvironmentModifications`
objects. Package authors can call methods on them to alter
the environment within Spack and at runtime.
Both `spack_env` and `run_env` are applied within the build
process, before this package's `install()` method is called.
Modifications in `run_env` will *also* be added to the
generated environment modules for this package.
Default implementation does nothing, but this can be
overridden by an extendable package to set up the install
environment for its extensions. This is useful if there are
some common steps to installing all extensions for a
certain package.
overridden if the package needs a particular environment.
Some examples:
Examples:
1. Installing python modules generally requires PYTHONPATH to
point to the lib/pythonX.Y/site-packages directory in the
module's install prefix. This could set that variable.
1. Qt extensions need `QTDIR` set.
2. Extensions often need to invoke the 'python' interpreter
from the Python installation being extended. This routine can
put a 'python' Execuable object in the module scope for the
extension package to simplify extension installs.
Args:
spack_env (EnvironmentModifications): list of
modifications to be applied when this package is built
within Spack.
3. A lot of Qt extensions need QTDIR set. This can be used to do that.
run_env (EnvironmentModifications): list of environment
changes to be applied when this package is run outside
of Spack.
"""
pass
def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
"""Set up the environment of packages that depend on this one.
This is similar to `setup_environment`, but it is used to
modify the compile and runtime environments of packages that
*depend* on this one. This gives packages like Python and
others that follow the extension model a way to implement
common environment or compile-time settings for dependencies.
By default, this delegates to self.setup_environment()
Example :
1. Installing python modules generally requires
`PYTHONPATH` to point to the lib/pythonX.Y/site-packages
directory in the module's install prefix. This could
set that variable.
Args:
spack_env (EnvironmentModifications): list of
modifications to be applied when the dependent package
is bulit within Spack.
run_env (EnvironmentModifications): list of environment
changes to be applied when the dependent package is
run outside of Spack.
dependent_spec (Spec): The spec of the dependent package
about to be built. This allows the extendee (self) to
query the dependent's state. Note that *this*
package's spec is available as `self.spec`.
This is useful if there are some common steps to installing
all extensions for a certain package.
"""
self.setup_environment(spack_env, run_env)
def setup_dependent_package(self, module, dependent_spec):
"""Set up Python module-scope variables for dependent packages.
Called before the install() method of dependents.
Default implementation does nothing, but this can be
overridden by an extendable package to set up the module of
its extensions. This is useful if there are some common steps
to installing all extensions for a certain package.
Example :
1. Extensions often need to invoke the `python`
interpreter from the Python installation being
extended. This routine can put a 'python' Executable
object in the module scope for the extension package to
simplify extension installs.
2. MPI compilers could set some variables in the
dependent's scope that point to `mpicc`, `mpicxx`,
etc., allowing them to be called by common names
regardless of which MPI is used.
3. BLAS/LAPACK implementations can set some variables
indicating the path to their libraries, since these
paths differ by BLAS/LAPACK implementation.
Args:
module (module): The Python `module` object of the
dependent package. Packages can use this to set
module-scope variables for the dependent to use.
dependent_spec (Spec): The spec of the dependent package
about to be built. This allows the extendee (self) to
query the dependent's state. Note that *this*
package's spec is available as `self.spec`.
This is useful if there are some common steps to installing
all extensions for a certain package.
"""
pass
def install(self, spec, prefix):
"""Package implementations override this with their own build configuration."""
raise InstallError("Package %s provides no install method!" % self.name)
def do_uninstall(self, force=False):
if not self.installed:
raise InstallError(str(self.spec) + " is not installed.")
@@ -1361,6 +1448,10 @@ def __init__(self, message, long_msg=None):
super(InstallError, self).__init__(message, long_msg)
class ExternalPackageError(InstallError):
"""Raised by install() when a package is only for external use."""
class PackageStillNeededError(InstallError):
"""Raised when package is still needed by another on uninstall."""
def __init__(self, spec, dependents):

7
lib/spack/spack/stage.py
View File

@@ -89,7 +89,7 @@ class Stage(object):
"""
def __init__(self, url_or_fetch_strategy,
name=None, mirror_path=None, keep=False):
name=None, mirror_path=None, keep=False, path=None):
"""Create a stage object.
Parameters:
url_or_fetch_strategy
@@ -135,7 +135,10 @@ def __init__(self, url_or_fetch_strategy,
# Try to construct here a temporary name for the stage directory
# If this is a named stage, then construct a named path.
self.path = join_path(spack.stage_path, self.name)
if path is not None:
self.path = path
else:
self.path = join_path(spack.stage_path, self.name)
# Flag to decide whether to delete the stage folder on exit or not
self.keep = keep

3
lib/spack/spack/test/__init__.py
View File

@@ -66,7 +66,8 @@
'database',
'namespace_trie',
'yaml',
'sbang']
'sbang',
'environment']
def list_tests():

127
lib/spack/spack/test/cc.py
View File

@@ -28,6 +28,8 @@
"""
import os
import unittest
import tempfile
import shutil
from llnl.util.filesystem import *
import spack
@@ -55,13 +57,40 @@ def setUp(self):
self.ld = Executable(join_path(spack.build_env_path, "ld"))
self.cpp = Executable(join_path(spack.build_env_path, "cpp"))
os.environ['SPACK_CC'] = "/bin/mycc"
os.environ['SPACK_PREFIX'] = "/usr"
self.realcc = "/bin/mycc"
self.prefix = "/spack-test-prefix"
os.environ['SPACK_CC'] = self.realcc
os.environ['SPACK_PREFIX'] = self.prefix
os.environ['SPACK_ENV_PATH']="test"
os.environ['SPACK_DEBUG_LOG_DIR'] = "."
os.environ['SPACK_COMPILER_SPEC'] = "gcc@4.4.7"
os.environ['SPACK_SHORT_SPEC'] = "foo@1.2"
# Make some fake dependencies
self.tmp_deps = tempfile.mkdtemp()
self.dep1 = join_path(self.tmp_deps, 'dep1')
self.dep2 = join_path(self.tmp_deps, 'dep2')
self.dep3 = join_path(self.tmp_deps, 'dep3')
self.dep4 = join_path(self.tmp_deps, 'dep4')
mkdirp(join_path(self.dep1, 'include'))
mkdirp(join_path(self.dep1, 'lib'))
mkdirp(join_path(self.dep2, 'lib64'))
mkdirp(join_path(self.dep3, 'include'))
mkdirp(join_path(self.dep3, 'lib64'))
mkdirp(join_path(self.dep4, 'include'))
if 'SPACK_DEPENDENCIES' in os.environ:
del os.environ['SPACK_DEPENDENCIES']
def tearDown(self):
shutil.rmtree(self.tmp_deps, True)
def check_cc(self, command, args, expected):
os.environ['SPACK_TEST_COMMAND'] = command
@@ -92,6 +121,10 @@ def test_cpp_mode(self):
self.check_cpp('dump-mode', [], "cpp")
def test_as_mode(self):
self.check_cc('dump-mode', ['-S'], "as")
def test_ccld_mode(self):
self.check_cc('dump-mode', [], "ccld")
self.check_cc('dump-mode', ['foo.c', '-o', 'foo'], "ccld")
@@ -104,27 +137,85 @@ def test_ld_mode(self):
self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ld")
def test_includes(self):
self.check_cc('dump-includes', test_command,
"\n".join(["/test/include", "/other/include"]))
def test_dep_rpath(self):
"""Ensure RPATHs for root package are added."""
self.check_cc('dump-args', test_command,
self.realcc + ' ' +
'-Wl,-rpath,' + self.prefix + '/lib ' +
'-Wl,-rpath,' + self.prefix + '/lib64 ' +
' '.join(test_command))
def test_libraries(self):
self.check_cc('dump-libraries', test_command,
"\n".join(["/test/lib", "/other/lib"]))
def test_dep_include(self):
"""Ensure a single dependency include directory is added."""
os.environ['SPACK_DEPENDENCIES'] = self.dep4
self.check_cc('dump-args', test_command,
self.realcc + ' ' +
'-Wl,-rpath,' + self.prefix + '/lib ' +
'-Wl,-rpath,' + self.prefix + '/lib64 ' +
'-I' + self.dep4 + '/include ' +
' '.join(test_command))
def test_libs(self):
self.check_cc('dump-libs', test_command,
"\n".join(["lib1", "lib2", "lib3", "lib4"]))
def test_dep_lib(self):
"""Ensure a single dependency RPATH is added."""
os.environ['SPACK_DEPENDENCIES'] = self.dep2
self.check_cc('dump-args', test_command,
self.realcc + ' ' +
'-Wl,-rpath,' + self.prefix + '/lib ' +
'-Wl,-rpath,' + self.prefix + '/lib64 ' +
'-L' + self.dep2 + '/lib64 ' +
'-Wl,-rpath,' + self.dep2 + '/lib64 ' +
' '.join(test_command))
def test_rpaths(self):
self.check_cc('dump-rpaths', test_command,
"\n".join(["/first/rpath", "/second/rpath", "/third/rpath", "/fourth/rpath"]))
def test_all_deps(self):
"""Ensure includes and RPATHs for all deps are added. """
os.environ['SPACK_DEPENDENCIES'] = ':'.join([
self.dep1, self.dep2, self.dep3, self.dep4])
# This is probably more constrained than it needs to be; it
# checks order within prepended args and doesn't strictly have
# to. We could loosen that if it becomes necessary
self.check_cc('dump-args', test_command,
self.realcc + ' ' +
'-Wl,-rpath,' + self.prefix + '/lib ' +
'-Wl,-rpath,' + self.prefix + '/lib64 ' +
'-I' + self.dep4 + '/include ' +
'-L' + self.dep3 + '/lib64 ' +
'-Wl,-rpath,' + self.dep3 + '/lib64 ' +
'-I' + self.dep3 + '/include ' +
'-L' + self.dep2 + '/lib64 ' +
'-Wl,-rpath,' + self.dep2 + '/lib64 ' +
'-L' + self.dep1 + '/lib ' +
'-Wl,-rpath,' + self.dep1 + '/lib ' +
'-I' + self.dep1 + '/include ' +
' '.join(test_command))
def test_other_args(self):
self.check_cc('dump-other-args', test_command,
"\n".join(["arg1", "-Wl,--start-group", "arg2", "arg3", "arg4",
"-Wl,--end-group", "arg5", "arg6"]))
def test_ld_deps(self):
"""Ensure no (extra) -I args or -Wl, are passed in ld mode."""
os.environ['SPACK_DEPENDENCIES'] = ':'.join([
self.dep1, self.dep2, self.dep3, self.dep4])
self.check_ld('dump-args', test_command,
'ld ' +
'-rpath ' + self.prefix + '/lib ' +
'-rpath ' + self.prefix + '/lib64 ' +
'-L' + self.dep3 + '/lib64 ' +
'-rpath ' + self.dep3 + '/lib64 ' +
'-L' + self.dep2 + '/lib64 ' +
'-rpath ' + self.dep2 + '/lib64 ' +
'-L' + self.dep1 + '/lib ' +
'-rpath ' + self.dep1 + '/lib ' +
' '.join(test_command))

16
lib/spack/spack/test/concretize.py
View File

@@ -24,6 +24,7 @@
##############################################################################
import spack
from spack.spec import Spec, CompilerSpec
from spack.version import ver
from spack.concretize import find_spec
from spack.test.mock_packages_test import *
@@ -77,6 +78,14 @@ def test_concretize_variant(self):
self.check_concretize('mpich')
def test_concretize_preferred_version(self):
spec = self.check_concretize('python')
self.assertEqual(spec.versions, ver('2.7.11'))
spec = self.check_concretize('python@3.5.1')
self.assertEqual(spec.versions, ver('3.5.1'))
def test_concretize_with_virtual(self):
self.check_concretize('mpileaks ^mpi')
self.check_concretize('mpileaks ^mpi@:1.1')
@@ -309,3 +318,10 @@ def test_find_spec_none(self):
Spec('d')),
Spec('e'))
self.assertEqual(None, find_spec(s['b'], lambda s: '+foo' in s))
def test_compiler_child(self):
s = Spec('mpileaks%clang ^dyninst%gcc')
s.concretize()
self.assertTrue(s['mpileaks'].satisfies('%clang'))
self.assertTrue(s['dyninst'].satisfies('%gcc'))

1
lib/spack/spack/test/database.py
View File

@@ -26,6 +26,7 @@
These tests check the database is functioning properly,
both in memory and in its file
"""
import os.path
import multiprocessing
import shutil
import tempfile

73
lib/spack/spack/test/environment.py Normal file
View File

@@ -0,0 +1,73 @@
import unittest
import os
from spack.environment import EnvironmentModifications
class EnvironmentTest(unittest.TestCase):
def setUp(self):
os.environ.clear()
os.environ['UNSET_ME'] = 'foo'
os.environ['EMPTY_PATH_LIST'] = ''
os.environ['PATH_LIST'] = '/path/second:/path/third'
os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g'
def test_set(self):
env = EnvironmentModifications()
env.set('A', 'dummy value')
env.set('B', 3)
env.apply_modifications()
self.assertEqual('dummy value', os.environ['A'])
self.assertEqual(str(3), os.environ['B'])
def test_unset(self):
env = EnvironmentModifications()
self.assertEqual('foo', os.environ['UNSET_ME'])
env.unset('UNSET_ME')
env.apply_modifications()
self.assertRaises(KeyError, os.environ.__getitem__, 'UNSET_ME')
def test_set_path(self):
env = EnvironmentModifications()
env.set_path('A', ['foo', 'bar', 'baz'])
env.apply_modifications()
self.assertEqual('foo:bar:baz', os.environ['A'])
def test_path_manipulation(self):
env = EnvironmentModifications()
env.append_path('PATH_LIST', '/path/last')
env.prepend_path('PATH_LIST', '/path/first')
env.append_path('EMPTY_PATH_LIST', '/path/middle')
env.append_path('EMPTY_PATH_LIST', '/path/last')
env.prepend_path('EMPTY_PATH_LIST', '/path/first')
env.append_path('NEWLY_CREATED_PATH_LIST', '/path/middle')
env.append_path('NEWLY_CREATED_PATH_LIST', '/path/last')
env.prepend_path('NEWLY_CREATED_PATH_LIST', '/path/first')
env.remove_path('REMOVE_PATH_LIST', '/remove/this')
env.remove_path('REMOVE_PATH_LIST', '/duplicate/')
env.apply_modifications()
self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST'])
self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST'])
self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST'])
self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST'])
def test_extra_arguments(self):
env = EnvironmentModifications()
env.set('A', 'dummy value', who='Pkg1')
for x in env:
assert 'who' in x.args
env.apply_modifications()
self.assertEqual('dummy value', os.environ['A'])
def test_extend(self):
env = EnvironmentModifications()
env.set('A', 'dummy value')
env.set('B', 3)
copy_construct = EnvironmentModifications(env)
self.assertEqual(len(copy_construct), 2)
for x, y in zip(env, copy_construct):
assert x is y

8
lib/spack/spack/util/environment.py
View File

@@ -59,14 +59,8 @@ def path_put_first(var_name, directories):
path_set(var_name, new_path)
def pop_keys(dictionary, *keys):
for key in keys:
if key in dictionary:
dictionary.pop(key)
def dump_environment(path):
"""Dump the current environment out to a file."""
with open(path, 'w') as env_file:
for key,val in sorted(os.environ.items()):
for key, val in sorted(os.environ.items()):
env_file.write("%s=%s\n" % (key, val))