Fix for SPACK-11: Spack compiler wrapper is now in bash.
- Startup is much faster - Added test for compiler wrapper parsing. - Removed old compilation module that had to be imported by old cc. - Removed cc from python version checks now that it's bash.
This commit is contained in:
Todd Gamblin
420
lib/spack/env/cc
vendored
420
lib/spack/env/cc
vendored
@@ -1,140 +1,326 @@
|
||||
#!/usr/bin/env python
|
||||
import sys
|
||||
if not sys.version_info[:2] >= (2,6):
|
||||
sys.exit("Spack requires Python 2.6. Version was %s." % sys.version_info)
|
||||
#!/bin/bash
|
||||
##############################################################################
|
||||
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
|
||||
# Produced at the Lawrence Livermore National Laboratory.
|
||||
#
|
||||
# This file is part of Spack.
|
||||
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
|
||||
# LLNL-CODE-647188
|
||||
#
|
||||
# For details, see https://scalability-llnl.github.io/spack
|
||||
# Please also see the LICENSE file for our notice and the LGPL.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License (as published by
|
||||
# the Free Software Foundation) version 2.1 dated February 1999.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
|
||||
# conditions of the GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
##############################################################################
|
||||
#
|
||||
# Spack compiler wrapper script.
|
||||
#
|
||||
# Compiler commands go through this compiler wrapper in Spack builds.
|
||||
# The compiler wrapper is a thin layer around the standard compilers.
|
||||
# It enables several key pieces of functionality:
|
||||
#
|
||||
# 1. It allows Spack to swap compilers into and out of builds easily.
|
||||
# 2. It adds several options to the compile line so that spack
|
||||
# packages can find their dependencies at build time and run time:
|
||||
# -I arguments for dependency /include directories.
|
||||
# -L arguments for dependency /lib directories.
|
||||
# -Wl,-rpath arguments for dependency /lib directories.
|
||||
#
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from contextlib import closing
|
||||
# This is the list of environment variables that need to be set before
|
||||
# the script runs. They are set by routines in spack.build_environment
|
||||
# as part of spack.package.Package.do_install().
|
||||
parameters="
|
||||
SPACK_PREFIX
|
||||
SPACK_ENV_PATH
|
||||
SPACK_DEBUG_LOG_DIR
|
||||
SPACK_COMPILER_SPEC
|
||||
SPACK_SHORT_SPEC"
|
||||
|
||||
# Import spack parameters through the build environment.
|
||||
spack_lib = os.environ.get("SPACK_LIB")
|
||||
if not spack_lib:
|
||||
print "Spack compiler must be run from spack!"
|
||||
sys.exit(1)
|
||||
# The compiler input variables are checked for sanity later:
|
||||
# SPACK_CC, SPACK_CXX, SPACK_F77, SPACK_FC
|
||||
# Debug flag is optional; set to true for debug logging:
|
||||
# SPACK_DEBUG
|
||||
# Test command is used to unit test the compiler script.
|
||||
# SPACK_TEST_COMMAND
|
||||
# Dependencies can be empty for pkgs with no deps:
|
||||
# SPACK_DEPENDENCIES
|
||||
|
||||
# Grab a minimal set of spack packages
|
||||
sys.path.append(spack_lib)
|
||||
from spack.compilation import *
|
||||
from external import argparse
|
||||
import llnl.util.tty as tty
|
||||
# die()
|
||||
# Prints a message and exits with error 1.
|
||||
function die {
|
||||
echo "$@"
|
||||
exit 1
|
||||
}
|
||||
|
||||
spack_prefix = get_env_var("SPACK_PREFIX")
|
||||
spack_debug = get_env_flag("SPACK_DEBUG")
|
||||
spack_deps = get_path("SPACK_DEPENDENCIES")
|
||||
spack_env_path = get_path("SPACK_ENV_PATH")
|
||||
spack_debug_log_dir = get_env_var("SPACK_DEBUG_LOG_DIR")
|
||||
spack_spec = get_env_var("SPACK_SPEC")
|
||||
for param in $parameters; do
|
||||
if [ -z "${!param}" ]; then
|
||||
die "Spack compiler must be run from spack! Input $param was missing!"
|
||||
fi
|
||||
done
|
||||
|
||||
compiler_spec = get_env_var("SPACK_COMPILER_SPEC")
|
||||
spack_cc = get_env_var("SPACK_CC", required=False)
|
||||
spack_cxx = get_env_var("SPACK_CXX", required=False)
|
||||
spack_f77 = get_env_var("SPACK_F77", required=False)
|
||||
spack_fc = get_env_var("SPACK_FC", required=False)
|
||||
#
|
||||
# Figure out the type of compiler, the language, and the mode so that
|
||||
# the compiler script knows what to do.
|
||||
#
|
||||
# Possible languages are C, C++, Fortran 77, and Fortran 90.
|
||||
# 'command' is set based on the input command to $SPACK_[CC|CXX|F77|F90]
|
||||
#
|
||||
# 'mode' is set to one of:
|
||||
# cc compile
|
||||
# ld link
|
||||
# ccld compile & link
|
||||
# cpp preprocessor
|
||||
# vcheck version check
|
||||
#
|
||||
command=$(basename "$0")
|
||||
case "$command" in
|
||||
cc|gcc|c89|c99|clang)
|
||||
command="$SPACK_CC"
|
||||
language="C"
|
||||
;;
|
||||
c++|CC|g++|clang++)
|
||||
command="$SPACK_CXX"
|
||||
language="C++"
|
||||
;;
|
||||
f77)
|
||||
command="$SPACK_F77"
|
||||
language="Fortran 77"
|
||||
;;
|
||||
fc|f90|f95)
|
||||
command="$SPACK_FC"
|
||||
language="Fortran 90"
|
||||
;;
|
||||
cpp)
|
||||
mode=cpp
|
||||
;;
|
||||
ld)
|
||||
mode=ld
|
||||
;;
|
||||
*)
|
||||
die "Unkown compiler: $command"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Figure out what type of operation we're doing
|
||||
command = os.path.basename(sys.argv[0])
|
||||
# Finish setting up the mode.
|
||||
if [ -z "$mode" ]; then
|
||||
mode=ccld
|
||||
for arg in "$@"; do
|
||||
if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then
|
||||
mode=vcheck
|
||||
break
|
||||
elif [ "$arg" = -E ]; then
|
||||
mode=cpp
|
||||
break
|
||||
elif [ "$arg" = -c ]; then
|
||||
mode=cc
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
cpp, cc, ccld, ld, version_check = range(5)
|
||||
# Dump the version and exist if we're in testing mode.
|
||||
if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then
|
||||
echo "$mode"
|
||||
exit
|
||||
fi
|
||||
|
||||
if command == 'cpp':
|
||||
mode = cpp
|
||||
elif command == 'ld':
|
||||
mode = ld
|
||||
elif '-E' in sys.argv:
|
||||
mode = cpp
|
||||
elif '-c' in sys.argv:
|
||||
mode = cc
|
||||
else:
|
||||
mode = ccld
|
||||
# Check that at least one of the real commands was actually selected,
|
||||
# otherwise we don't know what to execute.
|
||||
if [ -z "$command" ]; then
|
||||
die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs."
|
||||
fi
|
||||
|
||||
# Save original command for debug logging
|
||||
input_command="$@"
|
||||
|
||||
if command in ('cc', 'gcc', 'c89', 'c99', 'clang'):
|
||||
command = spack_cc
|
||||
language = "C"
|
||||
elif command in ('c++', 'CC', 'g++', 'clang++'):
|
||||
command = spack_cxx
|
||||
language = "C++"
|
||||
elif command in ('f77'):
|
||||
command = spack_f77
|
||||
language = "Fortran 77"
|
||||
elif command in ('fc', 'f90', 'f95'):
|
||||
command = spack_fc
|
||||
language = "Fortran 90"
|
||||
elif command in ('ld', 'cpp'):
|
||||
pass # leave it the same. TODO: what's the right thing?
|
||||
else:
|
||||
raise Exception("Unknown compiler: %s" % command)
|
||||
#
|
||||
# Now do real parsing of the command line args, trying hard to keep
|
||||
# non-rpath linker arguments in the proper order w.r.t. other command
|
||||
# line arguments. This is important for things like groups.
|
||||
#
|
||||
includes=()
|
||||
libraries=()
|
||||
libs=()
|
||||
rpaths=()
|
||||
other_args=()
|
||||
|
||||
if command is None:
|
||||
print "ERROR: Compiler '%s' does not support compiling %s programs." % (
|
||||
compiler_spec, language)
|
||||
sys.exit(1)
|
||||
while [ -n "$1" ]; do
|
||||
case "$1" in
|
||||
-I*)
|
||||
arg="${1#-I}"
|
||||
if [ -z "$arg" ]; then shift; arg="$1"; fi
|
||||
includes+=("$arg")
|
||||
;;
|
||||
-L*)
|
||||
arg="${1#-L}"
|
||||
if [ -z "$arg" ]; then shift; arg="$1"; fi
|
||||
libraries+=("$arg")
|
||||
;;
|
||||
-l*)
|
||||
arg="${1#-l}"
|
||||
if [ -z "$arg" ]; then shift; arg="$1"; fi
|
||||
libs+=("$arg")
|
||||
;;
|
||||
-Wl,*)
|
||||
arg="${1#-Wl,}"
|
||||
if [ -z "$arg" ]; then shift; arg="$1"; fi
|
||||
if [[ "$arg" = -rpath=* ]]; then
|
||||
rpaths+=("${arg#-rpath=}")
|
||||
elif [[ "$arg" = -rpath ]]; then
|
||||
shift; arg="$1"
|
||||
if [[ "$arg" != -Wl,* ]]; then
|
||||
die "-Wl,-rpath was not followed by -Wl,*"
|
||||
fi
|
||||
rpaths+=("${arg#-Wl,}")
|
||||
else
|
||||
other_args+=("-Wl,$arg")
|
||||
fi
|
||||
;;
|
||||
-Xlinker,*)
|
||||
arg="${1#-Xlinker,}"
|
||||
if [ -z "$arg" ]; then shift; arg="$1"; fi
|
||||
if [[ "$arg" = -rpath=* ]]; then
|
||||
rpaths+=("${arg#-rpath=}")
|
||||
elif [[ "$arg" = -rpath ]]; then
|
||||
shift; arg="$1"
|
||||
if [[ "$arg" != -Xlinker,* ]]; then
|
||||
die "-Xlinker,-rpath was not followed by -Xlinker,*"
|
||||
fi
|
||||
rpaths+=("${arg#-Xlinker,}")
|
||||
else
|
||||
other_args+=("-Xlinker,$arg")
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
other_args+=("$1")
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
version_args = ['-V', '-v', '--version', '-dumpversion']
|
||||
if any(arg in sys.argv for arg in version_args):
|
||||
mode = version_check
|
||||
# Dump parsed values for unit testing if asked for
|
||||
if [ -n "$SPACK_TEST_COMMAND" ]; then
|
||||
IFS=$'\n'
|
||||
case "$SPACK_TEST_COMMAND" in
|
||||
dump-includes) echo "${includes[*]}";;
|
||||
dump-libraries) echo "${libraries[*]}";;
|
||||
dump-libs) echo "${libs[*]}";;
|
||||
dump-rpaths) echo "${rpaths[*]}";;
|
||||
dump-other-args) echo "${other_args[*]}";;
|
||||
dump-all)
|
||||
echo "INCLUDES:"
|
||||
echo "${includes[*]}"
|
||||
echo
|
||||
echo "LIBRARIES:"
|
||||
echo "${libraries[*]}"
|
||||
echo
|
||||
echo "LIBS:"
|
||||
echo "${libs[*]}"
|
||||
echo
|
||||
echo "RPATHS:"
|
||||
echo "${rpaths[*]}"
|
||||
echo
|
||||
echo "ARGS:"
|
||||
echo "${other_args[*]}"
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: Unknown test command"
|
||||
exit 1 ;;
|
||||
esac
|
||||
exit
|
||||
fi
|
||||
|
||||
# Parse out the includes, libs, etc. so we can adjust them if need be.
|
||||
parser = argparse.ArgumentParser(add_help=False)
|
||||
parser.add_argument("-I", action='append', default=[], dest='include_path')
|
||||
parser.add_argument("-L", action='append', default=[], dest='lib_path')
|
||||
parser.add_argument("-l", action='append', default=[], dest='libs')
|
||||
# Read spack dependencies from the path environment variable
|
||||
IFS=':' read -ra deps <<< "$SPACK_DEPENDENCIES"
|
||||
for dep in "${deps[@]}"; do
|
||||
if [ -d "$dep/include" ]; then
|
||||
includes+=("$dep/include")
|
||||
fi
|
||||
|
||||
options, other_args = parser.parse_known_args()
|
||||
rpaths, other_args = parse_rpaths(other_args)
|
||||
if [ -d "$dep/lib" ]; then
|
||||
libraries+=("$dep/lib")
|
||||
rpaths+=("$dep/lib")
|
||||
fi
|
||||
|
||||
# Add dependencies' include and lib paths to our compiler flags.
|
||||
def add_if_dir(path_list, directory, index=None):
|
||||
if os.path.isdir(directory):
|
||||
if index is None:
|
||||
path_list.append(directory)
|
||||
else:
|
||||
path_list.insert(index, directory)
|
||||
if [ -d "$dep/lib64" ]; then
|
||||
libraries+=("$dep/lib64")
|
||||
rpaths+=("$dep/lib64")
|
||||
fi
|
||||
done
|
||||
|
||||
for dep_dir in spack_deps:
|
||||
add_if_dir(options.include_path, os.path.join(dep_dir, "include"))
|
||||
add_if_dir(options.lib_path, os.path.join(dep_dir, "lib"))
|
||||
add_if_dir(options.lib_path, os.path.join(dep_dir, "lib64"))
|
||||
# Include all -L's and prefix/whatever dirs in rpath
|
||||
for dir in "${libraries[@]}"; do
|
||||
[ "$dir" != "." ] && rpaths+=("$dir")
|
||||
done
|
||||
rpaths+=("$SPACK_PREFIX/lib")
|
||||
rpaths+=("$SPACK_PREFIX/lib64")
|
||||
|
||||
# Add our modified arguments to it.
|
||||
arguments = ['-I%s' % path for path in options.include_path]
|
||||
arguments += other_args
|
||||
arguments += ['-L%s' % path for path in options.lib_path]
|
||||
arguments += ['-l%s' % path for path in options.libs]
|
||||
# Put the arguments together
|
||||
args=()
|
||||
for dir in "${includes[@]}"; do args+=("-I$dir"); done
|
||||
args+=("${other_args[@]}")
|
||||
for dir in "${libraries[@]}"; do args+=("-L$dir"); done
|
||||
for lib in "${libs[@]}"; do args+=("-l$lib"); done
|
||||
|
||||
# Add rpaths to install dir and its dependencies. We add both lib and lib64
|
||||
# here because we don't know which will be created.
|
||||
rpaths.extend(options.lib_path)
|
||||
rpaths.append('%s/lib' % spack_prefix)
|
||||
rpaths.append('%s/lib64' % spack_prefix)
|
||||
if mode == ccld:
|
||||
arguments += ['-Wl,-rpath,%s' % p for p in rpaths]
|
||||
elif mode == ld:
|
||||
pairs = [('-rpath', '%s' % p) for p in rpaths]
|
||||
arguments += [item for sublist in pairs for item in sublist]
|
||||
if [ "$mode" = ccld ]; then
|
||||
for dir in "${rpaths[@]}"; do args+=("-Wl,-rpath=$dir"); done
|
||||
elif [ "$mode" = ld ]; then
|
||||
for dir in "${rpaths[@]}"; do args+=("-rpath=$dir"); done
|
||||
fi
|
||||
|
||||
# Unset some pesky environment variables
|
||||
for var in ["LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH"]:
|
||||
if var in os.environ:
|
||||
os.environ.pop(var)
|
||||
#
|
||||
# Unset pesky environment variables that could affect build sanity.
|
||||
#
|
||||
unset LD_LIBRARY_PATH
|
||||
unset LD_RUN_PATH
|
||||
unset DYLD_LIBRARY_PATH
|
||||
|
||||
# Ensure that the delegated command doesn't just call this script again.
|
||||
remove_paths = ['.'] + spack_env_path
|
||||
path = [p for p in get_path("PATH") if p not in remove_paths]
|
||||
os.environ["PATH"] = ":".join(path)
|
||||
#
|
||||
# Filter '.' and Spack environment directories out of PATH so that
|
||||
# this script doesn't just call itself
|
||||
#
|
||||
IFS=':' read -ra env_path <<< "$PATH"
|
||||
IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH"
|
||||
spack_env_dirs+=(".")
|
||||
PATH=""
|
||||
for dir in "${env_path[@]}"; do
|
||||
remove=""
|
||||
for rm_dir in "${spack_env_dirs[@]}"; do
|
||||
if [ "$dir" = "$rm_dir" ]; then remove=True; fi
|
||||
done
|
||||
if [ -z "$remove" ]; then
|
||||
if [ -z "$PATH" ]; then
|
||||
PATH="$dir"
|
||||
else
|
||||
PATH="$PATH:$dir"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
export PATH
|
||||
|
||||
full_command = [command] + arguments
|
||||
full_command=("$command")
|
||||
full_command+=("${args[@]}")
|
||||
|
||||
if spack_debug:
|
||||
input_log = os.path.join(spack_debug_log_dir, 'spack-cc-%s.in.log' % spack_spec)
|
||||
output_log = os.path.join(spack_debug_log_dir, 'spack-cc-%s.out.log' % spack_spec)
|
||||
with closing(open(input_log, 'a')) as log:
|
||||
args = [os.path.basename(sys.argv[0])] + sys.argv[1:]
|
||||
log.write("%s\n" % " ".join(arg.replace(' ', r'\ ') for arg in args))
|
||||
with closing(open(output_log, 'a')) as log:
|
||||
log.write("%s\n" % " ".join(full_command))
|
||||
#
|
||||
# Write the input and output commands to debug logs if it's asked for.
|
||||
#
|
||||
if [ "$SPACK_DEBUG" = "TRUE" ]; then
|
||||
input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.in.log"
|
||||
output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.out.log"
|
||||
echo "$input_command" >> $input_log
|
||||
echo "$mode ${full_command[@]}" >> $output_log
|
||||
fi
|
||||
|
||||
rcode = subprocess.call(full_command)
|
||||
sys.exit(rcode)
|
||||
exec "${full_command[@]}"
|
||||
|
||||
Reference in New Issue
Block a user