zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cc: Ensure that user-specified flags take precedence over others (#37376)

Browse Source 
Spack flags supplied by users should supersede flags from package build systems and
other places in Spack.  However, Spack currently adds user-supplied flags to the 
beginning of the compile line, which means that in some cases build system flags will
supersede user-supplied ones.

The right place to add a flag to ensure it has highest precedence for the compiler really
depends on the type of flag.  For example, search paths like `-L` and `-I` are examined
in order, so adding them first is highest precedence.  Compilers take the *last* occurrence
of optimization flags like `-O2`, so those should be placed *after* other such flags.  Shim
libraries with `-l` should go *before* other libraries on the command line, so we want
user-supplied libs to go first, etc.

`lib/spack/env/cc` already knows how to split arguments into categories like `libs_list`,
`rpath_dirs_list`, etc., so we can leverage that functionality to merge user flags into
the arg list correctly.

The general rules for injected flags are:

1. All `-L`, `-I`, `-isystem`, `-l`, and `*-rpath` flags from `spack_flags_*` to appear
   before their regular counterparts.
2. All other flags ordered with the ones from flags after their regular counterparts,
   i.e. `other_flags` before `spack_flags_other_flags`

- [x] Generalize argument categorization into its own function in the `cc` shell script
- [x] Apply the same splitting logic to injected flags and flags from the original compile line.
- [x] Use the resulting flag lists to merge user- and build-system-supplied flags by category.
- [x] Add tests.

Signed-off-by: Andrey Parfenov <andrey.parfenov@intel.com>

Co-authored-by: iermolae <igor.ermolaev@intel.com>
This commit is contained in:
Andrey Parfenov
2023-06-18 23:07:08 +02:00
committed by GitHub
parent 5c6c3b403b
commit 7dc485d288
2 changed files with 261 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
lib/spack/env/cc vendored
View File

@@ -416,30 +416,14 @@ input_command="$*"
# The lists are all bell-separated to be as flexible as possible, as their # The lists are all bell-separated to be as flexible as possible, as their
# contents may come from the command line, from ' '-separated lists, # contents may come from the command line, from ' '-separated lists,
# ':'-separated lists, etc. # ':'-separated lists, etc.
include_dirs_list=""
lib_dirs_list=""
rpath_dirs_list=""
system_include_dirs_list=""
system_lib_dirs_list=""
system_rpath_dirs_list=""
isystem_system_include_dirs_list=""
isystem_include_dirs_list=""
libs_list=""
other_args_list=""
# Global state for keeping track of -Wl,-rpath -Wl,/path
wl_expect_rpath=no
# Same, but for -Xlinker -rpath -Xlinker /path
xlinker_expect_rpath=no
parse_Wl() { parse_Wl() {
while [ $# -ne 0 ]; do while [ $# -ne 0 ]; do
if [ "$wl_expect_rpath" = yes ]; then if [ "$wl_expect_rpath" = yes ]; then
if system_dir "$1"; then if system_dir "$1"; then
append system_rpath_dirs_list "$1" append return_system_rpath_dirs_list "$1"
else else
append rpath_dirs_list "$1" append return_rpath_dirs_list "$1"
fi fi
wl_expect_rpath=no wl_expect_rpath=no
else else
@@ -449,9 +433,9 @@ parse_Wl() {
if [ -z "$arg" ]; then if [ -z "$arg" ]; then
shift; continue shift; continue
elif system_dir "$arg"; then elif system_dir "$arg"; then
append system_rpath_dirs_list "$arg" append return_system_rpath_dirs_list "$arg"
else else
append rpath_dirs_list "$arg" append return_rpath_dirs_list "$arg"
fi fi
;; ;;
--rpath=*) --rpath=*)
@@ -459,9 +443,9 @@ parse_Wl() {
if [ -z "$arg" ]; then if [ -z "$arg" ]; then
shift; continue shift; continue
elif system_dir "$arg"; then elif system_dir "$arg"; then
append system_rpath_dirs_list "$arg" append return_system_rpath_dirs_list "$arg"
else else
append rpath_dirs_list "$arg" append return_rpath_dirs_list "$arg"
fi fi
;; ;;
-rpath|--rpath) -rpath|--rpath)
@@ -475,7 +459,7 @@ parse_Wl() {
return 1 return 1
;; ;;
*) *)
append other_args_list "-Wl,$1" append return_other_args_list "-Wl,$1"
;; ;;
esac esac
fi fi
@@ -483,6 +467,26 @@ parse_Wl() {
done done
} }
categorize_arguments() {
unset IFS
return_other_args_list=""
return_isystem_was_used=""
return_isystem_system_include_dirs_list=""
return_isystem_include_dirs_list=""
return_system_include_dirs_list=""
return_include_dirs_list=""
return_system_lib_dirs_list=""
return_lib_dirs_list=""
return_system_rpath_dirs_list=""
return_rpath_dirs_list=""
# Global state for keeping track of -Wl,-rpath -Wl,/path
wl_expect_rpath=no
# Same, but for -Xlinker -rpath -Xlinker /path
xlinker_expect_rpath=no
while [ $# -ne 0 ]; do while [ $# -ne 0 ]; do
@@ -501,7 +505,7 @@ while [ $# -ne 0 ]; do
eval "\ eval "\
case \"\$1\" in case \"\$1\" in
$SPACK_COMPILER_FLAGS_KEEP) $SPACK_COMPILER_FLAGS_KEEP)
append other_args_list \"\$1\" append return_other_args_list \"\$1\"
shift shift
continue continue
;; ;;
@@ -540,30 +544,30 @@ while [ $# -ne 0 ]; do
case "$1" in case "$1" in
-isystem*) -isystem*)
arg="${1#-isystem}" arg="${1#-isystem}"
isystem_was_used=true return_isystem_was_used=true
if [ -z "$arg" ]; then shift; arg="$1"; fi if [ -z "$arg" ]; then shift; arg="$1"; fi
if system_dir "$arg"; then if system_dir "$arg"; then
append isystem_system_include_dirs_list "$arg" append return_isystem_system_include_dirs_list "$arg"
else else
append isystem_include_dirs_list "$arg" append return_isystem_include_dirs_list "$arg"
fi fi
;; ;;
-I*) -I*)
arg="${1#-I}" arg="${1#-I}"
if [ -z "$arg" ]; then shift; arg="$1"; fi if [ -z "$arg" ]; then shift; arg="$1"; fi
if system_dir "$arg"; then if system_dir "$arg"; then
append system_include_dirs_list "$arg" append return_system_include_dirs_list "$arg"
else else
append include_dirs_list "$arg" append return_include_dirs_list "$arg"
fi fi
;; ;;
-L*) -L*)
arg="${1#-L}" arg="${1#-L}"
if [ -z "$arg" ]; then shift; arg="$1"; fi if [ -z "$arg" ]; then shift; arg="$1"; fi
if system_dir "$arg"; then if system_dir "$arg"; then
append system_lib_dirs_list "$arg" append return_system_lib_dirs_list "$arg"
else else
append lib_dirs_list "$arg" append return_lib_dirs_list "$arg"
fi fi
;; ;;
-l*) -l*)
@@ -579,12 +583,12 @@ while [ $# -ne 0 ]; do
fi fi
arg="${1#-l}" arg="${1#-l}"
if [ -z "$arg" ]; then shift; arg="$1"; fi if [ -z "$arg" ]; then shift; arg="$1"; fi
append other_args_list "-l$arg" append return_other_args_list "-l$arg"
;; ;;
-Wl,*) -Wl,*)
IFS=, IFS=,
if ! parse_Wl ${1#-Wl,}; then if ! parse_Wl ${1#-Wl,}; then
append other_args_list "$1" append return_other_args_list "$1"
fi fi
unset IFS unset IFS
;; ;;
@@ -592,15 +596,15 @@ while [ $# -ne 0 ]; do
shift shift
if [ $# -eq 0 ]; then if [ $# -eq 0 ]; then
# -Xlinker without value: let the compiler error about it. # -Xlinker without value: let the compiler error about it.
append other_args_list -Xlinker append return_other_args_list -Xlinker
xlinker_expect_rpath=no xlinker_expect_rpath=no
break break
elif [ "$xlinker_expect_rpath" = yes ]; then elif [ "$xlinker_expect_rpath" = yes ]; then
# Register the path of -Xlinker -rpath <other args> -Xlinker <path> # Register the path of -Xlinker -rpath <other args> -Xlinker <path>
if system_dir "$1"; then if system_dir "$1"; then
append system_rpath_dirs_list "$1" append return_system_rpath_dirs_list "$1"
else else
append rpath_dirs_list "$1" append return_rpath_dirs_list "$1"
fi fi
xlinker_expect_rpath=no xlinker_expect_rpath=no
else else
@@ -608,17 +612,17 @@ while [ $# -ne 0 ]; do
-rpath=*) -rpath=*)
arg="${1#-rpath=}" arg="${1#-rpath=}"
if system_dir "$arg"; then if system_dir "$arg"; then
append system_rpath_dirs_list "$arg" append return_system_rpath_dirs_list "$arg"
else else
append rpath_dirs_list "$arg" append return_rpath_dirs_list "$arg"
fi fi
;; ;;
--rpath=*) --rpath=*)
arg="${1#--rpath=}" arg="${1#--rpath=}"
if system_dir "$arg"; then if system_dir "$arg"; then
append system_rpath_dirs_list "$arg" append return_system_rpath_dirs_list "$arg"
else else
append rpath_dirs_list "$arg" append return_rpath_dirs_list "$arg"
fi fi
;; ;;
-rpath|--rpath) -rpath|--rpath)
@@ -627,8 +631,8 @@ while [ $# -ne 0 ]; do
"$dtags_to_strip") "$dtags_to_strip")
;; ;;
*) *)
append other_args_list -Xlinker append return_other_args_list -Xlinker
append other_args_list "$1" append return_other_args_list "$1"
;; ;;
esac esac
fi fi
@@ -636,7 +640,7 @@ while [ $# -ne 0 ]; do
"$dtags_to_strip") "$dtags_to_strip")
;; ;;
*) *)
append other_args_list "$1" append return_other_args_list "$1"
;; ;;
esac esac
shift shift
@@ -646,14 +650,27 @@ done
# `-Xlinker -rpath` again and let the compiler or linker handle the error during arg # `-Xlinker -rpath` again and let the compiler or linker handle the error during arg
# parsing. # parsing.
if [ "$xlinker_expect_rpath" = yes ]; then if [ "$xlinker_expect_rpath" = yes ]; then
append other_args_list -Xlinker append return_other_args_list -Xlinker
append other_args_list -rpath append return_other_args_list -rpath
fi fi
# Same, but for -Wl flags. # Same, but for -Wl flags.
if [ "$wl_expect_rpath" = yes ]; then if [ "$wl_expect_rpath" = yes ]; then
append other_args_list -Wl,-rpath append return_other_args_list -Wl,-rpath
fi fi
}
categorize_arguments "$@"
include_dirs_list="$return_include_dirs_list"
lib_dirs_list="$return_lib_dirs_list"
rpath_dirs_list="$return_rpath_dirs_list"
system_include_dirs_list="$return_system_include_dirs_list"
system_lib_dirs_list="$return_system_lib_dirs_list"
system_rpath_dirs_list="$return_system_rpath_dirs_list"
isystem_was_used="$return_isystem_was_used"
isystem_system_include_dirs_list="$return_isystem_system_include_dirs_list"
isystem_include_dirs_list="$return_isystem_include_dirs_list"
other_args_list="$return_other_args_list"
# #
# Add flags from Spack's cppflags, cflags, cxxflags, fcflags, fflags, and # Add flags from Spack's cppflags, cflags, cxxflags, fcflags, fflags, and
@@ -673,12 +690,14 @@ elif [ "$SPACK_ADD_DEBUG_FLAGS" = "custom" ]; then
extend flags_list SPACK_DEBUG_FLAGS extend flags_list SPACK_DEBUG_FLAGS
fi fi
spack_flags_list=""
# Fortran flags come before CPPFLAGS # Fortran flags come before CPPFLAGS
case "$mode" in case "$mode" in
cc|ccld) cc|ccld)
case $lang_flags in case $lang_flags in
F) F)
extend flags_list SPACK_FFLAGS extend spack_flags_list SPACK_FFLAGS
;; ;;
esac esac
;; ;;
@@ -687,7 +706,7 @@ esac
# C preprocessor flags come before any C/CXX flags # C preprocessor flags come before any C/CXX flags
case "$mode" in case "$mode" in
cpp|as|cc|ccld) cpp|as|cc|ccld)
extend flags_list SPACK_CPPFLAGS extend spack_flags_list SPACK_CPPFLAGS
;; ;;
esac esac
@@ -697,10 +716,10 @@ case "$mode" in
cc|ccld) cc|ccld)
case $lang_flags in case $lang_flags in
C) C)
extend flags_list SPACK_CFLAGS extend spack_flags_list SPACK_CFLAGS
;; ;;
CXX) CXX)
extend flags_list SPACK_CXXFLAGS extend spack_flags_list SPACK_CXXFLAGS
;; ;;
esac esac
@@ -712,10 +731,25 @@ esac
# Linker flags # Linker flags
case "$mode" in case "$mode" in
ld|ccld) ld|ccld)
extend flags_list SPACK_LDFLAGS extend spack_flags_list SPACK_LDFLAGS
;; ;;
esac esac
IFS="$lsep"
categorize_arguments $spack_flags_list
unset IFS
spack_flags_include_dirs_list="$return_include_dirs_list"
spack_flags_lib_dirs_list="$return_lib_dirs_list"
spack_flags_rpath_dirs_list="$return_rpath_dirs_list"
spack_flags_system_include_dirs_list="$return_system_include_dirs_list"
spack_flags_system_lib_dirs_list="$return_system_lib_dirs_list"
spack_flags_system_rpath_dirs_list="$return_system_rpath_dirs_list"
spack_flags_isystem_was_used="$return_isystem_was_used"
spack_flags_isystem_system_include_dirs_list="$return_isystem_system_include_dirs_list"
spack_flags_isystem_include_dirs_list="$return_isystem_include_dirs_list"
spack_flags_other_args_list="$return_other_args_list"
# On macOS insert headerpad_max_install_names linker flag # On macOS insert headerpad_max_install_names linker flag
if [ "$mode" = ld ] || [ "$mode" = ccld ]; then if [ "$mode" = ld ] || [ "$mode" = ccld ]; then
if [ "${SPACK_SHORT_SPEC#*darwin}" != "${SPACK_SHORT_SPEC}" ]; then if [ "${SPACK_SHORT_SPEC#*darwin}" != "${SPACK_SHORT_SPEC}" ]; then
@@ -741,6 +775,8 @@ if [ "$mode" = ccld ] || [ "$mode" = ld ]; then
extend lib_dirs_list SPACK_LINK_DIRS extend lib_dirs_list SPACK_LINK_DIRS
fi fi
libs_list=""
# add RPATHs if we're in in any linking mode # add RPATHs if we're in in any linking mode
case "$mode" in case "$mode" in
ld|ccld) ld|ccld)
@@ -769,12 +805,16 @@ args_list="$flags_list"
# Insert include directories just prior to any system include directories # Insert include directories just prior to any system include directories
# NOTE: adding ${lsep} to the prefix here turns every added element into two # NOTE: adding ${lsep} to the prefix here turns every added element into two
extend args_list spack_flags_include_dirs_list "-I"
extend args_list include_dirs_list "-I" extend args_list include_dirs_list "-I"
extend args_list spack_flags_isystem_include_dirs_list "-isystem${lsep}"
extend args_list isystem_include_dirs_list "-isystem${lsep}" extend args_list isystem_include_dirs_list "-isystem${lsep}"
case "$mode" in case "$mode" in
cpp|cc|as|ccld) cpp|cc|as|ccld)
if [ "$isystem_was_used" = "true" ]; then if [ "$spack_flags_isystem_was_used" = "true" ]; then
extend args_list SPACK_INCLUDE_DIRS "-isystem${lsep}"
elif [ "$isystem_was_used" = "true" ]; then
extend args_list SPACK_INCLUDE_DIRS "-isystem${lsep}" extend args_list SPACK_INCLUDE_DIRS "-isystem${lsep}"
else else
extend args_list SPACK_INCLUDE_DIRS "-I" extend args_list SPACK_INCLUDE_DIRS "-I"
@@ -782,11 +822,15 @@ case "$mode" in
;; ;;
esac esac
extend args_list spack_flags_system_include_dirs_list -I
extend args_list system_include_dirs_list -I extend args_list system_include_dirs_list -I
extend args_list spack_flags_isystem_system_include_dirs_list "-isystem${lsep}"
extend args_list isystem_system_include_dirs_list "-isystem${lsep}" extend args_list isystem_system_include_dirs_list "-isystem${lsep}"
# Library search paths # Library search paths
extend args_list spack_flags_lib_dirs_list "-L"
extend args_list lib_dirs_list "-L" extend args_list lib_dirs_list "-L"
extend args_list spack_flags_system_lib_dirs_list "-L"
extend args_list system_lib_dirs_list "-L" extend args_list system_lib_dirs_list "-L"
# RPATHs arguments # RPATHs arguments
@@ -795,20 +839,25 @@ case "$mode" in
if [ -n "$dtags_to_add" ] ; then if [ -n "$dtags_to_add" ] ; then
append args_list "$linker_arg$dtags_to_add" append args_list "$linker_arg$dtags_to_add"
fi fi
extend args_list spack_flags_rpath_dirs_list "$rpath"
extend args_list rpath_dirs_list "$rpath" extend args_list rpath_dirs_list "$rpath"
extend args_list spack_flags_system_rpath_dirs_list "$rpath"
extend args_list system_rpath_dirs_list "$rpath" extend args_list system_rpath_dirs_list "$rpath"
;; ;;
ld) ld)
if [ -n "$dtags_to_add" ] ; then if [ -n "$dtags_to_add" ] ; then
append args_list "$dtags_to_add" append args_list "$dtags_to_add"
fi fi
extend args_list spack_flags_rpath_dirs_list "-rpath${lsep}"
extend args_list rpath_dirs_list "-rpath${lsep}" extend args_list rpath_dirs_list "-rpath${lsep}"
extend args_list spack_flags_system_rpath_dirs_list "-rpath${lsep}"
extend args_list system_rpath_dirs_list "-rpath${lsep}" extend args_list system_rpath_dirs_list "-rpath${lsep}"
;; ;;
esac esac
# Other arguments from the input command # Other arguments from the input command
extend args_list other_args_list extend args_list other_args_list
extend args_list spack_flags_other_args_list
# Inject SPACK_LDLIBS, if supplied # Inject SPACK_LDLIBS, if supplied
extend args_list libs_list "-l" extend args_list libs_list "-l"
@@ -864,3 +913,4 @@ fi
# Execute the full command, preserving spaces with IFS set # Execute the full command, preserving spaces with IFS set
# to the alarm bell separator. # to the alarm bell separator.
IFS="$lsep"; exec $full_command_list IFS="$lsep"; exec $full_command_list

33
lib/spack/spack/test/cc.py
View File

@@ -173,7 +173,7 @@ def wrapper_environment(working_env):
SPACK_DTAGS_TO_ADD="--disable-new-dtags", SPACK_DTAGS_TO_ADD="--disable-new-dtags",
SPACK_DTAGS_TO_STRIP="--enable-new-dtags", SPACK_DTAGS_TO_STRIP="--enable-new-dtags",
SPACK_COMPILER_FLAGS_KEEP="", SPACK_COMPILER_FLAGS_KEEP="",
SPACK_COMPILER_FLAGS_REPLACE="-Werror*", SPACK_COMPILER_FLAGS_REPLACE="-Werror*|",
): ):
yield yield
@@ -278,8 +278,8 @@ def test_ld_flags(wrapper_environment, wrapper_flags):
ld, ld,
test_args, test_args,
["ld"] ["ld"]
+ spack_ldflags
+ test_include_paths + test_include_paths
+ [spack_ldflags[i] + spack_ldflags[i + 1] for i in range(0, len(spack_ldflags), 2)]
+ test_library_paths + test_library_paths
+ ["--disable-new-dtags"] + ["--disable-new-dtags"]
+ test_rpaths + test_rpaths
@@ -293,10 +293,10 @@ def test_cpp_flags(wrapper_environment, wrapper_flags):
cpp, cpp,
test_args, test_args,
["cpp"] ["cpp"]
+ spack_cppflags
+ test_include_paths + test_include_paths
+ test_library_paths + test_library_paths
+ test_args_without_paths, + test_args_without_paths
+ spack_cppflags,
) )
@@ -306,10 +306,14 @@ def test_cc_flags(wrapper_environment, wrapper_flags):
test_args, test_args,
[real_cc] [real_cc]
+ target_args + target_args
+ test_include_paths
+ [spack_ldflags[i] + spack_ldflags[i + 1] for i in range(0, len(spack_ldflags), 2)]
+ test_library_paths
+ ["-Wl,--disable-new-dtags"]
+ test_wl_rpaths
+ test_args_without_paths
+ spack_cppflags + spack_cppflags
+ spack_cflags + spack_cflags
+ spack_ldflags
+ common_compile_args
+ spack_ldlibs, + spack_ldlibs,
) )
@@ -320,10 +324,13 @@ def test_cxx_flags(wrapper_environment, wrapper_flags):
test_args, test_args,
[real_cc] [real_cc]
+ target_args + target_args
+ test_include_paths
+ [spack_ldflags[i] + spack_ldflags[i + 1] for i in range(0, len(spack_ldflags), 2)]
+ test_library_paths
+ ["-Wl,--disable-new-dtags"]
+ test_wl_rpaths
+ test_args_without_paths
+ spack_cppflags + spack_cppflags
+ spack_cxxflags
+ spack_ldflags
+ common_compile_args
+ spack_ldlibs, + spack_ldlibs,
) )
@@ -334,10 +341,14 @@ def test_fc_flags(wrapper_environment, wrapper_flags):
test_args, test_args,
[real_cc] [real_cc]
+ target_args + target_args
+ test_include_paths
+ [spack_ldflags[i] + spack_ldflags[i + 1] for i in range(0, len(spack_ldflags), 2)]
+ test_library_paths
+ ["-Wl,--disable-new-dtags"]
+ test_wl_rpaths
+ test_args_without_paths
+ spack_fflags + spack_fflags
+ spack_cppflags + spack_cppflags
+ spack_ldflags
+ common_compile_args
+ spack_ldlibs, + spack_ldlibs,
) )