zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

shell support: spack load no longer needs modules (#14062)

Browse Source 
Previously the `spack load` command was a wrapper around `module load`. This required some bootstrapping of modules to make `spack load` work properly.

With this PR, the `spack` shell function handles the environment modifications necessary to add packages to your user environment. This removes the dependence on environment modules or lmod and removes the requirement to bootstrap spack (beyond using the setup-env scripts).

Included in this PR is support for MacOS when using Apple's System Integrity Protection (SIP), which is enabled by default in modern MacOS versions. SIP clears the `LD_LIBRARY_PATH` and `DYLD_LIBRARY_PATH` variables on process startup for executables that live in `/usr` (but not '/usr/local', `/System`, `/bin`, and `/sbin` among other system locations. Spack cannot know the `LD_LIBRARY_PATH` of the calling process when executed using `/bin/sh` and `/usr/bin/python`. The `spack` shell function now manually forwards these two variables, if they are present, as `SPACK_<VAR>` and recovers those values on startup.

- [x] spack load/unload no longer delegate to modules
- [x] refactor user_environment modification calculations
- [x] update documentation for spack load/unload

Co-authored-by: Todd Gamblin <tgamblin@llnl.gov>
This commit is contained in:
Greg Becker
2020-01-22 22:36:02 -08:00
committed by Todd Gamblin
parent 5053dfa259
commit c9e01ff9d7
17 changed files with 516 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
share/spack/csh/spack.csh
View File

@@ -27,6 +27,16 @@
# avoids the need to come up with a user-friendly naming scheme for
# spack module files.
########################################################################
# Store LD_LIBRARY_PATH variables from spack shell function
# This is necessary because MacOS System Integrity Protection clears
# (DY?)LD_LIBRARY_PATH variables on process start.
if ( ${?LD_LIBRARY_PATH} ) then
setenv SPACK_LD_LIBRARY_PATH $LD_LIBRARY_PATH
endif
if ( ${?DYLD_LIBRARY_PATH} ) then
setenv SPACK_DYLD_LIBRARY_PATH $DYLD_LIBRARY_PATH
endif
# accumulate initial flags for main spack command
set _sp_flags = ""
while ( $#_sp_args > 0 )
@@ -47,8 +57,7 @@ set _sp_spec=""
[ $#_sp_args -gt 0 ] && set _sp_subcommand = ($_sp_args[1])
[ $#_sp_args -gt 1 ] && set _sp_spec = ($_sp_args[2-])
# Figure out what type of module we're running here.
set _sp_modtype = ""
# Run subcommand
switch ($_sp_subcommand)
case cd:
shift _sp_args # get rid of 'cd'
@@ -106,35 +115,17 @@ case env:
endif
case load:
case unload:
set _sp_module_args=""""
if ( "$_sp_spec" =~ "-*" ) then
set _sp_module_args = $_sp_spec[1]
shift _sp_spec
set _sp_spec = ($_sp_spec)
# Space in `-h` portion is important for differentiating -h option
# from variants that begin with "h" or packages with "-h" in name
if ( "$_sp_spec" =~ "*--sh*" || "$_sp_spec" =~ "*--csh*" || \
" $_sp_spec" =~ "* -h*" || "$_sp_spec" =~ "*--help*") then
# IF a shell is given, print shell output
\spack $_sp_flags $_sp_subcommand $_sp_spec
else
# otherwise eval with csh
eval `\spack $_sp_flags $_sp_subcommand --csh $_sp_spec || \
echo "exit 1"`
endif
# Here the user has run load or unload with a spec. Find a matching
# spec using 'spack module find', then use the appropriate module
# tool's commands to add/remove the result from the environment.
switch ($_sp_subcommand)
case "load":
# _sp_module_args may be "-r" for recursive spec retrieval
set _sp_full_spec = ( "`\spack $_sp_flags module tcl find $_sp_module_args $_sp_spec`" )
if ( "$_sp_module_args" == "-r" ) then
# module load can handle the list of modules to load and "-r" is not a valid option
set _sp_module_args = ""
endif
if ( $? == 0 ) then
module load $_sp_module_args $_sp_full_spec
endif
breaksw
case "unload":
set _sp_full_spec = ( "`\spack $_sp_flags module tcl find $_sp_spec`" )
if ( $? == 0 ) then
module unload $_sp_module_args $_sp_full_spec
endif
breaksw
endsw
breaksw
default:
@@ -143,6 +134,5 @@ default:
endsw
_sp_end:
unset _sp_args _sp_full_spec _sp_modtype _sp_module_args
unset _sp_sh_cmd _sp_spec _sp_subcommand _sp_flags
unset _sp_args _sp_full_spec _sp_sh_cmd _sp_spec _sp_subcommand _sp_flags
unset _sp_arg _sp_env_arg

17
share/spack/qa/setup-env-test.sh
View File

@@ -104,20 +104,25 @@ contains "usage: spack module " spack -m module --help
contains "usage: spack module " spack -m module
title 'Testing `spack load`'
contains "module load $b_module" spack -m load b
contains "export LD_LIBRARY_PATH=$(spack -m location -i b)/lib" spack -m load --only package --sh b
succeeds spack -m load b
fails spack -m load -l
contains "module load -l --arg $b_module" spack -m load -l --arg b
contains "module load $b_module $a_module" spack -m load -r a
contains "module load $b_module $a_module" spack -m load --dependencies a
# test a variable MacOS clears and one it doesn't for recursive loads
contains "export LD_LIBRARY_PATH=$(spack -m location -i a)/lib:$(spack -m location -i b)/lib" spack -m load --sh a
contains "export LIBRARY_PATH=$(spack -m location -i a)/lib:$(spack -m location -i b)/lib" spack -m load --sh a
succeeds spack -m load --only dependencies a
succeeds spack -m load --only package a
fails spack -m load d
contains "usage: spack load " spack -m load -h
contains "usage: spack load " spack -m load -h d
contains "usage: spack load " spack -m load --help
title 'Testing `spack unload`'
contains "module unload $b_module" spack -m unload b
spack -m load b a # setup
succeeds spack -m unload b
succeeds spack -m unload --all
spack -m unload --all # cleanup
fails spack -m unload -l
contains "module unload -l --arg $b_module" spack -m unload -l --arg b
fails spack -m unload d
contains "usage: spack unload " spack -m unload -h
contains "usage: spack unload " spack -m unload -h d

61
share/spack/setup-env.sh
View File

@@ -40,6 +40,16 @@
########################################################################
spack() {
# Store LD_LIBRARY_PATH variables from spack shell function
# This is necessary because MacOS System Integrity Protection clears
# (DY?)LD_LIBRARY_PATH variables on process start.
if [ -n "${LD_LIBRARY_PATH-}" ]; then
export SPACK_LD_LIBRARY_PATH=$LD_LIBRARY_PATH
fi
if [ -n "${DYLD_LIBRARY_PATH-}" ]; then
export SPACK_DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH
fi
# Zsh does not do word splitting by default, this enables it for this
# function only
if [ -n "${ZSH_VERSION:-}" ]; then
@@ -141,41 +151,22 @@ spack() {
return
;;
"load"|"unload")
# Shift any other args for use off before parsing spec.
_sp_subcommand_args=""
_sp_module_args=""
while [ "${1#-}" != "${1}" ]; do
if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
command spack $_sp_flags $_sp_subcommand $_sp_subcommand_args "$@"
return
elif [ "$1" = "-r" ] || [ "$1" = "--dependencies" ]; then
_sp_subcommand_args="$_sp_subcommand_args $1"
else
_sp_module_args="$_sp_module_args $1"
fi
shift
done
# Here the user has run use or unuse with a spec. Find a matching
# spec using 'spack module find', then use the appropriate module
# tool's commands to add/remove the result from the environment.
# If spack module command comes back with an error, do nothing.
case $_sp_subcommand in
"load")
if _sp_full_spec=$(command spack $_sp_flags module tcl find $_sp_subcommand_args "$@"); then
module load $_sp_module_args $_sp_full_spec
else
$(exit 1)
fi
;;
"unload")
if _sp_full_spec=$(command spack $_sp_flags module tcl find $_sp_subcommand_args "$@"); then
module unload $_sp_module_args $_sp_full_spec
else
$(exit 1)
fi
;;
esac
# get --sh, --csh, --help, or -h arguments
# space is important for -h case to differentiate between `-h`
# argument and specs with "-h" in package name or variant settings
_a=" $@"
if [ "${_a#* --sh}" != "$_a" ] || \
[ "${_a#* --csh}" != "$_a" ] || \
[ "${_a#* -h}" != "$_a" ] || \
[ "${_a#* --help}" != "$_a" ];
then
# just execute the command if --sh or --csh are provided
# or if the -h or --help arguments are provided
command spack $_sp_flags $_sp_subcommand "$@"
else
eval $(command spack $_sp_flags $_sp_subcommand --sh "$@" || \
echo "return 1") # return 1 if spack command fails
fi
;;
*)
command spack $_sp_flags $_sp_subcommand "$@"

6
share/spack/spack-completion.bash
View File

@@ -817,7 +817,7 @@ _spack_fetch() {
_spack_find() {
if $list_options
then
SPACK_COMPREPLY="-h --help --format --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tags -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants -M --only-missing --deprecated --only-deprecated -N --namespace --start-date --end-date"
SPACK_COMPREPLY="-h --help --format --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tags -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants --loaded -M --only-missing --deprecated --only-deprecated -N --namespace --start-date --end-date"
else
_installed_packages
fi
@@ -972,7 +972,7 @@ _spack_list() {
_spack_load() {
if $list_options
then
SPACK_COMPREPLY="-h --help -r --dependencies"
SPACK_COMPREPLY="-h --help -r --dependencies --sh --csh --only"
else
_installed_packages
fi
@@ -1420,7 +1420,7 @@ _spack_uninstall() {
_spack_unload() {
if $list_options
then
SPACK_COMPREPLY="-h --help"
SPACK_COMPREPLY="-h --help --sh --csh -a --all"
else
_installed_packages
fi