shell support: spack load no longer needs modules (#14062)

Previously the `spack load` command was a wrapper around `module load`. This required some bootstrapping of modules to make `spack load` work properly.

With this PR, the `spack` shell function handles the environment modifications necessary to add packages to your user environment. This removes the dependence on environment modules or lmod and removes the requirement to bootstrap spack (beyond using the setup-env scripts).

Included in this PR is support for MacOS when using Apple's System Integrity Protection (SIP), which is enabled by default in modern MacOS versions. SIP clears the `LD_LIBRARY_PATH` and `DYLD_LIBRARY_PATH` variables on process startup for executables that live in `/usr` (but not '/usr/local', `/System`, `/bin`, and `/sbin` among other system locations. Spack cannot know the `LD_LIBRARY_PATH` of the calling process when executed using `/bin/sh` and `/usr/bin/python`. The `spack` shell function now manually forwards these two variables, if they are present, as `SPACK_<VAR>` and recovers those values on startup.

- [x] spack load/unload no longer delegate to modules
- [x] refactor user_environment modification calculations
- [x] update documentation for spack load/unload

Co-authored-by: Todd Gamblin <tgamblin@llnl.gov>

share/spack/csh/spack.csh

@@ -27,6 +27,16 @@
 # avoids the need to come up with a user-friendly naming scheme for
 # spack module files.
 ########################################################################
+# Store LD_LIBRARY_PATH variables from spack shell function
+# This is necessary because MacOS System Integrity Protection clears
+# (DY?)LD_LIBRARY_PATH variables on process start.
+if ( ${?LD_LIBRARY_PATH} ) then
+    setenv SPACK_LD_LIBRARY_PATH $LD_LIBRARY_PATH
+endif
+if ( ${?DYLD_LIBRARY_PATH} ) then
+    setenv SPACK_DYLD_LIBRARY_PATH $DYLD_LIBRARY_PATH
+endif
+
 # accumulate initial flags for main spack command
 set _sp_flags = ""
 while ( $#_sp_args > 0 )
@@ -47,8 +57,7 @@ set _sp_spec=""
 [ $#_sp_args -gt 0 ] && set _sp_subcommand = ($_sp_args[1])
 [ $#_sp_args -gt 1 ] && set _sp_spec = ($_sp_args[2-])
 
-# Figure out what type of module we're running here.
-set _sp_modtype = ""
+# Run subcommand
 switch ($_sp_subcommand)
 case cd:
     shift _sp_args  # get rid of 'cd'
@@ -106,35 +115,17 @@ case env:
     endif
 case load:
 case unload:
-    set _sp_module_args=""""
-    if ( "$_sp_spec" =~ "-*" ) then
-        set _sp_module_args = $_sp_spec[1]
-        shift _sp_spec
-        set _sp_spec = ($_sp_spec)
+    # Space in `-h` portion is important for differentiating -h option
+    # from variants that begin with "h" or packages with "-h" in name
+    if ( "$_sp_spec" =~ "*--sh*" || "$_sp_spec" =~ "*--csh*" || \
+         " $_sp_spec" =~ "* -h*" || "$_sp_spec" =~ "*--help*") then
+        # IF a shell is given, print shell output
+        \spack $_sp_flags $_sp_subcommand $_sp_spec
+    else
+        # otherwise eval with csh
+        eval `\spack $_sp_flags $_sp_subcommand --csh $_sp_spec || \
+             echo "exit 1"`
     endif
-
-    # Here the user has run load or unload with a spec.  Find a matching
-    # spec using 'spack module find', then use the appropriate module
-    # tool's commands to add/remove the result from the environment.
-    switch ($_sp_subcommand)
-        case "load":
-            # _sp_module_args may be "-r" for recursive spec retrieval
-            set _sp_full_spec = ( "`\spack $_sp_flags module tcl find $_sp_module_args $_sp_spec`" )
-            if ( "$_sp_module_args" == "-r" ) then
-                # module load can handle the list of modules to load and "-r" is not a valid option
-                set _sp_module_args = ""
-            endif
-            if ( $? == 0 ) then
-                module load $_sp_module_args $_sp_full_spec
-            endif
-            breaksw
-        case "unload":
-            set _sp_full_spec = ( "`\spack $_sp_flags module tcl find $_sp_spec`" )
-            if ( $? == 0 ) then
-                module unload $_sp_module_args $_sp_full_spec
-            endif
-            breaksw
-    endsw
     breaksw
 
 default:
@@ -143,6 +134,5 @@ default:
 endsw
 
 _sp_end:
-unset _sp_args _sp_full_spec _sp_modtype _sp_module_args
-unset _sp_sh_cmd _sp_spec _sp_subcommand _sp_flags
+unset _sp_args _sp_full_spec _sp_sh_cmd _sp_spec _sp_subcommand _sp_flags
 unset _sp_arg _sp_env_arg