zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: move most of spack.util.environment -> llnl.util.envmod

Browse Source
This commit is contained in:
Todd Gamblin
2022-12-31 03:29:59 -08:00
parent d3f8bf1b96
commit b4a2e2b82c
66 changed files with 1278 additions and 1273 deletions
Download Patch File Download Diff File Expand all files Collapse all files

983
lib/spack/llnl/util/envmod.py Normal file
View File

@@ -0,0 +1,983 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Utilities for setting and modifying environment variables."""
import collections
import contextlib
import inspect
import json
import os
import os.path
import re
import shlex
import sys
import llnl.util.lang
import llnl.util.tty as tty
from llnl.util.path import path_to_os_path
import spack.util.executable as executable
is_windows = sys.platform == "win32"
system_paths = (
["/", "/usr", "/usr/local"]
if not is_windows
else ["C:\\", "C:\\Program Files", "C:\\Program Files (x86)", "C:\\Users", "C:\\ProgramData"]
)
suffixes = ["bin", "bin64", "include", "lib", "lib64"] if not is_windows else []
system_dirs = [os.path.join(p, s) for s in suffixes for p in system_paths] + system_paths
def is_system_path(path):
"""Predicate that given a path returns True if it is a system path,
False otherwise.
Args:
path (str): path to a directory
Returns:
True or False
"""
return path and os.path.normpath(path) in system_dirs
def filter_system_paths(paths):
"""Return only paths that are not system paths."""
return [p for p in paths if not is_system_path(p)]
def deprioritize_system_paths(paths):
"""Put system paths at the end of paths, otherwise preserving order."""
filtered_paths = filter_system_paths(paths)
fp = set(filtered_paths)
return filtered_paths + [p for p in paths if p not in fp]
_shell_set_strings = {
"sh": "export {0}={1};\n",
"csh": "setenv {0} {1};\n",
"fish": "set -gx {0} {1};\n",
"bat": 'set "{0}={1}"\n',
}
_shell_unset_strings = {
"sh": "unset {0};\n",
"csh": "unsetenv {0};\n",
"fish": "set -e {0};\n",
"bat": 'set "{0}="\n',
}
tracing_enabled = False
def prune_duplicate_paths(paths):
"""Returns the paths with duplicates removed, order preserved."""
return list(llnl.util.lang.dedupe(paths))
def get_path(name):
path = os.environ.get(name, "").strip()
if path:
return path.split(os.pathsep)
else:
return []
def env_flag(name):
if name in os.environ:
value = os.environ[name].lower()
return value == "true" or value == "1"
return False
def path_set(var_name, directories):
path_str = os.pathsep.join(str(dir) for dir in directories)
os.environ[var_name] = path_str
def path_put_first(var_name, directories):
"""Puts the provided directories first in the path, adding them
if they're not already there.
"""
path = os.environ.get(var_name, "").split(os.pathsep)
for dir in directories:
if dir in path:
path.remove(dir)
new_path = tuple(directories) + tuple(path)
path_set(var_name, new_path)
@contextlib.contextmanager
def set_env(**kwargs):
"""Temporarily sets and restores environment variables.
Variables can be set as keyword arguments to this function.
"""
saved = {}
for var, value in kwargs.items():
if var in os.environ:
saved[var] = os.environ[var]
if value is None:
if var in os.environ:
del os.environ[var]
else:
os.environ[var] = value
yield
for var, value in kwargs.items():
if var in saved:
os.environ[var] = saved[var]
else:
if var in os.environ:
del os.environ[var]
class NameModifier(object):
def __init__(self, name, **kwargs):
self.name = name
self.separator = kwargs.get("separator", os.pathsep)
self.args = {"name": name, "separator": self.separator}
self.args.update(kwargs)
def __eq__(self, other):
if not isinstance(other, NameModifier):
return False
return self.name == other.name
def update_args(self, **kwargs):
self.__dict__.update(kwargs)
self.args.update(kwargs)
class NameValueModifier(object):
def __init__(self, name, value, **kwargs):
self.name = name
self.value = value
self.separator = kwargs.get("separator", os.pathsep)
self.args = {"name": name, "value": value, "separator": self.separator}
self.args.update(kwargs)
def __eq__(self, other):
if not isinstance(other, NameValueModifier):
return False
return (
self.name == other.name
and self.value == other.value
and self.separator == other.separator
)
def update_args(self, **kwargs):
self.__dict__.update(kwargs)
self.args.update(kwargs)
class SetEnv(NameValueModifier):
def execute(self, env):
tty.debug("SetEnv: {0}={1}".format(self.name, str(self.value)), level=3)
env[self.name] = str(self.value)
class AppendFlagsEnv(NameValueModifier):
def execute(self, env):
tty.debug("AppendFlagsEnv: {0}={1}".format(self.name, str(self.value)), level=3)
if self.name in env and env[self.name]:
env[self.name] += self.separator + str(self.value)
else:
env[self.name] = str(self.value)
class UnsetEnv(NameModifier):
def execute(self, env):
tty.debug("UnsetEnv: {0}".format(self.name), level=3)
# Avoid throwing if the variable was not set
env.pop(self.name, None)
class RemoveFlagsEnv(NameValueModifier):
def execute(self, env):
tty.debug("RemoveFlagsEnv: {0}-{1}".format(self.name, str(self.value)), level=3)
environment_value = env.get(self.name, "")
flags = environment_value.split(self.separator) if environment_value else []
flags = [f for f in flags if f != self.value]
env[self.name] = self.separator.join(flags)
class SetPath(NameValueModifier):
def execute(self, env):
string_path = concatenate_paths(self.value, separator=self.separator)
tty.debug("SetPath: {0}={1}".format(self.name, string_path), level=3)
env[self.name] = string_path
class AppendPath(NameValueModifier):
def execute(self, env):
tty.debug("AppendPath: {0}+{1}".format(self.name, str(self.value)), level=3)
environment_value = env.get(self.name, "")
directories = environment_value.split(self.separator) if environment_value else []
directories.append(path_to_os_path(os.path.normpath(self.value)).pop())
env[self.name] = self.separator.join(directories)
class PrependPath(NameValueModifier):
def execute(self, env):
tty.debug("PrependPath: {0}+{1}".format(self.name, str(self.value)), level=3)
environment_value = env.get(self.name, "")
directories = environment_value.split(self.separator) if environment_value else []
directories = [path_to_os_path(os.path.normpath(self.value)).pop()] + directories
env[self.name] = self.separator.join(directories)
class RemovePath(NameValueModifier):
def execute(self, env):
tty.debug("RemovePath: {0}-{1}".format(self.name, str(self.value)), level=3)
environment_value = env.get(self.name, "")
directories = environment_value.split(self.separator) if environment_value else []
directories = [
path_to_os_path(os.path.normpath(x)).pop()
for x in directories
if x != path_to_os_path(os.path.normpath(self.value)).pop()
]
env[self.name] = self.separator.join(directories)
class DeprioritizeSystemPaths(NameModifier):
def execute(self, env):
tty.debug("DeprioritizeSystemPaths: {0}".format(self.name), level=3)
environment_value = env.get(self.name, "")
directories = environment_value.split(self.separator) if environment_value else []
directories = deprioritize_system_paths(
[path_to_os_path(os.path.normpath(x)).pop() for x in directories]
)
env[self.name] = self.separator.join(directories)
class PruneDuplicatePaths(NameModifier):
def execute(self, env):
tty.debug("PruneDuplicatePaths: {0}".format(self.name), level=3)
environment_value = env.get(self.name, "")
directories = environment_value.split(self.separator) if environment_value else []
directories = prune_duplicate_paths(
[path_to_os_path(os.path.normpath(x)).pop() for x in directories]
)
env[self.name] = self.separator.join(directories)
class EnvironmentModifications(object):
"""Keeps track of requests to modify the current environment.
Each call to a method to modify the environment stores the extra
information on the caller in the request:
* 'filename' : filename of the module where the caller is defined
* 'lineno': line number where the request occurred
* 'context' : line of code that issued the request that failed
"""
def __init__(self, other=None, traced=None):
"""Initializes a new instance, copying commands from 'other'
if it is not None.
Args:
other (EnvironmentModifications): list of environment modifications
to be extended (optional)
traced (bool): enable or disable stack trace inspection to log the origin
of the environment modifications.
"""
self.traced = tracing_enabled if traced is None else bool(traced)
self.env_modifications = []
if other is not None:
self.extend(other)
def __iter__(self):
return iter(self.env_modifications)
def __len__(self):
return len(self.env_modifications)
def extend(self, other):
self._check_other(other)
self.env_modifications.extend(other.env_modifications)
@staticmethod
def _check_other(other):
if not isinstance(other, EnvironmentModifications):
raise TypeError("other must be an instance of EnvironmentModifications")
def _maybe_trace(self, kwargs):
"""Provide the modification with stack trace info so that we can track its
origin to find issues in packages. This is very slow and expensive."""
if not self.traced:
return
stack = inspect.stack()
try:
_, filename, lineno, _, context, index = stack[2]
context = context[index].strip()
except Exception:
filename = "unknown file"
lineno = "unknown line"
context = "unknown context"
kwargs.update({"filename": filename, "lineno": lineno, "context": context})
def set(self, name, value, **kwargs):
"""Stores a request to set an environment variable.
Args:
name: name of the environment variable to be set
value: value of the environment variable
"""
self._maybe_trace(kwargs)
item = SetEnv(name, value, **kwargs)
self.env_modifications.append(item)
def append_flags(self, name, value, sep=" ", **kwargs):
"""
Stores in the current object a request to append to an env variable
Args:
name: name of the environment variable to be appended to
value: value to append to the environment variable
Appends with spaces separating different additions to the variable
"""
self._maybe_trace(kwargs)
kwargs.update({"separator": sep})
item = AppendFlagsEnv(name, value, **kwargs)
self.env_modifications.append(item)
def unset(self, name, **kwargs):
"""Stores a request to unset an environment variable.
Args:
name: name of the environment variable to be unset
"""
self._maybe_trace(kwargs)
item = UnsetEnv(name, **kwargs)
self.env_modifications.append(item)
def remove_flags(self, name, value, sep=" ", **kwargs):
"""
Stores in the current object a request to remove flags from an
env variable
Args:
name: name of the environment variable to be removed from
value: value to remove to the environment variable
sep: separator to assume for environment variable
"""
self._maybe_trace(kwargs)
kwargs.update({"separator": sep})
item = RemoveFlagsEnv(name, value, **kwargs)
self.env_modifications.append(item)
def set_path(self, name, elements, **kwargs):
"""Stores a request to set a path generated from a list.
Args:
name: name o the environment variable to be set.
elements: elements of the path to set.
"""
self._maybe_trace(kwargs)
item = SetPath(name, elements, **kwargs)
self.env_modifications.append(item)
def append_path(self, name, path, **kwargs):
"""Stores a request to append a path to a path list.
Args:
name: name of the path list in the environment
path: path to be appended
"""
self._maybe_trace(kwargs)
item = AppendPath(name, path, **kwargs)
self.env_modifications.append(item)
def prepend_path(self, name, path, **kwargs):
"""Same as `append_path`, but the path is pre-pended.
Args:
name: name of the path list in the environment
path: path to be pre-pended
"""
self._maybe_trace(kwargs)
item = PrependPath(name, path, **kwargs)
self.env_modifications.append(item)
def remove_path(self, name, path, **kwargs):
"""Stores a request to remove a path from a path list.
Args:
name: name of the path list in the environment
path: path to be removed
"""
self._maybe_trace(kwargs)
item = RemovePath(name, path, **kwargs)
self.env_modifications.append(item)
def deprioritize_system_paths(self, name, **kwargs):
"""Stores a request to deprioritize system paths in a path list,
otherwise preserving the order.
Args:
name: name of the path list in the environment.
"""
self._maybe_trace(kwargs)
item = DeprioritizeSystemPaths(name, **kwargs)
self.env_modifications.append(item)
def prune_duplicate_paths(self, name, **kwargs):
"""Stores a request to remove duplicates from a path list, otherwise
preserving the order.
Args:
name: name of the path list in the environment.
"""
self._maybe_trace(kwargs)
item = PruneDuplicatePaths(name, **kwargs)
self.env_modifications.append(item)
def group_by_name(self):
"""Returns a dict of the modifications grouped by variable name.
Returns:
dict mapping the environment variable name to the modifications to
be done on it
"""
modifications = collections.defaultdict(list)
for item in self:
modifications[item.name].append(item)
return modifications
def is_unset(self, var_name):
modifications = self.group_by_name()
var_updates = modifications.get(var_name, None)
if not var_updates:
# We did not explicitly unset it
return False
# The last modification must unset the variable for it to be considered
# unset
return type(var_updates[-1]) == UnsetEnv
def clear(self):
"""
Clears the current list of modifications
"""
self.env_modifications = []
def reversed(self):
"""
Returns the EnvironmentModifications object that will reverse self
Only creates reversals for additions to the environment, as reversing
``unset`` and ``remove_path`` modifications is impossible.
Reversable operations are set(), prepend_path(), append_path(),
set_path(), and append_flags().
"""
rev = EnvironmentModifications()
for envmod in reversed(self.env_modifications):
if type(envmod) == SetEnv:
tty.debug("Reversing `Set` environment operation may lose " "original value")
rev.unset(envmod.name)
elif type(envmod) == AppendPath:
rev.remove_path(envmod.name, envmod.value)
elif type(envmod) == PrependPath:
rev.remove_path(envmod.name, envmod.value)
elif type(envmod) == SetPath:
tty.debug("Reversing `SetPath` environment operation may lose " "original value")
rev.unset(envmod.name)
elif type(envmod) == AppendFlagsEnv:
rev.remove_flags(envmod.name, envmod.value)
else:
# This is an un-reversable operation
tty.warn(
"Skipping reversal of unreversable operation"
"%s %s" % (type(envmod), envmod.name)
)
return rev
def apply_modifications(self, env=None):
"""Applies the modifications and clears the list."""
# Use os.environ if not specified
# Do not copy, we want to modify it in place
if env is None:
env = os.environ
modifications = self.group_by_name()
# Apply modifications one variable at a time
for name, actions in sorted(modifications.items()):
for x in actions:
x.execute(env)
def shell_modifications(self, shell="sh", explicit=False, env=None):
"""Return shell code to apply the modifications and clears the list."""
modifications = self.group_by_name()
if env is None:
env = os.environ
new_env = env.copy()
for name, actions in sorted(modifications.items()):
for x in actions:
x.execute(new_env)
if "MANPATH" in new_env and not new_env.get("MANPATH").endswith(":"):
new_env["MANPATH"] += ":"
cmds = ""
for name in sorted(set(modifications)):
new = new_env.get(name, None)
old = env.get(name, None)
if explicit or new != old:
if new is None:
cmds += _shell_unset_strings[shell].format(name)
else:
if sys.platform != "win32":
cmd = _shell_set_strings[shell].format(name, shlex.quote(new_env[name]))
else:
cmd = _shell_set_strings[shell].format(name, new_env[name])
cmds += cmd
return cmds
@staticmethod
def from_sourcing_file(filename, *arguments, **kwargs):
"""Constructs an instance of a
:py:class:`llnl.util.envmod.EnvironmentModifications` object
that has the same effect as sourcing a file.
Args:
filename (str): the file to be sourced
*arguments (list): arguments to pass on the command line
Keyword Args:
shell (str): the shell to use (default: ``bash``)
shell_options (str): options passed to the shell (default: ``-c``)
source_command (str): the command to run (default: ``source``)
suppress_output (str): redirect used to suppress output of command
(default: ``&> /dev/null``)
concatenate_on_success (str): operator used to execute a command
only when the previous command succeeds (default: ``&&``)
exclude ([str or re]): ignore any modifications of these
variables (default: [])
include ([str or re]): always respect modifications of these
variables (default: []). Supersedes any excluded variables.
clean (bool): in addition to removing empty entries,
also remove duplicate entries (default: False).
"""
tty.debug("EnvironmentModifications.from_sourcing_file: {0}".format(filename))
# Check if the file actually exists
if not os.path.isfile(filename):
msg = "Trying to source non-existing file: {0}".format(filename)
raise RuntimeError(msg)
# Prepare include and exclude lists of environment variable names
exclude = kwargs.get("exclude", [])
include = kwargs.get("include", [])
clean = kwargs.get("clean", False)
# Other variables unrelated to sourcing a file
exclude.extend(
[
# Bash internals
"SHLVL",
"_",
"PWD",
"OLDPWD",
"PS1",
"PS2",
"ENV",
# Environment modules v4
"LOADEDMODULES",
"_LMFILES_",
"BASH_FUNC_module()",
"MODULEPATH",
"MODULES_(.*)",
r"(\w*)_mod(quar|share)",
# Lmod configuration
r"LMOD_(.*)",
"MODULERCFILE",
]
)
# Compute the environments before and after sourcing
before = sanitize(
environment_after_sourcing_files(os.devnull, **kwargs),
exclude=exclude,
include=include,
)
file_and_args = (filename,) + arguments
after = sanitize(
environment_after_sourcing_files(file_and_args, **kwargs),
exclude=exclude,
include=include,
)
# Delegate to the other factory
return EnvironmentModifications.from_environment_diff(before, after, clean)
@staticmethod
def from_environment_diff(before, after, clean=False):
"""Constructs an instance of a
:py:class:`llnl.util.envmod.EnvironmentModifications` object
from the diff of two dictionaries.
Args:
before (dict): environment before the modifications are applied
after (dict): environment after the modifications are applied
clean (bool): in addition to removing empty entries, also remove
duplicate entries
"""
# Fill the EnvironmentModifications instance
env = EnvironmentModifications()
# New variables
new_variables = list(set(after) - set(before))
# Variables that have been unset
unset_variables = list(set(before) - set(after))
# Variables that have been modified
common_variables = set(before).intersection(set(after))
modified_variables = [x for x in common_variables if before[x] != after[x]]
# Consistent output order - looks nicer, easier comparison...
new_variables.sort()
unset_variables.sort()
modified_variables.sort()
def return_separator_if_any(*args):
separators = ":", ";"
for separator in separators:
for arg in args:
if separator in arg:
return separator
return None
# Add variables to env.
# Assume that variables with 'PATH' in the name or that contain
# separators like ':' or ';' are more likely to be paths
for x in new_variables:
sep = return_separator_if_any(after[x])
if sep:
env.prepend_path(x, after[x], separator=sep)
elif "PATH" in x:
env.prepend_path(x, after[x])
else:
# We just need to set the variable to the new value
env.set(x, after[x])
for x in unset_variables:
env.unset(x)
for x in modified_variables:
value_before = before[x]
value_after = after[x]
sep = return_separator_if_any(value_before, value_after)
if sep:
before_list = value_before.split(sep)
after_list = value_after.split(sep)
# Filter out empty strings
before_list = list(filter(None, before_list))
after_list = list(filter(None, after_list))
# Remove duplicate entries (worse matching, bloats env)
if clean:
before_list = list(llnl.util.lang.dedupe(before_list))
after_list = list(llnl.util.lang.dedupe(after_list))
# The reassembled cleaned entries
value_before = sep.join(before_list)
value_after = sep.join(after_list)
# Paths that have been removed
remove_list = [ii for ii in before_list if ii not in after_list]
# Check that nothing has been added in the middle of
# before_list
remaining_list = [ii for ii in before_list if ii in after_list]
try:
start = after_list.index(remaining_list[0])
end = after_list.index(remaining_list[-1])
search = sep.join(after_list[start : end + 1])
except IndexError:
env.prepend_path(x, value_after)
continue
if search not in value_before:
# We just need to set the variable to the new value
env.prepend_path(x, value_after)
else:
try:
prepend_list = after_list[:start]
prepend_list.reverse() # Preserve order after prepend
except KeyError:
prepend_list = []
try:
append_list = after_list[end + 1 :]
except KeyError:
append_list = []
for item in remove_list:
env.remove_path(x, item)
for item in append_list:
env.append_path(x, item)
for item in prepend_list:
env.prepend_path(x, item)
else:
# We just need to set the variable to the new value
env.set(x, value_after)
return env
def concatenate_paths(paths, separator=os.pathsep):
"""Concatenates an iterable of paths into a string of paths separated by
separator, defaulting to colon.
Args:
paths: iterable of paths
separator: the separator to use, default ';' windows, ':' otherwise
Returns:
string
"""
return separator.join(str(item) for item in paths)
def set_or_unset_not_first(variable, changes, errstream):
"""Check if we are going to set or unset something after other
modifications have already been requested.
"""
indexes = [
ii
for ii, item in enumerate(changes)
if ii != 0 and not item.args.get("force", False) and type(item) in [SetEnv, UnsetEnv]
]
if indexes:
good = "\t \t{context} at {filename}:{lineno}"
nogood = "\t--->\t{context} at {filename}:{lineno}"
message = "Suspicious requests to set or unset '{var}' found"
errstream(message.format(var=variable))
for ii, item in enumerate(changes):
print_format = nogood if ii in indexes else good
errstream(print_format.format(**item.args))
def validate(env, errstream):
"""Validates the environment modifications to check for the presence of
suspicious patterns. Prompts a warning for everything that was found.
Current checks:
- set or unset variables after other changes on the same variable
Args:
env: list of environment modifications
"""
if not env.traced:
return
modifications = env.group_by_name()
for variable, list_of_changes in sorted(modifications.items()):
set_or_unset_not_first(variable, list_of_changes, errstream)
def inspect_path(root, inspections, exclude=None):
"""Inspects ``root`` to search for the subdirectories in ``inspections``.
Adds every path found to a list of prepend-path commands and returns it.
Args:
root (str): absolute path where to search for subdirectories
inspections (dict): maps relative paths to a list of environment
variables that will be modified if the path exists. The
modifications are not performed immediately, but stored in a
command object that is returned to client
exclude (typing.Callable): optional callable. If present it must accept an
absolute path and return True if it should be excluded from the
inspection
Examples:
The following lines execute an inspection in ``/usr`` to search for
``/usr/include`` and ``/usr/lib64``. If found we want to prepend
``/usr/include`` to ``CPATH`` and ``/usr/lib64`` to ``MY_LIB64_PATH``.
.. code-block:: python
# Set up the dictionary containing the inspection
inspections = {
'include': ['CPATH'],
'lib64': ['MY_LIB64_PATH']
}
# Get back the list of command needed to modify the environment
env = inspect_path('/usr', inspections)
# Eventually execute the commands
env.apply_modifications()
Returns:
instance of EnvironmentModifications containing the requested
modifications
"""
if exclude is None:
exclude = lambda x: False
env = EnvironmentModifications()
# Inspect the prefix to check for the existence of common directories
for relative_path, variables in inspections.items():
expected = os.path.join(root, relative_path)
if os.path.isdir(expected) and not exclude(expected):
for variable in variables:
env.prepend_path(variable, expected)
return env
@contextlib.contextmanager
def preserve_environment(*variables):
"""Ensures that the value of the environment variables passed as
arguments is the same before entering to the context manager and after
exiting it.
Variables that are unset before entering the context manager will be
explicitly unset on exit.
Args:
variables (list): list of environment variables to be preserved
"""
cache = {}
for var in variables:
# The environment variable to be preserved might not be there.
# In that case store None as a placeholder.
cache[var] = os.environ.get(var, None)
yield
for var in variables:
value = cache[var]
msg = "[PRESERVE_ENVIRONMENT]"
if value is not None:
# Print a debug statement if the value changed
if var not in os.environ:
msg += ' {0} was unset, will be reset to "{1}"'
tty.debug(msg.format(var, value))
elif os.environ[var] != value:
msg += ' {0} was set to "{1}", will be reset to "{2}"'
tty.debug(msg.format(var, os.environ[var], value))
os.environ[var] = value
elif var in os.environ:
msg += ' {0} was set to "{1}", will be unset'
tty.debug(msg.format(var, os.environ[var]))
del os.environ[var]
def environment_after_sourcing_files(*files, **kwargs):
"""Returns a dictionary with the environment that one would have
after sourcing the files passed as argument.
Args:
*files: each item can either be a string containing the path
of the file to be sourced or a sequence, where the first element
is the file to be sourced and the remaining are arguments to be
passed to the command line
Keyword Args:
env (dict): the initial environment (default: current environment)
shell (str): the shell to use (default: ``/bin/bash``)
shell_options (str): options passed to the shell (default: ``-c``)
source_command (str): the command to run (default: ``source``)
suppress_output (str): redirect used to suppress output of command
(default: ``&> /dev/null``)
concatenate_on_success (str): operator used to execute a command
only when the previous command succeeds (default: ``&&``)
"""
# Set the shell executable that will be used to source files
shell_cmd = kwargs.get("shell", "/bin/bash")
shell_options = kwargs.get("shell_options", "-c")
source_command = kwargs.get("source_command", "source")
suppress_output = kwargs.get("suppress_output", "&> /dev/null")
concatenate_on_success = kwargs.get("concatenate_on_success", "&&")
shell = executable.Executable(" ".join([shell_cmd, shell_options]))
def _source_single_file(file_and_args, environment):
source_file = [source_command]
source_file.extend(x for x in file_and_args)
source_file = " ".join(source_file)
# If the environment contains 'python' use it, if not
# go with sys.executable. Below we just need a working
# Python interpreter, not necessarily sys.executable.
python_cmd = executable.which("python3", "python", "python2")
python_cmd = python_cmd.path if python_cmd else sys.executable
dump_cmd = "import os, json; print(json.dumps(dict(os.environ)))"
dump_environment = python_cmd + ' -E -c "{0}"'.format(dump_cmd)
# Try to source the file
source_file_arguments = " ".join(
[
source_file,
suppress_output,
concatenate_on_success,
dump_environment,
]
)
output = shell(source_file_arguments, output=str, env=environment, ignore_quotes=True)
return json.loads(output)
current_environment = kwargs.get("env", dict(os.environ))
for f in files:
# Normalize the input to the helper function
if isinstance(f, str):
f = [f]
current_environment = _source_single_file(f, environment=current_environment)
return current_environment
def sanitize(environment, exclude, include):
"""Returns a copy of the input dictionary where all the keys that
match an excluded pattern and don't match an included pattern are
removed.
Args:
environment (dict): input dictionary
exclude (list): literals or regex patterns to be excluded
include (list): literals or regex patterns to be included
"""
def set_intersection(fullset, *args):
# A set intersection using string literals and regexs
meta = "[" + re.escape("[$()*?[]^{|}") + "]"
subset = fullset & set(args) # As literal
for name in args:
if re.search(meta, name):
pattern = re.compile(name)
for k in fullset:
if re.match(pattern, k):
subset.add(k)
return subset
# Don't modify input, make a copy instead
environment = dict(environment)
# include supersedes any excluded items
prune = set_intersection(set(environment), *exclude)
prune -= set_intersection(prune, *include)
for k in prune:
environment.pop(k, None)
return environment

1
lib/spack/llnl/util/filesystem.py
View File

@@ -25,7 +25,6 @@
from spack.util.executable import CommandNotFoundError, Executable, which
is_windows = _platform == "win32"
if not is_windows:

1
lib/spack/llnl/util/lock.py
View File

@@ -14,7 +14,6 @@
import llnl.util.tty as tty
from llnl.util.lang import pretty_seconds
if sys.platform != "win32":
import fcntl

4
lib/spack/spack/bootstrap/_common.py
View File

@@ -12,11 +12,11 @@
import archspec.cpu
import llnl.util.envmod
import llnl.util.filesystem as fs
from llnl.util import tty
import spack.store
import spack.util.environment
import spack.util.executable
from .config import spec_for_current_python
@@ -188,7 +188,7 @@ def _executables_in_store(executables, query_spec, query_info=None):
and os.path.isdir(bin_dir)
and spack.util.executable.which_string(*executables, path=bin_dir)
):
spack.util.environment.path_put_first("PATH", [bin_dir])
llnl.util.envmod.path_put_first("PATH", [bin_dir])
if query_info is not None:
query_info["command"] = spack.util.executable.which(*executables, path=bin_dir)
query_info["spec"] = concrete_spec

4
lib/spack/spack/bootstrap/core.py
View File

@@ -31,6 +31,7 @@
import uuid
from typing import Callable, List, Optional
import llnl.util.envmod
from llnl.util import tty
from llnl.util.lang import GroupedExceptionHandler
@@ -46,7 +47,6 @@
import spack.spec
import spack.store
import spack.user_environment
import spack.util.environment
import spack.util.executable
import spack.util.path
import spack.util.spack_yaml
@@ -439,7 +439,7 @@ def ensure_executables_in_path_or_raise(
current_bootstrapper.last_search["spec"],
current_bootstrapper.last_search["command"],
)
env_mods = spack.util.environment.EnvironmentModifications()
env_mods = llnl.util.envmod.EnvironmentModifications()
for dep in concrete_spec.traverse(
root=True, order="post", deptype=("link", "run")
):

20
lib/spack/spack/build_environment.py
View File

@@ -43,6 +43,16 @@
from typing import List, Tuple
import llnl.util.tty as tty
from llnl.util.envmod import (
EnvironmentModifications,
env_flag,
filter_system_paths,
get_path,
inspect_path,
is_system_path,
system_dirs,
validate,
)
from llnl.util.lang import dedupe
from llnl.util.string import plural
from llnl.util.symlink import symlink
@@ -68,16 +78,6 @@
from spack.error import NoHeadersError, NoLibrariesError
from spack.installer import InstallError
from spack.util.cpus import cpus_available
from spack.util.environment import (
EnvironmentModifications,
env_flag,
filter_system_paths,
get_path,
inspect_path,
is_system_path,
system_dirs,
validate,
)
from spack.util.executable import Executable
from spack.util.log_parse import make_log_context, parse_log_events
from spack.util.module_cmd import load_module, module, path_from_modules

2
lib/spack/spack/build_systems/intel.py
View File

@@ -11,6 +11,7 @@
import xml.etree.ElementTree as ElementTree
import llnl.util.tty as tty
from llnl.util.envmod import EnvironmentModifications
from llnl.util.filesystem import (
HeaderList,
LibraryList,
@@ -25,7 +26,6 @@
import spack.error
from spack.build_environment import dso_suffix
from spack.package_base import InstallError
from spack.util.environment import EnvironmentModifications
from spack.util.executable import Executable
from spack.util.prefix import Prefix
from spack.version import Version, ver

2
lib/spack/spack/build_systems/oneapi.py
View File

@@ -8,10 +8,10 @@
import shutil
from os.path import basename, dirname, isdir
from llnl.util.envmod import EnvironmentModifications
from llnl.util.filesystem import find_headers, find_libraries, join_path
from spack.directives import conflicts, variant
from spack.util.environment import EnvironmentModifications
from spack.util.executable import Executable
from .generic import Package

2
lib/spack/spack/build_systems/racket.py
View File

@@ -8,12 +8,12 @@
import llnl.util.filesystem as fs
import llnl.util.lang as lang
import llnl.util.tty as tty
from llnl.util.envmod import env_flag
import spack.builder
from spack.build_environment import SPACK_NO_PARALLEL_MAKE, determine_number_of_jobs
from spack.directives import build_system, extends
from spack.package_base import PackageBase
from spack.util.environment import env_flag
from spack.util.executable import Executable, ProcessError

4
lib/spack/spack/builder.py
View File

@@ -518,7 +518,7 @@ def setup_build_environment(self, env):
Spack's store.
Args:
env (spack.util.environment.EnvironmentModifications): environment
env (llnl.util.envmod.EnvironmentModifications): environment
modifications to be applied when the package is built. Package authors
can call methods on it to alter the build environment.
"""
@@ -546,7 +546,7 @@ def setup_dependent_build_environment(self, env, dependent_spec):
variable.
Args:
env (spack.util.environment.EnvironmentModifications): environment
env (llnl.util.envmod.EnvironmentModifications): environment
modifications to be applied when the dependent package is built.
Package authors can call methods on it to alter the build environment.

1
lib/spack/spack/cmd/diff.py
View File

@@ -13,7 +13,6 @@
import spack.cmd.common.arguments as arguments
import spack.environment as ev
import spack.solver.asp as asp
import spack.util.environment
import spack.util.spack_json as sjson
description = "compare two specs"

2
lib/spack/spack/cmd/env.py
View File

@@ -13,6 +13,7 @@
import llnl.util.filesystem as fs
import llnl.util.string as string
import llnl.util.tty as tty
from llnl.util.envmod import EnvironmentModifications
from llnl.util.tty.colify import colify
from llnl.util.tty.color import colorize
@@ -29,7 +30,6 @@
import spack.schema.env
import spack.tengine
import spack.traverse as traverse
from spack.util.environment import EnvironmentModifications
description = "manage virtual environments"
section = "environments"

1
lib/spack/spack/cmd/external.py
View File

@@ -18,7 +18,6 @@
import spack.cray_manifest as cray_manifest
import spack.detection
import spack.error
import spack.util.environment
description = "manage external packages in Spack configuration"
section = "config"

5
lib/spack/spack/cmd/load.py
View File

@@ -5,13 +5,14 @@
import sys
import llnl.util.envmod
import spack.cmd
import spack.cmd.common.arguments as arguments
import spack.cmd.find
import spack.environment as ev
import spack.store
import spack.user_environment as uenv
import spack.util.environment
description = "add package to the user environment"
section = "user environment"
@@ -110,7 +111,7 @@ def load(parser, args):
dep for spec in specs for dep in spec.traverse(root=include_roots, order="post")
]
env_mod = spack.util.environment.EnvironmentModifications()
env_mod = llnl.util.envmod.EnvironmentModifications()
for spec in specs:
env_mod.extend(uenv.environment_modifications_for_spec(spec))
env_mod.prepend_path(uenv.spack_loaded_hashes_var, spec.dag_hash())

5
lib/spack/spack/cmd/unload.py
View File

@@ -6,11 +6,12 @@
import os
import sys
import llnl.util.envmod
import spack.cmd
import spack.cmd.common.arguments as arguments
import spack.error
import spack.user_environment as uenv
import spack.util.environment
description = "remove package from the user environment"
section = "user environment"
@@ -82,7 +83,7 @@ def unload(parser, args):
)
return 1
env_mod = spack.util.environment.EnvironmentModifications()
env_mod = llnl.util.envmod.EnvironmentModifications()
for spec in specs:
env_mod.extend(uenv.environment_modifications_for_spec(spec).reversed())
env_mod.remove_path(uenv.spack_loaded_hashes_var, spec.dag_hash())

8
lib/spack/spack/compiler.py
View File

@@ -13,9 +13,11 @@
import tempfile
from typing import List, Optional, Sequence
import llnl.util.envmod as envmod
import llnl.util.lang
import llnl.util.tty as tty
from llnl.util.filesystem import path_contains_subdirectory, paths_containing_libs
from llnl.util.path import system_path_filter
import spack.compilers
import spack.error
@@ -23,8 +25,6 @@
import spack.util.executable
import spack.util.module_cmd
import spack.version
from spack.util.environment import filter_system_paths
from llnl.util.path import system_path_filter
__all__ = ["Compiler"]
@@ -175,7 +175,7 @@ def _parse_non_system_link_dirs(string: str) -> List[str]:
# system paths. Note that 'filter_system_paths' only checks for an
# exact match, while 'in_system_subdirectory' checks if a path contains
# a system directory as a subdirectory
link_dirs = filter_system_paths(link_dirs)
link_dirs = envmod.filter_system_paths(link_dirs)
return list(p for p in link_dirs if not in_system_subdirectory(p))
@@ -658,7 +658,7 @@ def compiler_environment(self):
spack.util.module_cmd.load_module(module)
# apply other compiler environment changes
env = spack.util.environment.EnvironmentModifications()
env = llnl.util.envmod.EnvironmentModifications()
env.extend(spack.schema.environment.parse(self.environment))
env.apply_modifications()

2
lib/spack/spack/compilers/__init__.py
View File

@@ -17,6 +17,7 @@
import llnl.util.filesystem as fs
import llnl.util.lang
import llnl.util.tty as tty
from llnl.util.envmod import get_path
import spack.compiler
import spack.config
@@ -24,7 +25,6 @@
import spack.paths
import spack.platforms
import spack.spec
from spack.util.environment import get_path
from spack.util.naming import mod_to_class
_path_instance_vars = ["cc", "cxx", "f77", "fc"]

2
lib/spack/spack/detection/common.py
View File

@@ -347,7 +347,7 @@ def find_win32_additional_install_paths():
windows_search_ext.extend(
spack.config.get("config:additional_external_search_paths", default=[])
)
windows_search_ext.extend(spack.util.environment.get_path("PATH"))
windows_search_ext.extend(llnl.util.envmod.get_path("PATH"))
return windows_search_ext

12
lib/spack/spack/detection/path.py
View File

@@ -12,10 +12,10 @@
import sys
import warnings
import llnl.util.envmod
import llnl.util.filesystem
import llnl.util.tty
import spack.util.environment
import spack.util.ld_so_conf
from .common import ( # find_windows_compiler_bundled_packages,
@@ -83,9 +83,9 @@ def libraries_in_ld_and_system_library_path(path_hints=None):
"""
path_hints = (
path_hints
or spack.util.environment.get_path("LD_LIBRARY_PATH")
+ spack.util.environment.get_path("DYLD_LIBRARY_PATH")
+ spack.util.environment.get_path("DYLD_FALLBACK_LIBRARY_PATH")
or llnl.util.envmod.get_path("LD_LIBRARY_PATH")
+ llnl.util.envmod.get_path("DYLD_LIBRARY_PATH")
+ llnl.util.envmod.get_path("DYLD_FALLBACK_LIBRARY_PATH")
+ spack.util.ld_so_conf.host_dynamic_linker_search_paths()
)
search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
@@ -93,7 +93,7 @@ def libraries_in_ld_and_system_library_path(path_hints=None):
def libraries_in_windows_paths(path_hints):
path_hints.extend(spack.util.environment.get_path("PATH"))
path_hints.extend(llnl.util.envmod.get_path("PATH"))
search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
# on Windows, some libraries (.dlls) are found in the bin directory or sometimes
# at the search root. Add both of those options to the search scheme
@@ -236,7 +236,7 @@ def by_executable(packages_to_check, path_hints=None):
path_hints (list): list of paths to be searched. If None the list will be
constructed based on the PATH environment variable.
"""
path_hints = spack.util.environment.get_path("PATH") if path_hints is None else path_hints
path_hints = llnl.util.envmod.get_path("PATH") if path_hints is None else path_hints
exe_pattern_to_pkgs = collections.defaultdict(list)
for pkg in packages_to_check:
if hasattr(pkg, "executables"):

3
lib/spack/spack/directory_layout.py
View File

@@ -20,6 +20,7 @@
import spack.spec
import spack.util.spack_json as sjson
from spack.error import SpackError
from spack.util.environment import get_host_environment_metadata
is_windows = sys.platform == "win32"
# Note: Posixpath is used here as opposed to
@@ -120,8 +121,6 @@ def write_host_environment(self, spec):
versioning. We use it in the case that an analysis later needs to
easily access this information.
"""
from spack.util.environment import get_host_environment_metadata
env_file = self.env_metadata_path(spec)
environ = get_host_environment_metadata()
with open(env_file, "w") as fd:

7
lib/spack/spack/environment/environment.py
View File

@@ -16,6 +16,7 @@
import ruamel.yaml as yaml
import llnl.util.envmod
import llnl.util.filesystem as fs
import llnl.util.tty as tty
from llnl.util.lang import dedupe
@@ -1535,7 +1536,7 @@ def check_views(self):
)
def _env_modifications_for_default_view(self, reverse=False):
all_mods = spack.util.environment.EnvironmentModifications()
all_mods = llnl.util.envmod.EnvironmentModifications()
visited = set()
@@ -1576,7 +1577,7 @@ def add_default_view_to_env(self, env_mod):
default view. Removes duplicate paths.
Args:
env_mod (spack.util.environment.EnvironmentModifications): the environment
env_mod (llnl.util.envmod.EnvironmentModifications): the environment
modifications object that is modified.
"""
if default_view_name not in self.views:
@@ -1603,7 +1604,7 @@ def rm_default_view_from_env(self, env_mod):
default view. Reverses the action of ``add_default_view_to_env``.
Args:
env_mod (spack.util.environment.EnvironmentModifications): the environment
env_mod (llnl.util.envmod.EnvironmentModifications): the environment
modifications object that is modified.
"""
if default_view_name not in self.views:

6
lib/spack/spack/environment/shell.py
View File

@@ -5,12 +5,12 @@
import os
import llnl.util.tty as tty
from llnl.util.envmod import EnvironmentModifications
from llnl.util.tty.color import colorize
import spack.environment as ev
import spack.repo
import spack.store
from spack.util.environment import EnvironmentModifications
def activate_header(env, shell, prompt=None):
@@ -111,7 +111,7 @@ def activate(env, use_env_repo=False, add_view=True):
add_view (bool): generate commands to add view to path variables
Returns:
spack.util.environment.EnvironmentModifications: Environment variables
llnl.util.envmod.EnvironmentModifications: Environment variables
modifications to activate environment.
"""
ev.activate(env, use_env_repo=use_env_repo)
@@ -150,7 +150,7 @@ def deactivate():
after activation are not unloaded.
Returns:
spack.util.environment.EnvironmentModifications: Environment variables
llnl.util.envmod.EnvironmentModifications: Environment variables
modifications to activate environment.
"""
env_mods = EnvironmentModifications()

2
lib/spack/spack/fetch_strategy.py
View File

@@ -42,8 +42,8 @@
temp_rename,
working_dir,
)
from llnl.util.symlink import symlink
from llnl.util.string import comma_and, quote
from llnl.util.symlink import symlink
import spack.config
import spack.error

2
lib/spack/spack/install_test.py
View File

@@ -13,8 +13,8 @@
import spack.error
import spack.paths
import spack.util.prefix
import spack.util.path
import spack.util.prefix
import spack.util.spack_json as sjson
from spack.spec import Spec

3
lib/spack/spack/installer.py
View File

@@ -40,6 +40,7 @@
import llnl.util.filesystem as fs
import llnl.util.lock as lk
import llnl.util.tty as tty
from llnl.util.envmod import EnvironmentModifications
from llnl.util.lang import pretty_seconds
from llnl.util.tty.color import colorize
from llnl.util.tty.log import log_output
@@ -56,7 +57,7 @@
import spack.util.executable
import spack.util.path
import spack.util.timer as timer
from spack.util.environment import EnvironmentModifications, dump_environment
from spack.util.environment import dump_environment
from spack.util.executable import which
#: Counter to support unique spec sequencing that is used to ensure packages

4
lib/spack/spack/main.py
View File

@@ -26,6 +26,7 @@
import archspec.cpu
import llnl.util.envmod
import llnl.util.lang
import llnl.util.tty as tty
import llnl.util.tty.colify
@@ -44,7 +45,6 @@
import spack.spec
import spack.store
import spack.util.debug
import spack.util.environment
import spack.util.git
from spack.error import SpackError
@@ -574,7 +574,7 @@ def setup_main_options(args):
if args.debug:
spack.util.debug.register_interrupt_handler()
spack.config.set("config:debug", True, scope="command_line")
spack.util.environment.tracing_enabled = True
llnl.util.envmod.tracing_enabled = True
if args.timestamp:
tty.set_timestamp(True)

6
lib/spack/spack/modules/common.py
View File

@@ -36,6 +36,7 @@
import re
from typing import Optional
import llnl.util.envmod
import llnl.util.filesystem
import llnl.util.tty as tty
from llnl.util.lang import dedupe
@@ -51,7 +52,6 @@
import spack.schema.environment
import spack.store
import spack.tengine as tengine
import spack.util.environment
import spack.util.file_permissions as fp
import spack.util.path
import spack.util.spack_yaml as syaml
@@ -732,8 +732,8 @@ def environment_modifications(self):
spec.prefix = view.get_projection_for_spec(spec)
env = spack.util.environment.inspect_path(
spec.prefix, prefix_inspections, exclude=spack.util.environment.is_system_path
env = llnl.util.envmod.inspect_path(
spec.prefix, prefix_inspections, exclude=llnl.util.envmod.is_system_path
)
# Let the extendee/dependency modify their extensions/dependencies

4
lib/spack/spack/modules/lmod.py
View File

@@ -9,6 +9,7 @@
import posixpath
from typing import Any, Dict
import llnl.util.envmod
import llnl.util.lang as lang
import spack.compilers
@@ -17,7 +18,6 @@
import spack.repo
import spack.spec
import spack.tengine as tengine
import spack.util.environment
from .common import BaseConfiguration, BaseContext, BaseFileLayout, BaseModuleFileWriter
@@ -71,7 +71,7 @@ def guess_core_compilers(name, store=False):
# A compiler is considered to be a core compiler if any of the
# C, C++ or Fortran compilers reside in a system directory
is_system_compiler = any(
os.path.dirname(x) in spack.util.environment.system_dirs
os.path.dirname(x) in llnl.util.envmod.system_dirs
for x in compiler["paths"].values()
if x is not None
)

2
lib/spack/spack/operating_systems/cray_frontend.py
View File

@@ -10,8 +10,8 @@
import llnl.util.filesystem as fs
import llnl.util.lang
import llnl.util.tty as tty
from llnl.util.envmod import get_path
from spack.util.environment import get_path
from spack.util.module_cmd import module
from .linux_distro import LinuxDistro

8
lib/spack/spack/package_base.py
View File

@@ -29,9 +29,10 @@
import warnings
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type
import llnl.util.envmod
import llnl.util.filesystem as fsys
import llnl.util.tty as tty
import llnl.util.path
import llnl.util.tty as tty
from llnl.util.lang import classproperty, memoized, nullcontext
from llnl.util.link_tree import LinkTree
@@ -52,7 +53,6 @@
import spack.spec
import spack.store
import spack.url
import spack.util.environment
import spack.util.path
import spack.util.web
from spack.filesystem_view import YamlFilesystemView
@@ -2102,7 +2102,7 @@ def setup_run_environment(self, env):
"""Sets up the run environment for a package.
Args:
env (spack.util.environment.EnvironmentModifications): environment
env (llnl.util.envmod.EnvironmentModifications): environment
modifications to be applied when the package is run. Package authors
can call methods on it to alter the run environment.
"""
@@ -2119,7 +2119,7 @@ def setup_dependent_run_environment(self, env, dependent_spec):
for dependencies.
Args:
env (spack.util.environment.EnvironmentModifications): environment
env (llnl.util.envmod.EnvironmentModifications): environment
modifications to be applied when the dependent package is run.
Package authors can call methods on it to alter the build environment.

5
lib/spack/spack/platforms/_functions.py
View File

@@ -4,10 +4,9 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import contextlib
import llnl.util.envmod
import llnl.util.lang
import spack.util.environment
from .cray import Cray
from .darwin import Darwin
from .linux import Linux
@@ -64,7 +63,7 @@ def prevent_cray_detection():
"""Context manager that prevents the detection of the Cray platform"""
reset()
try:
with spack.util.environment.set_env(MODULEPATH=""):
with llnl.util.envmod.set_env(MODULEPATH=""):
yield
finally:
reset()

4
lib/spack/spack/schema/environment.py
View File

@@ -40,9 +40,9 @@ def parse(config_obj):
config_obj: a configuration dictionary conforming to the
schema definition for environment modifications
"""
import spack.util.environment as ev
import llnl.util.envmod as envmod
env = ev.EnvironmentModifications()
env = envmod.EnvironmentModifications()
for command, variable in config_obj.items():
# Distinguish between commands that take only a name as argument
# (e.g. unset) and commands that take a name and a value.

2
lib/spack/spack/spec.py
View File

@@ -59,10 +59,10 @@
import ruamel.yaml as yaml
import llnl.util.string
import llnl.util.filesystem as fs
import llnl.util.lang as lang
import llnl.util.path as pth
import llnl.util.string
import llnl.util.tty as tty
import llnl.util.tty.color as clr

2
lib/spack/spack/test/build_environment.py
View File

@@ -10,6 +10,7 @@
import pytest
from llnl.util.envmod import EnvironmentModifications
from llnl.util.filesystem import HeaderList, LibraryList
from llnl.util.path import Path, convert_to_platform_path
@@ -24,7 +25,6 @@
dso_suffix,
)
from spack.paths import build_env_path
from spack.util.environment import EnvironmentModifications
from spack.util.executable import Executable

3
lib/spack/spack/test/cc.py
View File

@@ -12,11 +12,12 @@
import pytest
from llnl.util.envmod import set_env, system_dirs
import spack.build_environment
import spack.config
import spack.spec
from spack.paths import build_env_path
from spack.util.environment import set_env, system_dirs
from spack.util.executable import Executable, ProcessError
#

2
lib/spack/spack/test/cmd/env.py
View File

@@ -638,7 +638,7 @@ def test_env_view_external_prefix(tmpdir_factory, mutable_database, mock_package
e.install_all()
e.write()
env_mod = spack.util.environment.EnvironmentModifications()
env_mod = llnl.util.envmod.EnvironmentModifications()
e.add_default_view_to_env(env_mod)
env_variables = {}
env_mod.apply_modifications(env_variables)

6
lib/spack/spack/test/compilers/basics.py
View File

@@ -10,12 +10,12 @@
import pytest
import llnl.util.envmod
import llnl.util.filesystem as fs
import spack.compiler
import spack.compilers as compilers
import spack.spec
import spack.util.environment
from spack.compiler import Compiler
from spack.util.executable import ProcessError
@@ -868,7 +868,7 @@ class MockPackage(object):
"x86_64",
["/usr/bin/clang", "/usr/bin/clang++", None, None],
)
env = spack.util.environment.EnvironmentModifications()
env = llnl.util.envmod.EnvironmentModifications()
# Check a package that doesn't use xcode and ensure we don't add changes
# to the environment
pkg = MockPackage()
@@ -955,7 +955,7 @@ def test_xcode_not_available(xcode_select_output, mock_executable, monkeypatch):
"x86_64",
["/usr/bin/clang", "/usr/bin/clang++", None, None],
)
env = spack.util.environment.EnvironmentModifications()
env = llnl.util.envmod.EnvironmentModifications()
class MockPackage(object):
use_xcode = True

77
lib/spack/spack/test/environment_modifications.py
View File

@@ -8,18 +8,9 @@
import pytest
import spack.util.environment as environment
import llnl.util.envmod as envmod
from spack.paths import spack_root
from spack.util.environment import (
AppendPath,
EnvironmentModifications,
PrependPath,
RemovePath,
SetEnv,
UnsetEnv,
filter_system_paths,
is_system_path,
)
datadir = os.path.join(spack_root, "lib", "spack", "spack", "test", "data")
@@ -43,7 +34,7 @@ def test_inspect_path(tmpdir):
tmpdir.mkdir("lib")
tmpdir.mkdir("include")
env = environment.inspect_path(str(tmpdir), inspections)
env = envmod.inspect_path(str(tmpdir), inspections)
names = [item.name for item in env]
assert "PATH" in names
assert "LIBRARY_PATH" in names
@@ -58,7 +49,7 @@ def test_exclude_paths_from_inspection():
"include": ["CPATH"],
}
env = environment.inspect_path("/usr", inspections, exclude=is_system_path)
env = envmod.inspect_path("/usr", inspections, exclude=envmod.is_system_path)
assert len(env) == 0
@@ -79,7 +70,7 @@ def prepare_environment_for_tests(working_env):
@pytest.fixture
def env(prepare_environment_for_tests):
"""Returns an empty EnvironmentModifications object."""
return EnvironmentModifications()
return envmod.EnvironmentModifications()
@pytest.fixture
@@ -162,7 +153,7 @@ def test_unset(env):
@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)")
def test_filter_system_paths(miscellaneous_paths):
"""Tests that the filtering of system paths works as expected."""
filtered = filter_system_paths(miscellaneous_paths)
filtered = envmod.filter_system_paths(miscellaneous_paths)
expected = [
"/usr/local/Cellar/gcc/5.3.0/lib",
"/usr/local/opt/some-package/lib",
@@ -246,7 +237,7 @@ def test_extend(env):
"""
env.set("A", "dummy value")
env.set("B", 3)
copy_construct = EnvironmentModifications(env)
copy_construct = envmod.EnvironmentModifications(env)
assert len(copy_construct) == 2
@@ -260,12 +251,12 @@ def test_source_files(files_to_be_sourced):
"""Tests the construction of a list of environment modifications that are
the result of sourcing a file.
"""
env = EnvironmentModifications()
env = envmod.EnvironmentModifications()
for filename in files_to_be_sourced:
if filename.endswith("sourceme_parameters.sh"):
env.extend(EnvironmentModifications.from_sourcing_file(filename, "intel64"))
env.extend(envmod.EnvironmentModifications.from_sourcing_file(filename, "intel64"))
else:
env.extend(EnvironmentModifications.from_sourcing_file(filename))
env.extend(envmod.EnvironmentModifications.from_sourcing_file(filename))
modifications = env.group_by_name()
@@ -277,28 +268,28 @@ def test_source_files(files_to_be_sourced):
# Set new variables
assert len(modifications["NEW_VAR"]) == 1
assert isinstance(modifications["NEW_VAR"][0], SetEnv)
assert isinstance(modifications["NEW_VAR"][0], envmod.SetEnv)
assert modifications["NEW_VAR"][0].value == "new"
assert len(modifications["FOO"]) == 1
assert isinstance(modifications["FOO"][0], SetEnv)
assert isinstance(modifications["FOO"][0], envmod.SetEnv)
assert modifications["FOO"][0].value == "intel64"
# Unset variables
assert len(modifications["EMPTY_PATH_LIST"]) == 1
assert isinstance(modifications["EMPTY_PATH_LIST"][0], UnsetEnv)
assert isinstance(modifications["EMPTY_PATH_LIST"][0], envmod.UnsetEnv)
# Modified variables
assert len(modifications["UNSET_ME"]) == 1
assert isinstance(modifications["UNSET_ME"][0], SetEnv)
assert isinstance(modifications["UNSET_ME"][0], envmod.SetEnv)
assert modifications["UNSET_ME"][0].value == "overridden"
assert len(modifications["PATH_LIST"]) == 3
assert isinstance(modifications["PATH_LIST"][0], RemovePath)
assert isinstance(modifications["PATH_LIST"][0], envmod.RemovePath)
assert modifications["PATH_LIST"][0].value == "/path/third"
assert isinstance(modifications["PATH_LIST"][1], AppendPath)
assert isinstance(modifications["PATH_LIST"][1], envmod.AppendPath)
assert modifications["PATH_LIST"][1].value == "/path/fourth"
assert isinstance(modifications["PATH_LIST"][2], PrependPath)
assert isinstance(modifications["PATH_LIST"][2], envmod.PrependPath)
assert modifications["PATH_LIST"][2].value == "/path/first"
@@ -307,7 +298,7 @@ def test_preserve_environment(prepare_environment_for_tests):
# UNSET_ME is defined, and will be unset in the context manager,
# NOT_SET is not in the environment and will be set within the
# context manager, PATH_LIST is set and will be changed.
with environment.preserve_environment("UNSET_ME", "NOT_SET", "PATH_LIST"):
with envmod.preserve_environment("UNSET_ME", "NOT_SET", "PATH_LIST"):
os.environ["NOT_SET"] = "a"
assert os.environ["NOT_SET"] == "a"
@@ -363,7 +354,7 @@ def test_preserve_environment(prepare_environment_for_tests):
@pytest.mark.usefixtures("prepare_environment_for_tests")
def test_environment_from_sourcing_files(files, expected, deleted):
env = environment.environment_after_sourcing_files(*files)
env = envmod.environment_after_sourcing_files(*files)
# Test that variables that have been modified are still there and contain
# the expected output
@@ -394,7 +385,7 @@ def test_clear(env):
)
def test_sanitize_literals(env, exclude, include):
after = environment.sanitize(env, exclude, include)
after = envmod.sanitize(env, exclude, include)
# Check that all the included variables are there
assert all(x in after for x in include)
@@ -431,7 +422,7 @@ def test_sanitize_literals(env, exclude, include):
)
def test_sanitize_regex(env, exclude, include, expected, deleted):
after = environment.sanitize(env, exclude, include)
after = envmod.sanitize(env, exclude, include)
assert all(x in after for x in expected)
assert all(x not in after for x in deleted)
@@ -442,39 +433,39 @@ def test_sanitize_regex(env, exclude, include, expected, deleted):
"before,after,search_list",
[
# Set environment variables
({}, {"FOO": "foo"}, [environment.SetEnv("FOO", "foo")]),
({}, {"FOO": "foo"}, [envmod.SetEnv("FOO", "foo")]),
# Unset environment variables
({"FOO": "foo"}, {}, [environment.UnsetEnv("FOO")]),
({"FOO": "foo"}, {}, [envmod.UnsetEnv("FOO")]),
# Append paths to an environment variable
(
{"FOO_PATH": "/a/path"},
{"FOO_PATH": "/a/path:/b/path"},
[environment.AppendPath("FOO_PATH", "/b/path")],
[envmod.AppendPath("FOO_PATH", "/b/path")],
),
(
{},
{"FOO_PATH": "/a/path" + os.sep + "/b/path"},
[environment.AppendPath("FOO_PATH", "/a/path" + os.sep + "/b/path")],
[envmod.AppendPath("FOO_PATH", "/a/path" + os.sep + "/b/path")],
),
(
{"FOO_PATH": "/a/path:/b/path"},
{"FOO_PATH": "/b/path"},
[environment.RemovePath("FOO_PATH", "/a/path")],
[envmod.RemovePath("FOO_PATH", "/a/path")],
),
(
{"FOO_PATH": "/a/path:/b/path"},
{"FOO_PATH": "/a/path:/c/path"},
[
environment.RemovePath("FOO_PATH", "/b/path"),
environment.AppendPath("FOO_PATH", "/c/path"),
envmod.RemovePath("FOO_PATH", "/b/path"),
envmod.AppendPath("FOO_PATH", "/c/path"),
],
),
(
{"FOO_PATH": "/a/path:/b/path"},
{"FOO_PATH": "/c/path:/a/path"},
[
environment.RemovePath("FOO_PATH", "/b/path"),
environment.PrependPath("FOO_PATH", "/c/path"),
envmod.RemovePath("FOO_PATH", "/b/path"),
envmod.PrependPath("FOO_PATH", "/c/path"),
],
),
# Modify two variables in the same environment
@@ -482,15 +473,15 @@ def test_sanitize_regex(env, exclude, include, expected, deleted):
{"FOO": "foo", "BAR": "bar"},
{"FOO": "baz", "BAR": "baz"},
[
environment.SetEnv("FOO", "baz"),
environment.SetEnv("BAR", "baz"),
envmod.SetEnv("FOO", "baz"),
envmod.SetEnv("BAR", "baz"),
],
),
],
)
def test_from_environment_diff(before, after, search_list):
mod = environment.EnvironmentModifications.from_environment_diff(before, after)
mod = envmod.EnvironmentModifications.from_environment_diff(before, after)
for item in search_list:
assert item in mod
@@ -501,7 +492,7 @@ def test_from_environment_diff(before, after, search_list):
def test_exclude_lmod_variables():
# Construct the list of environment modifications
file = os.path.join(datadir, "sourceme_lmod.sh")
env = EnvironmentModifications.from_sourcing_file(file)
env = envmod.EnvironmentModifications.from_sourcing_file(file)
# Check that variables related to lmod are not in there
modifications = env.group_by_name()

138
lib/spack/spack/test/llnl/util/envmod.py Normal file
View File

@@ -0,0 +1,138 @@
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Test Spack's environment utility functions."""
import os
import sys
import pytest
import llnl.util.envmod as envmod
is_windows = sys.platform == "win32"
def test_is_system_path():
sys_path = "C:\\Users" if is_windows else "/usr/bin"
assert envmod.is_system_path(sys_path)
assert not envmod.is_system_path("/nonsense_path/bin")
assert not envmod.is_system_path("")
assert not envmod.is_system_path(None)
if is_windows:
test_paths = [
"C:\\Users",
"C:\\",
"C:\\ProgramData",
"C:\\nonsense_path",
"C:\\Program Files",
"C:\\nonsense_path\\extra\\bin",
]
else:
test_paths = [
"/usr/bin",
"/nonsense_path/lib",
"/usr/local/lib",
"/bin",
"/nonsense_path/extra/bin",
"/usr/lib64",
]
def test_filter_system_paths():
nonsense_prefix = "C:\\nonsense_path" if is_windows else "/nonsense_path"
expected = [p for p in test_paths if p.startswith(nonsense_prefix)]
filtered = envmod.filter_system_paths(test_paths)
assert expected == filtered
def deprioritize_system_paths():
expected = [p for p in test_paths if p.startswith("/nonsense_path")]
expected.extend([p for p in test_paths if not p.startswith("/nonsense_path")])
filtered = envmod.deprioritize_system_paths(test_paths)
assert expected == filtered
def test_prune_duplicate_paths():
test_paths = ["/a/b", "/a/c", "/a/b", "/a/a", "/a/c", "/a/a/.."]
expected = ["/a/b", "/a/c", "/a/a", "/a/a/.."]
assert expected == envmod.prune_duplicate_paths(test_paths)
def test_get_path(working_env):
os.environ["TEST_ENV_VAR"] = os.pathsep.join(["/a", "/b", "/c/d"])
expected = ["/a", "/b", "/c/d"]
assert envmod.get_path("TEST_ENV_VAR") == expected
def test_env_flag(working_env):
assert not envmod.env_flag("TEST_NO_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "1"
assert envmod.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "TRUE"
assert envmod.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "True"
assert envmod.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "TRue"
assert envmod.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "true"
assert envmod.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "27"
assert not envmod.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "-2.3"
assert not envmod.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "0"
assert not envmod.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "False"
assert not envmod.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "false"
assert not envmod.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "garbage"
assert not envmod.env_flag("TEST_ENV_VAR")
def test_path_set(working_env):
envmod.path_set("TEST_ENV_VAR", ["/a", "/a/b", "/a/a"])
assert os.environ["TEST_ENV_VAR"] == "/a" + os.pathsep + "/a/b" + os.pathsep + "/a/a"
def test_path_put_first(working_env):
envmod.path_set("TEST_ENV_VAR", test_paths)
expected = ["/usr/bin", "/new_nonsense_path/a/b"]
expected.extend([p for p in test_paths if p != "/usr/bin"])
envmod.path_put_first("TEST_ENV_VAR", expected)
assert envmod.get_path("TEST_ENV_VAR") == expected
def test_reverse_environment_modifications(working_env):
start_env = {
"PREPEND_PATH": os.sep + os.path.join("path", "to", "prepend", "to"),
"APPEND_PATH": os.sep + os.path.join("path", "to", "append", "to"),
"UNSET": "var_to_unset",
"APPEND_FLAGS": "flags to append to",
}
to_reverse = envmod.EnvironmentModifications()
to_reverse.prepend_path("PREPEND_PATH", "/new/path/prepended")
to_reverse.append_path("APPEND_PATH", "/new/path/appended")
to_reverse.set_path("SET_PATH", ["/one/set/path", "/two/set/path"])
to_reverse.set("SET", "a var")
to_reverse.unset("UNSET")
to_reverse.append_flags("APPEND_FLAGS", "more_flags")
reversal = to_reverse.reversed()
os.environ = start_env.copy()
print(os.environ)
to_reverse.apply_modifications()
print(os.environ)
reversal.apply_modifications()
print(os.environ)
start_env.pop("UNSET")
assert os.environ == start_env

9
lib/spack/spack/test/llnl/util/path.py
View File

@@ -3,15 +3,18 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import llnl.util.path as lup
import os
import sys
import llnl.util.path
def test_sanitze_file_path(tmpdir):
"""Test filtering illegal characters out of potential file paths"""
# *nix illegal files characters are '/' and none others
illegal_file_path = str(tmpdir) + "//" + "abcdefghi.txt"
if is_windows:
if sys.platform == "win32":
# Windows has a larger set of illegal characters
illegal_file_path = os.path.join(tmpdir, 'a<b>cd?e:f"g|h*i.txt')
real_path = lup.sanitize_file_path(illegal_file_path)
real_path = llnl.util.path.sanitize_file_path(illegal_file_path)
assert real_path == os.path.join(str(tmpdir), "abcdefghi.txt")

3
lib/spack/spack/test/make_executable.py
View File

@@ -16,8 +16,9 @@
import pytest
from llnl.util.envmod import path_put_first
from spack.build_environment import MakeExecutable
from spack.util.environment import path_put_first
pytestmark = pytest.mark.skipif(
sys.platform == "win32",

4
lib/spack/spack/test/modules/lmod.py
View File

@@ -7,6 +7,8 @@
import pytest
import llnl.util.envmod
import spack.environment as ev
import spack.main
import spack.modules.lmod
@@ -240,7 +242,7 @@ def test_guess_core_compilers(self, factory, module_configuration, monkeypatch):
module_configuration("missing_core_compilers")
# Our mock paths must be detected as system paths
monkeypatch.setattr(spack.util.environment, "system_dirs", ["/path/to"])
monkeypatch.setattr(llnl.util.envmod, "system_dirs", ["/path/to"])
# We don't want to really write into user configuration
# when running tests

144
lib/spack/spack/test/util/environment.py
View File

@@ -3,153 +3,15 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Test Spack's environment utility functions."""
import os
import sys
import pytest
import spack.util.environment as envutil
is_windows = sys.platform == "win32"
from spack.util.environment import dump_environment
@pytest.fixture()
def prepare_environment_for_tests():
if "TEST_ENV_VAR" in os.environ:
del os.environ["TEST_ENV_VAR"]
yield
del os.environ["TEST_ENV_VAR"]
def test_is_system_path():
sys_path = "C:\\Users" if is_windows else "/usr/bin"
assert envutil.is_system_path(sys_path)
assert not envutil.is_system_path("/nonsense_path/bin")
assert not envutil.is_system_path("")
assert not envutil.is_system_path(None)
if is_windows:
test_paths = [
"C:\\Users",
"C:\\",
"C:\\ProgramData",
"C:\\nonsense_path",
"C:\\Program Files",
"C:\\nonsense_path\\extra\\bin",
]
else:
test_paths = [
"/usr/bin",
"/nonsense_path/lib",
"/usr/local/lib",
"/bin",
"/nonsense_path/extra/bin",
"/usr/lib64",
]
def test_filter_system_paths():
nonsense_prefix = "C:\\nonsense_path" if is_windows else "/nonsense_path"
expected = [p for p in test_paths if p.startswith(nonsense_prefix)]
filtered = envutil.filter_system_paths(test_paths)
assert expected == filtered
def deprioritize_system_paths():
expected = [p for p in test_paths if p.startswith("/nonsense_path")]
expected.extend([p for p in test_paths if not p.startswith("/nonsense_path")])
filtered = envutil.deprioritize_system_paths(test_paths)
assert expected == filtered
def test_prune_duplicate_paths():
test_paths = ["/a/b", "/a/c", "/a/b", "/a/a", "/a/c", "/a/a/.."]
expected = ["/a/b", "/a/c", "/a/a", "/a/a/.."]
assert expected == envutil.prune_duplicate_paths(test_paths)
def test_get_path(prepare_environment_for_tests):
os.environ["TEST_ENV_VAR"] = os.pathsep.join(["/a", "/b", "/c/d"])
expected = ["/a", "/b", "/c/d"]
assert envutil.get_path("TEST_ENV_VAR") == expected
def test_env_flag(prepare_environment_for_tests):
assert not envutil.env_flag("TEST_NO_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "1"
assert envutil.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "TRUE"
assert envutil.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "True"
assert envutil.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "TRue"
assert envutil.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "true"
assert envutil.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "27"
assert not envutil.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "-2.3"
assert not envutil.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "0"
assert not envutil.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "False"
assert not envutil.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "false"
assert not envutil.env_flag("TEST_ENV_VAR")
os.environ["TEST_ENV_VAR"] = "garbage"
assert not envutil.env_flag("TEST_ENV_VAR")
def test_path_set(prepare_environment_for_tests):
envutil.path_set("TEST_ENV_VAR", ["/a", "/a/b", "/a/a"])
assert os.environ["TEST_ENV_VAR"] == "/a" + os.pathsep + "/a/b" + os.pathsep + "/a/a"
def test_path_put_first(prepare_environment_for_tests):
envutil.path_set("TEST_ENV_VAR", test_paths)
expected = ["/usr/bin", "/new_nonsense_path/a/b"]
expected.extend([p for p in test_paths if p != "/usr/bin"])
envutil.path_put_first("TEST_ENV_VAR", expected)
assert envutil.get_path("TEST_ENV_VAR") == expected
def test_dump_environment(prepare_environment_for_tests, tmpdir):
def test_dump_environment(working_env, tmpdir):
test_paths = "/a:/b/x:/b/c"
os.environ["TEST_ENV_VAR"] = test_paths
dumpfile_path = str(tmpdir.join("envdump.txt"))
envutil.dump_environment(dumpfile_path)
dump_environment(dumpfile_path)
with open(dumpfile_path, "r") as dumpfile:
assert "TEST_ENV_VAR={0}; export TEST_ENV_VAR\n".format(test_paths) in list(dumpfile)
def test_reverse_environment_modifications(working_env):
start_env = {
"PREPEND_PATH": os.sep + os.path.join("path", "to", "prepend", "to"),
"APPEND_PATH": os.sep + os.path.join("path", "to", "append", "to"),
"UNSET": "var_to_unset",
"APPEND_FLAGS": "flags to append to",
}
to_reverse = envutil.EnvironmentModifications()
to_reverse.prepend_path("PREPEND_PATH", "/new/path/prepended")
to_reverse.append_path("APPEND_PATH", "/new/path/appended")
to_reverse.set_path("SET_PATH", ["/one/set/path", "/two/set/path"])
to_reverse.set("SET", "a var")
to_reverse.unset("UNSET")
to_reverse.append_flags("APPEND_FLAGS", "more_flags")
reversal = to_reverse.reversed()
os.environ = start_env.copy()
print(os.environ)
to_reverse.apply_modifications()
print(os.environ)
reversal.apply_modifications()
print(os.environ)
start_env.pop("UNSET")
assert os.environ == start_env

9
lib/spack/spack/user_environment.py
View File

@@ -5,9 +5,10 @@
import os
import sys
import llnl.util.envmod as envmod
import spack.build_environment
import spack.config
import spack.util.environment as environment
import spack.util.prefix as prefix
#: Environment variable name Spack uses to track individually loaded packages
@@ -52,7 +53,7 @@ def unconditional_environment_modifications(view):
"""List of environment (shell) modifications to be processed for view.
This list does not depend on the specs in this environment"""
env = environment.EnvironmentModifications()
env = envmod.EnvironmentModifications()
for subdir, vars in prefix_inspections(sys.platform).items():
full_subdir = os.path.join(view.root, subdir)
@@ -81,8 +82,8 @@ def environment_modifications_for_spec(spec, view=None, set_package_py_globals=T
# generic environment modifications determined by inspecting the spec
# prefix
env = environment.inspect_path(
spec.prefix, prefix_inspections(spec.platform), exclude=environment.is_system_path
env = envmod.inspect_path(
spec.prefix, prefix_inspections(spec.platform), exclude=envmod.is_system_path
)
# Let the extendee/dependency modify their extensions/dependents

977
lib/spack/spack/util/environment.py
View File

@@ -2,122 +2,18 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Utilities for setting and modifying environment variables."""
import collections
import contextlib
import inspect
import json
"""Spack-specific environment metadata Utilities."""
import os
import os.path
import pickle
import platform
import re
import shlex
import socket
import sys
import llnl.util.tty as tty
from llnl.util.lang import dedupe
from llnl.util.path import path_to_os_path, system_path_filter
from llnl.util.path import system_path_filter
import spack.config
import spack.platforms
import spack.spec
import spack.util.executable as executable
is_windows = sys.platform == "win32"
system_paths = (
["/", "/usr", "/usr/local"]
if not is_windows
else ["C:\\", "C:\\Program Files", "C:\\Program Files (x86)", "C:\\Users", "C:\\ProgramData"]
)
suffixes = ["bin", "bin64", "include", "lib", "lib64"] if not is_windows else []
system_dirs = [os.path.join(p, s) for s in suffixes for p in system_paths] + system_paths
_shell_set_strings = {
"sh": "export {0}={1};\n",
"csh": "setenv {0} {1};\n",
"fish": "set -gx {0} {1};\n",
"bat": 'set "{0}={1}"\n',
}
_shell_unset_strings = {
"sh": "unset {0};\n",
"csh": "unsetenv {0};\n",
"fish": "set -e {0};\n",
"bat": 'set "{0}="\n',
}
tracing_enabled = False
def is_system_path(path):
"""Predicate that given a path returns True if it is a system path,
False otherwise.
Args:
path (str): path to a directory
Returns:
True or False
"""
return path and os.path.normpath(path) in system_dirs
def filter_system_paths(paths):
"""Return only paths that are not system paths."""
return [p for p in paths if not is_system_path(p)]
def deprioritize_system_paths(paths):
"""Put system paths at the end of paths, otherwise preserving order."""
filtered_paths = filter_system_paths(paths)
fp = set(filtered_paths)
return filtered_paths + [p for p in paths if p not in fp]
def prune_duplicate_paths(paths):
"""Returns the paths with duplicates removed, order preserved."""
return list(dedupe(paths))
def get_path(name):
path = os.environ.get(name, "").strip()
if path:
return path.split(os.pathsep)
else:
return []
def env_flag(name):
if name in os.environ:
value = os.environ[name].lower()
return value == "true" or value == "1"
return False
def path_set(var_name, directories):
path_str = os.pathsep.join(str(dir) for dir in directories)
os.environ[var_name] = path_str
def path_put_first(var_name, directories):
"""Puts the provided directories first in the path, adding them
if they're not already there.
"""
path = os.environ.get(var_name, "").split(os.pathsep)
for dir in directories:
if dir in path:
path.remove(dir)
new_path = tuple(directories) + tuple(path)
path_set(var_name, new_path)
bash_function_finder = re.compile(r"BASH_FUNC_(.*?)\(\)")
@@ -189,872 +85,3 @@ def get_host_environment():
"arch_str": str(arch_spec),
"hostname": socket.gethostname(),
}
@contextlib.contextmanager
def set_env(**kwargs):
"""Temporarily sets and restores environment variables.
Variables can be set as keyword arguments to this function.
"""
saved = {}
for var, value in kwargs.items():
if var in os.environ:
saved[var] = os.environ[var]
if value is None:
if var in os.environ:
del os.environ[var]
else:
os.environ[var] = value
yield
for var, value in kwargs.items():
if var in saved:
os.environ[var] = saved[var]
else:
if var in os.environ:
del os.environ[var]
class NameModifier(object):
def __init__(self, name, **kwargs):
self.name = name
self.separator = kwargs.get("separator", os.pathsep)
self.args = {"name": name, "separator": self.separator}
self.args.update(kwargs)
def __eq__(self, other):
if not isinstance(other, NameModifier):
return False
return self.name == other.name
def update_args(self, **kwargs):
self.__dict__.update(kwargs)
self.args.update(kwargs)
class NameValueModifier(object):
def __init__(self, name, value, **kwargs):
self.name = name
self.value = value
self.separator = kwargs.get("separator", os.pathsep)
self.args = {"name": name, "value": value, "separator": self.separator}
self.args.update(kwargs)
def __eq__(self, other):
if not isinstance(other, NameValueModifier):
return False
return (
self.name == other.name
and self.value == other.value
and self.separator == other.separator
)
def update_args(self, **kwargs):
self.__dict__.update(kwargs)
self.args.update(kwargs)
class SetEnv(NameValueModifier):
def execute(self, env):
tty.debug("SetEnv: {0}={1}".format(self.name, str(self.value)), level=3)
env[self.name] = str(self.value)
class AppendFlagsEnv(NameValueModifier):
def execute(self, env):
tty.debug("AppendFlagsEnv: {0}={1}".format(self.name, str(self.value)), level=3)
if self.name in env and env[self.name]:
env[self.name] += self.separator + str(self.value)
else:
env[self.name] = str(self.value)
class UnsetEnv(NameModifier):
def execute(self, env):
tty.debug("UnsetEnv: {0}".format(self.name), level=3)
# Avoid throwing if the variable was not set
env.pop(self.name, None)
class RemoveFlagsEnv(NameValueModifier):
def execute(self, env):
tty.debug("RemoveFlagsEnv: {0}-{1}".format(self.name, str(self.value)), level=3)
environment_value = env.get(self.name, "")
flags = environment_value.split(self.separator) if environment_value else []
flags = [f for f in flags if f != self.value]
env[self.name] = self.separator.join(flags)
class SetPath(NameValueModifier):
def execute(self, env):
string_path = concatenate_paths(self.value, separator=self.separator)
tty.debug("SetPath: {0}={1}".format(self.name, string_path), level=3)
env[self.name] = string_path
class AppendPath(NameValueModifier):
def execute(self, env):
tty.debug("AppendPath: {0}+{1}".format(self.name, str(self.value)), level=3)
environment_value = env.get(self.name, "")
directories = environment_value.split(self.separator) if environment_value else []
directories.append(path_to_os_path(os.path.normpath(self.value)).pop())
env[self.name] = self.separator.join(directories)
class PrependPath(NameValueModifier):
def execute(self, env):
tty.debug("PrependPath: {0}+{1}".format(self.name, str(self.value)), level=3)
environment_value = env.get(self.name, "")
directories = environment_value.split(self.separator) if environment_value else []
directories = [path_to_os_path(os.path.normpath(self.value)).pop()] + directories
env[self.name] = self.separator.join(directories)
class RemovePath(NameValueModifier):
def execute(self, env):
tty.debug("RemovePath: {0}-{1}".format(self.name, str(self.value)), level=3)
environment_value = env.get(self.name, "")
directories = environment_value.split(self.separator) if environment_value else []
directories = [
path_to_os_path(os.path.normpath(x)).pop()
for x in directories
if x != path_to_os_path(os.path.normpath(self.value)).pop()
]
env[self.name] = self.separator.join(directories)
class DeprioritizeSystemPaths(NameModifier):
def execute(self, env):
tty.debug("DeprioritizeSystemPaths: {0}".format(self.name), level=3)
environment_value = env.get(self.name, "")
directories = environment_value.split(self.separator) if environment_value else []
directories = deprioritize_system_paths(
[path_to_os_path(os.path.normpath(x)).pop() for x in directories]
)
env[self.name] = self.separator.join(directories)
class PruneDuplicatePaths(NameModifier):
def execute(self, env):
tty.debug("PruneDuplicatePaths: {0}".format(self.name), level=3)
environment_value = env.get(self.name, "")
directories = environment_value.split(self.separator) if environment_value else []
directories = prune_duplicate_paths(
[path_to_os_path(os.path.normpath(x)).pop() for x in directories]
)
env[self.name] = self.separator.join(directories)
class EnvironmentModifications(object):
"""Keeps track of requests to modify the current environment.
Each call to a method to modify the environment stores the extra
information on the caller in the request:
* 'filename' : filename of the module where the caller is defined
* 'lineno': line number where the request occurred
* 'context' : line of code that issued the request that failed
"""
def __init__(self, other=None, traced=None):
"""Initializes a new instance, copying commands from 'other'
if it is not None.
Args:
other (EnvironmentModifications): list of environment modifications
to be extended (optional)
traced (bool): enable or disable stack trace inspection to log the origin
of the environment modifications.
"""
self.traced = tracing_enabled if traced is None else bool(traced)
self.env_modifications = []
if other is not None:
self.extend(other)
def __iter__(self):
return iter(self.env_modifications)
def __len__(self):
return len(self.env_modifications)
def extend(self, other):
self._check_other(other)
self.env_modifications.extend(other.env_modifications)
@staticmethod
def _check_other(other):
if not isinstance(other, EnvironmentModifications):
raise TypeError("other must be an instance of EnvironmentModifications")
def _maybe_trace(self, kwargs):
"""Provide the modification with stack trace info so that we can track its
origin to find issues in packages. This is very slow and expensive."""
if not self.traced:
return
stack = inspect.stack()
try:
_, filename, lineno, _, context, index = stack[2]
context = context[index].strip()
except Exception:
filename = "unknown file"
lineno = "unknown line"
context = "unknown context"
kwargs.update({"filename": filename, "lineno": lineno, "context": context})
def set(self, name, value, **kwargs):
"""Stores a request to set an environment variable.
Args:
name: name of the environment variable to be set
value: value of the environment variable
"""
self._maybe_trace(kwargs)
item = SetEnv(name, value, **kwargs)
self.env_modifications.append(item)
def append_flags(self, name, value, sep=" ", **kwargs):
"""
Stores in the current object a request to append to an env variable
Args:
name: name of the environment variable to be appended to
value: value to append to the environment variable
Appends with spaces separating different additions to the variable
"""
self._maybe_trace(kwargs)
kwargs.update({"separator": sep})
item = AppendFlagsEnv(name, value, **kwargs)
self.env_modifications.append(item)
def unset(self, name, **kwargs):
"""Stores a request to unset an environment variable.
Args:
name: name of the environment variable to be unset
"""
self._maybe_trace(kwargs)
item = UnsetEnv(name, **kwargs)
self.env_modifications.append(item)
def remove_flags(self, name, value, sep=" ", **kwargs):
"""
Stores in the current object a request to remove flags from an
env variable
Args:
name: name of the environment variable to be removed from
value: value to remove to the environment variable
sep: separator to assume for environment variable
"""
self._maybe_trace(kwargs)
kwargs.update({"separator": sep})
item = RemoveFlagsEnv(name, value, **kwargs)
self.env_modifications.append(item)
def set_path(self, name, elements, **kwargs):
"""Stores a request to set a path generated from a list.
Args:
name: name o the environment variable to be set.
elements: elements of the path to set.
"""
self._maybe_trace(kwargs)
item = SetPath(name, elements, **kwargs)
self.env_modifications.append(item)
def append_path(self, name, path, **kwargs):
"""Stores a request to append a path to a path list.
Args:
name: name of the path list in the environment
path: path to be appended
"""
self._maybe_trace(kwargs)
item = AppendPath(name, path, **kwargs)
self.env_modifications.append(item)
def prepend_path(self, name, path, **kwargs):
"""Same as `append_path`, but the path is pre-pended.
Args:
name: name of the path list in the environment
path: path to be pre-pended
"""
self._maybe_trace(kwargs)
item = PrependPath(name, path, **kwargs)
self.env_modifications.append(item)
def remove_path(self, name, path, **kwargs):
"""Stores a request to remove a path from a path list.
Args:
name: name of the path list in the environment
path: path to be removed
"""
self._maybe_trace(kwargs)
item = RemovePath(name, path, **kwargs)
self.env_modifications.append(item)
def deprioritize_system_paths(self, name, **kwargs):
"""Stores a request to deprioritize system paths in a path list,
otherwise preserving the order.
Args:
name: name of the path list in the environment.
"""
self._maybe_trace(kwargs)
item = DeprioritizeSystemPaths(name, **kwargs)
self.env_modifications.append(item)
def prune_duplicate_paths(self, name, **kwargs):
"""Stores a request to remove duplicates from a path list, otherwise
preserving the order.
Args:
name: name of the path list in the environment.
"""
self._maybe_trace(kwargs)
item = PruneDuplicatePaths(name, **kwargs)
self.env_modifications.append(item)
def group_by_name(self):
"""Returns a dict of the modifications grouped by variable name.
Returns:
dict mapping the environment variable name to the modifications to
be done on it
"""
modifications = collections.defaultdict(list)
for item in self:
modifications[item.name].append(item)
return modifications
def is_unset(self, var_name):
modifications = self.group_by_name()
var_updates = modifications.get(var_name, None)
if not var_updates:
# We did not explicitly unset it
return False
# The last modification must unset the variable for it to be considered
# unset
return type(var_updates[-1]) == UnsetEnv
def clear(self):
"""
Clears the current list of modifications
"""
self.env_modifications = []
def reversed(self):
"""
Returns the EnvironmentModifications object that will reverse self
Only creates reversals for additions to the environment, as reversing
``unset`` and ``remove_path`` modifications is impossible.
Reversable operations are set(), prepend_path(), append_path(),
set_path(), and append_flags().
"""
rev = EnvironmentModifications()
for envmod in reversed(self.env_modifications):
if type(envmod) == SetEnv:
tty.debug("Reversing `Set` environment operation may lose " "original value")
rev.unset(envmod.name)
elif type(envmod) == AppendPath:
rev.remove_path(envmod.name, envmod.value)
elif type(envmod) == PrependPath:
rev.remove_path(envmod.name, envmod.value)
elif type(envmod) == SetPath:
tty.debug("Reversing `SetPath` environment operation may lose " "original value")
rev.unset(envmod.name)
elif type(envmod) == AppendFlagsEnv:
rev.remove_flags(envmod.name, envmod.value)
else:
# This is an un-reversable operation
tty.warn(
"Skipping reversal of unreversable operation"
"%s %s" % (type(envmod), envmod.name)
)
return rev
def apply_modifications(self, env=None):
"""Applies the modifications and clears the list."""
# Use os.environ if not specified
# Do not copy, we want to modify it in place
if env is None:
env = os.environ
modifications = self.group_by_name()
# Apply modifications one variable at a time
for name, actions in sorted(modifications.items()):
for x in actions:
x.execute(env)
def shell_modifications(self, shell="sh", explicit=False, env=None):
"""Return shell code to apply the modifications and clears the list."""
modifications = self.group_by_name()
if env is None:
env = os.environ
new_env = env.copy()
for name, actions in sorted(modifications.items()):
for x in actions:
x.execute(new_env)
if "MANPATH" in new_env and not new_env.get("MANPATH").endswith(":"):
new_env["MANPATH"] += ":"
cmds = ""
for name in sorted(set(modifications)):
new = new_env.get(name, None)
old = env.get(name, None)
if explicit or new != old:
if new is None:
cmds += _shell_unset_strings[shell].format(name)
else:
if sys.platform != "win32":
cmd = _shell_set_strings[shell].format(name, shlex.quote(new_env[name]))
else:
cmd = _shell_set_strings[shell].format(name, new_env[name])
cmds += cmd
return cmds
@staticmethod
def from_sourcing_file(filename, *arguments, **kwargs):
"""Constructs an instance of a
:py:class:`spack.util.environment.EnvironmentModifications` object
that has the same effect as sourcing a file.
Args:
filename (str): the file to be sourced
*arguments (list): arguments to pass on the command line
Keyword Args:
shell (str): the shell to use (default: ``bash``)
shell_options (str): options passed to the shell (default: ``-c``)
source_command (str): the command to run (default: ``source``)
suppress_output (str): redirect used to suppress output of command
(default: ``&> /dev/null``)
concatenate_on_success (str): operator used to execute a command
only when the previous command succeeds (default: ``&&``)
exclude ([str or re]): ignore any modifications of these
variables (default: [])
include ([str or re]): always respect modifications of these
variables (default: []). Supersedes any excluded variables.
clean (bool): in addition to removing empty entries,
also remove duplicate entries (default: False).
"""
tty.debug("EnvironmentModifications.from_sourcing_file: {0}".format(filename))
# Check if the file actually exists
if not os.path.isfile(filename):
msg = "Trying to source non-existing file: {0}".format(filename)
raise RuntimeError(msg)
# Prepare include and exclude lists of environment variable names
exclude = kwargs.get("exclude", [])
include = kwargs.get("include", [])
clean = kwargs.get("clean", False)
# Other variables unrelated to sourcing a file
exclude.extend(
[
# Bash internals
"SHLVL",
"_",
"PWD",
"OLDPWD",
"PS1",
"PS2",
"ENV",
# Environment modules v4
"LOADEDMODULES",
"_LMFILES_",
"BASH_FUNC_module()",
"MODULEPATH",
"MODULES_(.*)",
r"(\w*)_mod(quar|share)",
# Lmod configuration
r"LMOD_(.*)",
"MODULERCFILE",
]
)
# Compute the environments before and after sourcing
before = sanitize(
environment_after_sourcing_files(os.devnull, **kwargs),
exclude=exclude,
include=include,
)
file_and_args = (filename,) + arguments
after = sanitize(
environment_after_sourcing_files(file_and_args, **kwargs),
exclude=exclude,
include=include,
)
# Delegate to the other factory
return EnvironmentModifications.from_environment_diff(before, after, clean)
@staticmethod
def from_environment_diff(before, after, clean=False):
"""Constructs an instance of a
:py:class:`spack.util.environment.EnvironmentModifications` object
from the diff of two dictionaries.
Args:
before (dict): environment before the modifications are applied
after (dict): environment after the modifications are applied
clean (bool): in addition to removing empty entries, also remove
duplicate entries
"""
# Fill the EnvironmentModifications instance
env = EnvironmentModifications()
# New variables
new_variables = list(set(after) - set(before))
# Variables that have been unset
unset_variables = list(set(before) - set(after))
# Variables that have been modified
common_variables = set(before).intersection(set(after))
modified_variables = [x for x in common_variables if before[x] != after[x]]
# Consistent output order - looks nicer, easier comparison...
new_variables.sort()
unset_variables.sort()
modified_variables.sort()
def return_separator_if_any(*args):
separators = ":", ";"
for separator in separators:
for arg in args:
if separator in arg:
return separator
return None
# Add variables to env.
# Assume that variables with 'PATH' in the name or that contain
# separators like ':' or ';' are more likely to be paths
for x in new_variables:
sep = return_separator_if_any(after[x])
if sep:
env.prepend_path(x, after[x], separator=sep)
elif "PATH" in x:
env.prepend_path(x, after[x])
else:
# We just need to set the variable to the new value
env.set(x, after[x])
for x in unset_variables:
env.unset(x)
for x in modified_variables:
value_before = before[x]
value_after = after[x]
sep = return_separator_if_any(value_before, value_after)
if sep:
before_list = value_before.split(sep)
after_list = value_after.split(sep)
# Filter out empty strings
before_list = list(filter(None, before_list))
after_list = list(filter(None, after_list))
# Remove duplicate entries (worse matching, bloats env)
if clean:
before_list = list(dedupe(before_list))
after_list = list(dedupe(after_list))
# The reassembled cleaned entries
value_before = sep.join(before_list)
value_after = sep.join(after_list)
# Paths that have been removed
remove_list = [ii for ii in before_list if ii not in after_list]
# Check that nothing has been added in the middle of
# before_list
remaining_list = [ii for ii in before_list if ii in after_list]
try:
start = after_list.index(remaining_list[0])
end = after_list.index(remaining_list[-1])
search = sep.join(after_list[start : end + 1])
except IndexError:
env.prepend_path(x, value_after)
continue
if search not in value_before:
# We just need to set the variable to the new value
env.prepend_path(x, value_after)
else:
try:
prepend_list = after_list[:start]
prepend_list.reverse() # Preserve order after prepend
except KeyError:
prepend_list = []
try:
append_list = after_list[end + 1 :]
except KeyError:
append_list = []
for item in remove_list:
env.remove_path(x, item)
for item in append_list:
env.append_path(x, item)
for item in prepend_list:
env.prepend_path(x, item)
else:
# We just need to set the variable to the new value
env.set(x, value_after)
return env
def concatenate_paths(paths, separator=os.pathsep):
"""Concatenates an iterable of paths into a string of paths separated by
separator, defaulting to colon.
Args:
paths: iterable of paths
separator: the separator to use, default ';' windows, ':' otherwise
Returns:
string
"""
return separator.join(str(item) for item in paths)
def set_or_unset_not_first(variable, changes, errstream):
"""Check if we are going to set or unset something after other
modifications have already been requested.
"""
indexes = [
ii
for ii, item in enumerate(changes)
if ii != 0 and not item.args.get("force", False) and type(item) in [SetEnv, UnsetEnv]
]
if indexes:
good = "\t \t{context} at {filename}:{lineno}"
nogood = "\t--->\t{context} at {filename}:{lineno}"
message = "Suspicious requests to set or unset '{var}' found"
errstream(message.format(var=variable))
for ii, item in enumerate(changes):
print_format = nogood if ii in indexes else good
errstream(print_format.format(**item.args))
def validate(env, errstream):
"""Validates the environment modifications to check for the presence of
suspicious patterns. Prompts a warning for everything that was found.
Current checks:
- set or unset variables after other changes on the same variable
Args:
env: list of environment modifications
"""
if not env.traced:
return
modifications = env.group_by_name()
for variable, list_of_changes in sorted(modifications.items()):
set_or_unset_not_first(variable, list_of_changes, errstream)
def inspect_path(root, inspections, exclude=None):
"""Inspects ``root`` to search for the subdirectories in ``inspections``.
Adds every path found to a list of prepend-path commands and returns it.
Args:
root (str): absolute path where to search for subdirectories
inspections (dict): maps relative paths to a list of environment
variables that will be modified if the path exists. The
modifications are not performed immediately, but stored in a
command object that is returned to client
exclude (typing.Callable): optional callable. If present it must accept an
absolute path and return True if it should be excluded from the
inspection
Examples:
The following lines execute an inspection in ``/usr`` to search for
``/usr/include`` and ``/usr/lib64``. If found we want to prepend
``/usr/include`` to ``CPATH`` and ``/usr/lib64`` to ``MY_LIB64_PATH``.
.. code-block:: python
# Set up the dictionary containing the inspection
inspections = {
'include': ['CPATH'],
'lib64': ['MY_LIB64_PATH']
}
# Get back the list of command needed to modify the environment
env = inspect_path('/usr', inspections)
# Eventually execute the commands
env.apply_modifications()
Returns:
instance of EnvironmentModifications containing the requested
modifications
"""
if exclude is None:
exclude = lambda x: False
env = EnvironmentModifications()
# Inspect the prefix to check for the existence of common directories
for relative_path, variables in inspections.items():
expected = os.path.join(root, relative_path)
if os.path.isdir(expected) and not exclude(expected):
for variable in variables:
env.prepend_path(variable, expected)
return env
@contextlib.contextmanager
def preserve_environment(*variables):
"""Ensures that the value of the environment variables passed as
arguments is the same before entering to the context manager and after
exiting it.
Variables that are unset before entering the context manager will be
explicitly unset on exit.
Args:
variables (list): list of environment variables to be preserved
"""
cache = {}
for var in variables:
# The environment variable to be preserved might not be there.
# In that case store None as a placeholder.
cache[var] = os.environ.get(var, None)
yield
for var in variables:
value = cache[var]
msg = "[PRESERVE_ENVIRONMENT]"
if value is not None:
# Print a debug statement if the value changed
if var not in os.environ:
msg += ' {0} was unset, will be reset to "{1}"'
tty.debug(msg.format(var, value))
elif os.environ[var] != value:
msg += ' {0} was set to "{1}", will be reset to "{2}"'
tty.debug(msg.format(var, os.environ[var], value))
os.environ[var] = value
elif var in os.environ:
msg += ' {0} was set to "{1}", will be unset'
tty.debug(msg.format(var, os.environ[var]))
del os.environ[var]
def environment_after_sourcing_files(*files, **kwargs):
"""Returns a dictionary with the environment that one would have
after sourcing the files passed as argument.
Args:
*files: each item can either be a string containing the path
of the file to be sourced or a sequence, where the first element
is the file to be sourced and the remaining are arguments to be
passed to the command line
Keyword Args:
env (dict): the initial environment (default: current environment)
shell (str): the shell to use (default: ``/bin/bash``)
shell_options (str): options passed to the shell (default: ``-c``)
source_command (str): the command to run (default: ``source``)
suppress_output (str): redirect used to suppress output of command
(default: ``&> /dev/null``)
concatenate_on_success (str): operator used to execute a command
only when the previous command succeeds (default: ``&&``)
"""
# Set the shell executable that will be used to source files
shell_cmd = kwargs.get("shell", "/bin/bash")
shell_options = kwargs.get("shell_options", "-c")
source_command = kwargs.get("source_command", "source")
suppress_output = kwargs.get("suppress_output", "&> /dev/null")
concatenate_on_success = kwargs.get("concatenate_on_success", "&&")
shell = executable.Executable(" ".join([shell_cmd, shell_options]))
def _source_single_file(file_and_args, environment):
source_file = [source_command]
source_file.extend(x for x in file_and_args)
source_file = " ".join(source_file)
# If the environment contains 'python' use it, if not
# go with sys.executable. Below we just need a working
# Python interpreter, not necessarily sys.executable.
python_cmd = executable.which("python3", "python", "python2")
python_cmd = python_cmd.path if python_cmd else sys.executable
dump_cmd = "import os, json; print(json.dumps(dict(os.environ)))"
dump_environment = python_cmd + ' -E -c "{0}"'.format(dump_cmd)
# Try to source the file
source_file_arguments = " ".join(
[
source_file,
suppress_output,
concatenate_on_success,
dump_environment,
]
)
output = shell(source_file_arguments, output=str, env=environment, ignore_quotes=True)
return json.loads(output)
current_environment = kwargs.get("env", dict(os.environ))
for f in files:
# Normalize the input to the helper function
if isinstance(f, str):
f = [f]
current_environment = _source_single_file(f, environment=current_environment)
return current_environment
def sanitize(environment, exclude, include):
"""Returns a copy of the input dictionary where all the keys that
match an excluded pattern and don't match an included pattern are
removed.
Args:
environment (dict): input dictionary
exclude (list): literals or regex patterns to be excluded
include (list): literals or regex patterns to be included
"""
def set_intersection(fullset, *args):
# A set intersection using string literals and regexs
meta = "[" + re.escape("[$()*?[]^{|}") + "]"
subset = fullset & set(args) # As literal
for name in args:
if re.search(meta, name):
pattern = re.compile(name)
for k in fullset:
if re.match(pattern, k):
subset.add(k)
return subset
# Don't modify input, make a copy instead
environment = dict(environment)
# include supersedes any excluded items
prune = set_intersection(set(environment), *exclude)
prune -= set_intersection(prune, *include)
for k in prune:
environment.pop(k, None)
return environment

10
lib/spack/spack/util/executable.py
View File

@@ -9,6 +9,7 @@
import subprocess
import sys
import llnl.util.envmod as envmod
import llnl.util.tty as tty
from llnl.util.path import Path, format_os_path, path_to_os_path, system_path_filter
@@ -27,9 +28,8 @@ def __init__(self, name):
# filter back to platform dependent path
self.exe = path_to_os_path(*self.exe)
self.default_env = {}
from spack.util.environment import EnvironmentModifications # no cycle
self.default_envmod = EnvironmentModifications()
self.default_envmod = envmod.EnvironmentModifications()
self.returncode = None
if not self.exe:
@@ -130,17 +130,15 @@ def __call__(self, *args, **kwargs):
self.default_envmod.apply_modifications(env)
env.update(self.default_env)
from spack.util.environment import EnvironmentModifications # no cycle
# Apply env argument
if isinstance(env_arg, EnvironmentModifications):
if isinstance(env_arg, envmod.EnvironmentModifications):
env_arg.apply_modifications(env)
elif env_arg:
env.update(env_arg)
# Apply extra env
extra_env = kwargs.get("extra_env", {})
if isinstance(extra_env, EnvironmentModifications):
if isinstance(extra_env, envmod.EnvironmentModifications):
extra_env.apply_modifications(env)
else:
env.update(extra_env)

2
var/spack/repos/builtin/packages/anaconda2/package.py
View File

@@ -6,7 +6,7 @@
from os.path import split
from spack.package import *
from spack.util.environment import EnvironmentModifications
from llnl.util.envmod import EnvironmentModifications
class Anaconda2(Package):

2
var/spack/repos/builtin/packages/anaconda3/package.py
View File

@@ -6,7 +6,7 @@
from os.path import split
from spack.package import *
from spack.util.environment import EnvironmentModifications
from llnl.util.envmod import EnvironmentModifications
class Anaconda3(Package):

2
var/spack/repos/builtin/packages/cdo/package.py
View File

@@ -6,7 +6,7 @@
from collections import defaultdict
from spack.package import *
from spack.util.environment import is_system_path
from llnl.util.envmod import is_system_path
class Cdo(AutotoolsPackage):

2
var/spack/repos/builtin/packages/conda4aarch64/package.py
View File

@@ -4,7 +4,7 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
from spack.util.environment import EnvironmentModifications
from llnl.util.envmod import EnvironmentModifications
class Conda4aarch64(Package):

6
var/spack/repos/builtin/packages/cp2k/package.py
View File

@@ -6,7 +6,7 @@
import os
import os.path
import spack.util.environment
import llnl.util.envmod
from spack.package import *
@@ -730,7 +730,7 @@ def build(self, spec, prefix):
# Apparently the Makefile bases its paths on PWD
# so we need to set PWD = self.build_directory
with spack.util.environment.set_env(PWD=self.build_directory):
with llnl.util.envmod.set_env(PWD=self.build_directory):
super(Cp2k, self).build(spec, prefix)
with working_dir(self.build_directory):
@@ -782,6 +782,6 @@ def check(self):
# CP2K < 7 still uses $PWD to detect the current working dir
# and Makefile is in a subdir, account for both facts here:
with spack.util.environment.set_env(CP2K_DATA_DIR=data_dir, PWD=self.build_directory):
with llnl.util.envmod.set_env(CP2K_DATA_DIR=data_dir, PWD=self.build_directory):
with working_dir(self.build_directory):
make("test", *self.build_targets)

2
var/spack/repos/builtin/packages/foam-extend/package.py
View File

@@ -42,7 +42,7 @@
rewrite_environ_files,
write_environ,
)
from spack.util.environment import EnvironmentModifications
from llnl.util.envmod import EnvironmentModifications
class FoamExtend(Package):

4
var/spack/repos/builtin/packages/fsl/package.py
View File

@@ -7,7 +7,7 @@
import os
from spack.package import *
from spack.util.environment import EnvironmentModifications
from llnl.util.envmod import EnvironmentModifications
class Fsl(Package, CudaPackage):
@@ -180,7 +180,7 @@ def postinstall(self):
# the post install script does not get confused.
vars_to_unset = ["PYTHONPATH", "PYTHONHOME"]
with spack.util.environment.preserve_environment(*vars_to_unset):
with llnl.util.envmod.preserve_environment(*vars_to_unset):
for v in vars_to_unset:
del os.environ[v]

2
var/spack/repos/builtin/packages/gdal/package.py
View File

@@ -9,7 +9,7 @@
from spack.build_systems.autotools import AutotoolsBuilder
from spack.build_systems.cmake import CMakeBuilder
from spack.package import *
from spack.util.environment import filter_system_paths
from llnl.util.envmod import filter_system_paths
class Gdal(CMakePackage, AutotoolsPackage, PythonExtension):

2
var/spack/repos/builtin/packages/heasoft/package.py
View File

@@ -8,7 +8,7 @@
import llnl.util.tty as tty
from spack.package import *
from spack.util.environment import EnvironmentModifications
from llnl.util.envmod import EnvironmentModifications
class Heasoft(AutotoolsPackage):

2
var/spack/repos/builtin/packages/magma/package.py
View File

@@ -166,7 +166,7 @@ def test(self):
test_dir = join_path(self.test_suite.current_test_cache_dir, self.test_src_dir)
with working_dir(test_dir, create=False):
pkg_config_path = "{0}/lib/pkgconfig".format(self.prefix)
with spack.util.environment.set_env(PKG_CONFIG_PATH=pkg_config_path):
with llnl.util.envmod.set_env(PKG_CONFIG_PATH=pkg_config_path):
make("c")
self.run_test("./example_sparse", purpose="MAGMA smoke test - sparse solver")
self.run_test(

2
var/spack/repos/builtin/packages/miniconda2/package.py
View File

@@ -6,7 +6,7 @@
from os.path import split
from spack.package import *
from spack.util.environment import EnvironmentModifications
from llnl.util.envmod import EnvironmentModifications
class Miniconda2(Package):

2
var/spack/repos/builtin/packages/miniconda3/package.py
View File

@@ -7,7 +7,7 @@
from os.path import split
from spack.package import *
from spack.util.environment import EnvironmentModifications
from llnl.util.envmod import EnvironmentModifications
_versions = {
"4.10.3": {

4
var/spack/repos/builtin/packages/octave/package.py
View File

@@ -8,7 +8,7 @@
import sys
import tempfile
import spack.util.environment
import llnl.util.envmod
from spack.package import *
@@ -141,7 +141,7 @@ def check_mkoctfile_works_outside_of_build_env(self):
# Spack's build environment when running tests
vars_to_unset = ["CC", "CXX", "F77", "FC"]
with spack.util.environment.preserve_environment(*vars_to_unset):
with llnl.util.envmod.preserve_environment(*vars_to_unset):
# Delete temporarily the environment variables that point
# to Spack compiler wrappers
for v in vars_to_unset:

2
var/spack/repos/builtin/packages/openfoam-org/package.py
View File

@@ -49,7 +49,7 @@
rewrite_environ_files,
write_environ,
)
from spack.util.environment import EnvironmentModifications
from llnl.util.envmod import EnvironmentModifications
class OpenfoamOrg(Package):

2
var/spack/repos/builtin/packages/openfoam/package.py
View File

@@ -48,7 +48,7 @@
from spack.package import *
from spack.pkg.builtin.boost import Boost
from spack.util.environment import EnvironmentModifications
from llnl.util.envmod import EnvironmentModifications
# Not the nice way of doing things, but is a start for refactoring
__all__ = [

2
var/spack/repos/builtin/packages/python/package.py
View File

@@ -22,7 +22,7 @@
from spack.build_environment import dso_suffix, stat_suffix
from spack.package import *
from spack.util.environment import is_system_path
from llnl.util.envmod import is_system_path
from spack.util.prefix import Prefix
is_windows = sys.platform == "win32"

2
var/spack/repos/builtin/packages/root/package.py
View File

@@ -8,7 +8,7 @@
from spack.operating_systems.mac_os import macos_version
from spack.package import *
from spack.util.environment import is_system_path
from llnl.util.envmod import is_system_path
class Root(CMakePackage):

2
var/spack/repos/builtin/packages/silo/package.py
View File

@@ -4,7 +4,7 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
from spack.util.environment import is_system_path
from llnl.util.envmod import is_system_path
class Silo(AutotoolsPackage):

2
var/spack/repos/builtin/packages/strumpack/package.py
View File

@@ -4,7 +4,7 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
from spack.util.environment import set_env
from llnl.util.envmod import set_env
class Strumpack(CMakePackage, CudaPackage, ROCmPackage):

2
var/spack/repos/builtin/packages/tcl/package.py
View File

@@ -6,7 +6,7 @@
import os
from spack.package import *
from spack.util.environment import is_system_path
from llnl.util.envmod import is_system_path
class Tcl(AutotoolsPackage, SourceforgePackage):