From b4a2e2b82ccd708a9f7490f54d2fee2207eb0542 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 31 Dec 2022 03:29:59 -0800 Subject: [PATCH] refactor: move most of `spack.util.environment` -> `llnl.util.envmod` --- lib/spack/llnl/util/envmod.py | 983 ++++++++++++++++++ lib/spack/llnl/util/filesystem.py | 1 - lib/spack/llnl/util/lock.py | 1 - lib/spack/spack/bootstrap/_common.py | 4 +- lib/spack/spack/bootstrap/core.py | 4 +- lib/spack/spack/build_environment.py | 20 +- lib/spack/spack/build_systems/intel.py | 2 +- lib/spack/spack/build_systems/oneapi.py | 2 +- lib/spack/spack/build_systems/racket.py | 2 +- lib/spack/spack/builder.py | 4 +- lib/spack/spack/cmd/diff.py | 1 - lib/spack/spack/cmd/env.py | 2 +- lib/spack/spack/cmd/external.py | 1 - lib/spack/spack/cmd/load.py | 5 +- lib/spack/spack/cmd/unload.py | 5 +- lib/spack/spack/compiler.py | 8 +- lib/spack/spack/compilers/__init__.py | 2 +- lib/spack/spack/detection/common.py | 2 +- lib/spack/spack/detection/path.py | 12 +- lib/spack/spack/directory_layout.py | 3 +- lib/spack/spack/environment/environment.py | 7 +- lib/spack/spack/environment/shell.py | 6 +- lib/spack/spack/fetch_strategy.py | 2 +- lib/spack/spack/install_test.py | 2 +- lib/spack/spack/installer.py | 3 +- lib/spack/spack/main.py | 4 +- lib/spack/spack/modules/common.py | 6 +- lib/spack/spack/modules/lmod.py | 4 +- .../spack/operating_systems/cray_frontend.py | 2 +- lib/spack/spack/package_base.py | 8 +- lib/spack/spack/platforms/_functions.py | 5 +- lib/spack/spack/schema/environment.py | 4 +- lib/spack/spack/spec.py | 2 +- lib/spack/spack/test/build_environment.py | 2 +- lib/spack/spack/test/cc.py | 3 +- lib/spack/spack/test/cmd/env.py | 2 +- lib/spack/spack/test/compilers/basics.py | 6 +- .../spack/test/environment_modifications.py | 77 +- lib/spack/spack/test/llnl/util/envmod.py | 138 +++ lib/spack/spack/test/llnl/util/path.py | 9 +- lib/spack/spack/test/make_executable.py | 3 +- lib/spack/spack/test/modules/lmod.py | 4 +- lib/spack/spack/test/util/environment.py | 144 +-- lib/spack/spack/user_environment.py | 9 +- lib/spack/spack/util/environment.py | 977 +---------------- lib/spack/spack/util/executable.py | 10 +- .../builtin/packages/anaconda2/package.py | 2 +- .../builtin/packages/anaconda3/package.py | 2 +- .../repos/builtin/packages/cdo/package.py | 2 +- .../builtin/packages/conda4aarch64/package.py | 2 +- .../repos/builtin/packages/cp2k/package.py | 6 +- .../builtin/packages/foam-extend/package.py | 2 +- .../repos/builtin/packages/fsl/package.py | 4 +- .../repos/builtin/packages/gdal/package.py | 2 +- .../repos/builtin/packages/heasoft/package.py | 2 +- .../repos/builtin/packages/magma/package.py | 2 +- .../builtin/packages/miniconda2/package.py | 2 +- .../builtin/packages/miniconda3/package.py | 2 +- .../repos/builtin/packages/octave/package.py | 4 +- .../builtin/packages/openfoam-org/package.py | 2 +- .../builtin/packages/openfoam/package.py | 2 +- .../repos/builtin/packages/python/package.py | 2 +- .../repos/builtin/packages/root/package.py | 2 +- .../repos/builtin/packages/silo/package.py | 2 +- .../builtin/packages/strumpack/package.py | 2 +- .../repos/builtin/packages/tcl/package.py | 2 +- 66 files changed, 1278 insertions(+), 1273 deletions(-) create mode 100644 lib/spack/llnl/util/envmod.py create mode 100644 lib/spack/spack/test/llnl/util/envmod.py diff --git a/lib/spack/llnl/util/envmod.py b/lib/spack/llnl/util/envmod.py new file mode 100644 index 00000000000..b06fd1a041b --- /dev/null +++ b/lib/spack/llnl/util/envmod.py @@ -0,0 +1,983 @@ +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Utilities for setting and modifying environment variables.""" +import collections +import contextlib +import inspect +import json +import os +import os.path +import re +import shlex +import sys + +import llnl.util.lang +import llnl.util.tty as tty +from llnl.util.path import path_to_os_path + +import spack.util.executable as executable + +is_windows = sys.platform == "win32" + +system_paths = ( + ["/", "/usr", "/usr/local"] + if not is_windows + else ["C:\\", "C:\\Program Files", "C:\\Program Files (x86)", "C:\\Users", "C:\\ProgramData"] +) +suffixes = ["bin", "bin64", "include", "lib", "lib64"] if not is_windows else [] +system_dirs = [os.path.join(p, s) for s in suffixes for p in system_paths] + system_paths + + +def is_system_path(path): + """Predicate that given a path returns True if it is a system path, + False otherwise. + + Args: + path (str): path to a directory + + Returns: + True or False + """ + return path and os.path.normpath(path) in system_dirs + + +def filter_system_paths(paths): + """Return only paths that are not system paths.""" + return [p for p in paths if not is_system_path(p)] + + +def deprioritize_system_paths(paths): + """Put system paths at the end of paths, otherwise preserving order.""" + filtered_paths = filter_system_paths(paths) + fp = set(filtered_paths) + return filtered_paths + [p for p in paths if p not in fp] + + +_shell_set_strings = { + "sh": "export {0}={1};\n", + "csh": "setenv {0} {1};\n", + "fish": "set -gx {0} {1};\n", + "bat": 'set "{0}={1}"\n', +} + + +_shell_unset_strings = { + "sh": "unset {0};\n", + "csh": "unsetenv {0};\n", + "fish": "set -e {0};\n", + "bat": 'set "{0}="\n', +} + + +tracing_enabled = False + + +def prune_duplicate_paths(paths): + """Returns the paths with duplicates removed, order preserved.""" + return list(llnl.util.lang.dedupe(paths)) + + +def get_path(name): + path = os.environ.get(name, "").strip() + if path: + return path.split(os.pathsep) + else: + return [] + + +def env_flag(name): + if name in os.environ: + value = os.environ[name].lower() + return value == "true" or value == "1" + return False + + +def path_set(var_name, directories): + path_str = os.pathsep.join(str(dir) for dir in directories) + os.environ[var_name] = path_str + + +def path_put_first(var_name, directories): + """Puts the provided directories first in the path, adding them + if they're not already there. + """ + + path = os.environ.get(var_name, "").split(os.pathsep) + + for dir in directories: + if dir in path: + path.remove(dir) + + new_path = tuple(directories) + tuple(path) + path_set(var_name, new_path) + + +@contextlib.contextmanager +def set_env(**kwargs): + """Temporarily sets and restores environment variables. + + Variables can be set as keyword arguments to this function. + """ + saved = {} + for var, value in kwargs.items(): + if var in os.environ: + saved[var] = os.environ[var] + + if value is None: + if var in os.environ: + del os.environ[var] + else: + os.environ[var] = value + + yield + + for var, value in kwargs.items(): + if var in saved: + os.environ[var] = saved[var] + else: + if var in os.environ: + del os.environ[var] + + +class NameModifier(object): + def __init__(self, name, **kwargs): + self.name = name + self.separator = kwargs.get("separator", os.pathsep) + self.args = {"name": name, "separator": self.separator} + + self.args.update(kwargs) + + def __eq__(self, other): + if not isinstance(other, NameModifier): + return False + return self.name == other.name + + def update_args(self, **kwargs): + self.__dict__.update(kwargs) + self.args.update(kwargs) + + +class NameValueModifier(object): + def __init__(self, name, value, **kwargs): + self.name = name + self.value = value + self.separator = kwargs.get("separator", os.pathsep) + self.args = {"name": name, "value": value, "separator": self.separator} + self.args.update(kwargs) + + def __eq__(self, other): + if not isinstance(other, NameValueModifier): + return False + return ( + self.name == other.name + and self.value == other.value + and self.separator == other.separator + ) + + def update_args(self, **kwargs): + self.__dict__.update(kwargs) + self.args.update(kwargs) + + +class SetEnv(NameValueModifier): + def execute(self, env): + tty.debug("SetEnv: {0}={1}".format(self.name, str(self.value)), level=3) + env[self.name] = str(self.value) + + +class AppendFlagsEnv(NameValueModifier): + def execute(self, env): + tty.debug("AppendFlagsEnv: {0}={1}".format(self.name, str(self.value)), level=3) + if self.name in env and env[self.name]: + env[self.name] += self.separator + str(self.value) + else: + env[self.name] = str(self.value) + + +class UnsetEnv(NameModifier): + def execute(self, env): + tty.debug("UnsetEnv: {0}".format(self.name), level=3) + # Avoid throwing if the variable was not set + env.pop(self.name, None) + + +class RemoveFlagsEnv(NameValueModifier): + def execute(self, env): + tty.debug("RemoveFlagsEnv: {0}-{1}".format(self.name, str(self.value)), level=3) + environment_value = env.get(self.name, "") + flags = environment_value.split(self.separator) if environment_value else [] + flags = [f for f in flags if f != self.value] + env[self.name] = self.separator.join(flags) + + +class SetPath(NameValueModifier): + def execute(self, env): + string_path = concatenate_paths(self.value, separator=self.separator) + tty.debug("SetPath: {0}={1}".format(self.name, string_path), level=3) + env[self.name] = string_path + + +class AppendPath(NameValueModifier): + def execute(self, env): + tty.debug("AppendPath: {0}+{1}".format(self.name, str(self.value)), level=3) + environment_value = env.get(self.name, "") + directories = environment_value.split(self.separator) if environment_value else [] + directories.append(path_to_os_path(os.path.normpath(self.value)).pop()) + env[self.name] = self.separator.join(directories) + + +class PrependPath(NameValueModifier): + def execute(self, env): + tty.debug("PrependPath: {0}+{1}".format(self.name, str(self.value)), level=3) + environment_value = env.get(self.name, "") + directories = environment_value.split(self.separator) if environment_value else [] + directories = [path_to_os_path(os.path.normpath(self.value)).pop()] + directories + env[self.name] = self.separator.join(directories) + + +class RemovePath(NameValueModifier): + def execute(self, env): + tty.debug("RemovePath: {0}-{1}".format(self.name, str(self.value)), level=3) + environment_value = env.get(self.name, "") + directories = environment_value.split(self.separator) if environment_value else [] + directories = [ + path_to_os_path(os.path.normpath(x)).pop() + for x in directories + if x != path_to_os_path(os.path.normpath(self.value)).pop() + ] + env[self.name] = self.separator.join(directories) + + +class DeprioritizeSystemPaths(NameModifier): + def execute(self, env): + tty.debug("DeprioritizeSystemPaths: {0}".format(self.name), level=3) + environment_value = env.get(self.name, "") + directories = environment_value.split(self.separator) if environment_value else [] + directories = deprioritize_system_paths( + [path_to_os_path(os.path.normpath(x)).pop() for x in directories] + ) + env[self.name] = self.separator.join(directories) + + +class PruneDuplicatePaths(NameModifier): + def execute(self, env): + tty.debug("PruneDuplicatePaths: {0}".format(self.name), level=3) + environment_value = env.get(self.name, "") + directories = environment_value.split(self.separator) if environment_value else [] + directories = prune_duplicate_paths( + [path_to_os_path(os.path.normpath(x)).pop() for x in directories] + ) + env[self.name] = self.separator.join(directories) + + +class EnvironmentModifications(object): + """Keeps track of requests to modify the current environment. + + Each call to a method to modify the environment stores the extra + information on the caller in the request: + + * 'filename' : filename of the module where the caller is defined + * 'lineno': line number where the request occurred + * 'context' : line of code that issued the request that failed + """ + + def __init__(self, other=None, traced=None): + """Initializes a new instance, copying commands from 'other' + if it is not None. + + Args: + other (EnvironmentModifications): list of environment modifications + to be extended (optional) + traced (bool): enable or disable stack trace inspection to log the origin + of the environment modifications. + """ + self.traced = tracing_enabled if traced is None else bool(traced) + self.env_modifications = [] + if other is not None: + self.extend(other) + + def __iter__(self): + return iter(self.env_modifications) + + def __len__(self): + return len(self.env_modifications) + + def extend(self, other): + self._check_other(other) + self.env_modifications.extend(other.env_modifications) + + @staticmethod + def _check_other(other): + if not isinstance(other, EnvironmentModifications): + raise TypeError("other must be an instance of EnvironmentModifications") + + def _maybe_trace(self, kwargs): + """Provide the modification with stack trace info so that we can track its + origin to find issues in packages. This is very slow and expensive.""" + if not self.traced: + return + + stack = inspect.stack() + try: + _, filename, lineno, _, context, index = stack[2] + context = context[index].strip() + except Exception: + filename = "unknown file" + lineno = "unknown line" + context = "unknown context" + kwargs.update({"filename": filename, "lineno": lineno, "context": context}) + + def set(self, name, value, **kwargs): + """Stores a request to set an environment variable. + + Args: + name: name of the environment variable to be set + value: value of the environment variable + """ + self._maybe_trace(kwargs) + item = SetEnv(name, value, **kwargs) + self.env_modifications.append(item) + + def append_flags(self, name, value, sep=" ", **kwargs): + """ + Stores in the current object a request to append to an env variable + + Args: + name: name of the environment variable to be appended to + value: value to append to the environment variable + Appends with spaces separating different additions to the variable + """ + self._maybe_trace(kwargs) + kwargs.update({"separator": sep}) + item = AppendFlagsEnv(name, value, **kwargs) + self.env_modifications.append(item) + + def unset(self, name, **kwargs): + """Stores a request to unset an environment variable. + + Args: + name: name of the environment variable to be unset + """ + self._maybe_trace(kwargs) + item = UnsetEnv(name, **kwargs) + self.env_modifications.append(item) + + def remove_flags(self, name, value, sep=" ", **kwargs): + """ + Stores in the current object a request to remove flags from an + env variable + + Args: + name: name of the environment variable to be removed from + value: value to remove to the environment variable + sep: separator to assume for environment variable + """ + self._maybe_trace(kwargs) + kwargs.update({"separator": sep}) + item = RemoveFlagsEnv(name, value, **kwargs) + self.env_modifications.append(item) + + def set_path(self, name, elements, **kwargs): + """Stores a request to set a path generated from a list. + + Args: + name: name o the environment variable to be set. + elements: elements of the path to set. + """ + self._maybe_trace(kwargs) + item = SetPath(name, elements, **kwargs) + self.env_modifications.append(item) + + def append_path(self, name, path, **kwargs): + """Stores a request to append a path to a path list. + + Args: + name: name of the path list in the environment + path: path to be appended + """ + self._maybe_trace(kwargs) + item = AppendPath(name, path, **kwargs) + self.env_modifications.append(item) + + def prepend_path(self, name, path, **kwargs): + """Same as `append_path`, but the path is pre-pended. + + Args: + name: name of the path list in the environment + path: path to be pre-pended + """ + self._maybe_trace(kwargs) + item = PrependPath(name, path, **kwargs) + self.env_modifications.append(item) + + def remove_path(self, name, path, **kwargs): + """Stores a request to remove a path from a path list. + + Args: + name: name of the path list in the environment + path: path to be removed + """ + self._maybe_trace(kwargs) + item = RemovePath(name, path, **kwargs) + self.env_modifications.append(item) + + def deprioritize_system_paths(self, name, **kwargs): + """Stores a request to deprioritize system paths in a path list, + otherwise preserving the order. + + Args: + name: name of the path list in the environment. + """ + self._maybe_trace(kwargs) + item = DeprioritizeSystemPaths(name, **kwargs) + self.env_modifications.append(item) + + def prune_duplicate_paths(self, name, **kwargs): + """Stores a request to remove duplicates from a path list, otherwise + preserving the order. + + Args: + name: name of the path list in the environment. + """ + self._maybe_trace(kwargs) + item = PruneDuplicatePaths(name, **kwargs) + self.env_modifications.append(item) + + def group_by_name(self): + """Returns a dict of the modifications grouped by variable name. + + Returns: + dict mapping the environment variable name to the modifications to + be done on it + """ + modifications = collections.defaultdict(list) + for item in self: + modifications[item.name].append(item) + return modifications + + def is_unset(self, var_name): + modifications = self.group_by_name() + var_updates = modifications.get(var_name, None) + if not var_updates: + # We did not explicitly unset it + return False + + # The last modification must unset the variable for it to be considered + # unset + return type(var_updates[-1]) == UnsetEnv + + def clear(self): + """ + Clears the current list of modifications + """ + self.env_modifications = [] + + def reversed(self): + """ + Returns the EnvironmentModifications object that will reverse self + + Only creates reversals for additions to the environment, as reversing + ``unset`` and ``remove_path`` modifications is impossible. + + Reversable operations are set(), prepend_path(), append_path(), + set_path(), and append_flags(). + """ + rev = EnvironmentModifications() + + for envmod in reversed(self.env_modifications): + if type(envmod) == SetEnv: + tty.debug("Reversing `Set` environment operation may lose " "original value") + rev.unset(envmod.name) + elif type(envmod) == AppendPath: + rev.remove_path(envmod.name, envmod.value) + elif type(envmod) == PrependPath: + rev.remove_path(envmod.name, envmod.value) + elif type(envmod) == SetPath: + tty.debug("Reversing `SetPath` environment operation may lose " "original value") + rev.unset(envmod.name) + elif type(envmod) == AppendFlagsEnv: + rev.remove_flags(envmod.name, envmod.value) + else: + # This is an un-reversable operation + tty.warn( + "Skipping reversal of unreversable operation" + "%s %s" % (type(envmod), envmod.name) + ) + + return rev + + def apply_modifications(self, env=None): + """Applies the modifications and clears the list.""" + # Use os.environ if not specified + # Do not copy, we want to modify it in place + if env is None: + env = os.environ + + modifications = self.group_by_name() + # Apply modifications one variable at a time + for name, actions in sorted(modifications.items()): + for x in actions: + x.execute(env) + + def shell_modifications(self, shell="sh", explicit=False, env=None): + """Return shell code to apply the modifications and clears the list.""" + modifications = self.group_by_name() + + if env is None: + env = os.environ + + new_env = env.copy() + + for name, actions in sorted(modifications.items()): + for x in actions: + x.execute(new_env) + + if "MANPATH" in new_env and not new_env.get("MANPATH").endswith(":"): + new_env["MANPATH"] += ":" + + cmds = "" + + for name in sorted(set(modifications)): + new = new_env.get(name, None) + old = env.get(name, None) + if explicit or new != old: + if new is None: + cmds += _shell_unset_strings[shell].format(name) + else: + if sys.platform != "win32": + cmd = _shell_set_strings[shell].format(name, shlex.quote(new_env[name])) + else: + cmd = _shell_set_strings[shell].format(name, new_env[name]) + cmds += cmd + return cmds + + @staticmethod + def from_sourcing_file(filename, *arguments, **kwargs): + """Constructs an instance of a + :py:class:`llnl.util.envmod.EnvironmentModifications` object + that has the same effect as sourcing a file. + + Args: + filename (str): the file to be sourced + *arguments (list): arguments to pass on the command line + + Keyword Args: + shell (str): the shell to use (default: ``bash``) + shell_options (str): options passed to the shell (default: ``-c``) + source_command (str): the command to run (default: ``source``) + suppress_output (str): redirect used to suppress output of command + (default: ``&> /dev/null``) + concatenate_on_success (str): operator used to execute a command + only when the previous command succeeds (default: ``&&``) + exclude ([str or re]): ignore any modifications of these + variables (default: []) + include ([str or re]): always respect modifications of these + variables (default: []). Supersedes any excluded variables. + clean (bool): in addition to removing empty entries, + also remove duplicate entries (default: False). + """ + tty.debug("EnvironmentModifications.from_sourcing_file: {0}".format(filename)) + # Check if the file actually exists + if not os.path.isfile(filename): + msg = "Trying to source non-existing file: {0}".format(filename) + raise RuntimeError(msg) + + # Prepare include and exclude lists of environment variable names + exclude = kwargs.get("exclude", []) + include = kwargs.get("include", []) + clean = kwargs.get("clean", False) + + # Other variables unrelated to sourcing a file + exclude.extend( + [ + # Bash internals + "SHLVL", + "_", + "PWD", + "OLDPWD", + "PS1", + "PS2", + "ENV", + # Environment modules v4 + "LOADEDMODULES", + "_LMFILES_", + "BASH_FUNC_module()", + "MODULEPATH", + "MODULES_(.*)", + r"(\w*)_mod(quar|share)", + # Lmod configuration + r"LMOD_(.*)", + "MODULERCFILE", + ] + ) + + # Compute the environments before and after sourcing + before = sanitize( + environment_after_sourcing_files(os.devnull, **kwargs), + exclude=exclude, + include=include, + ) + file_and_args = (filename,) + arguments + after = sanitize( + environment_after_sourcing_files(file_and_args, **kwargs), + exclude=exclude, + include=include, + ) + + # Delegate to the other factory + return EnvironmentModifications.from_environment_diff(before, after, clean) + + @staticmethod + def from_environment_diff(before, after, clean=False): + """Constructs an instance of a + :py:class:`llnl.util.envmod.EnvironmentModifications` object + from the diff of two dictionaries. + + Args: + before (dict): environment before the modifications are applied + after (dict): environment after the modifications are applied + clean (bool): in addition to removing empty entries, also remove + duplicate entries + """ + # Fill the EnvironmentModifications instance + env = EnvironmentModifications() + # New variables + new_variables = list(set(after) - set(before)) + # Variables that have been unset + unset_variables = list(set(before) - set(after)) + # Variables that have been modified + common_variables = set(before).intersection(set(after)) + modified_variables = [x for x in common_variables if before[x] != after[x]] + # Consistent output order - looks nicer, easier comparison... + new_variables.sort() + unset_variables.sort() + modified_variables.sort() + + def return_separator_if_any(*args): + separators = ":", ";" + for separator in separators: + for arg in args: + if separator in arg: + return separator + return None + + # Add variables to env. + # Assume that variables with 'PATH' in the name or that contain + # separators like ':' or ';' are more likely to be paths + for x in new_variables: + sep = return_separator_if_any(after[x]) + if sep: + env.prepend_path(x, after[x], separator=sep) + elif "PATH" in x: + env.prepend_path(x, after[x]) + else: + # We just need to set the variable to the new value + env.set(x, after[x]) + + for x in unset_variables: + env.unset(x) + + for x in modified_variables: + value_before = before[x] + value_after = after[x] + sep = return_separator_if_any(value_before, value_after) + if sep: + before_list = value_before.split(sep) + after_list = value_after.split(sep) + + # Filter out empty strings + before_list = list(filter(None, before_list)) + after_list = list(filter(None, after_list)) + + # Remove duplicate entries (worse matching, bloats env) + if clean: + before_list = list(llnl.util.lang.dedupe(before_list)) + after_list = list(llnl.util.lang.dedupe(after_list)) + # The reassembled cleaned entries + value_before = sep.join(before_list) + value_after = sep.join(after_list) + + # Paths that have been removed + remove_list = [ii for ii in before_list if ii not in after_list] + # Check that nothing has been added in the middle of + # before_list + remaining_list = [ii for ii in before_list if ii in after_list] + try: + start = after_list.index(remaining_list[0]) + end = after_list.index(remaining_list[-1]) + search = sep.join(after_list[start : end + 1]) + except IndexError: + env.prepend_path(x, value_after) + continue + + if search not in value_before: + # We just need to set the variable to the new value + env.prepend_path(x, value_after) + else: + try: + prepend_list = after_list[:start] + prepend_list.reverse() # Preserve order after prepend + except KeyError: + prepend_list = [] + try: + append_list = after_list[end + 1 :] + except KeyError: + append_list = [] + + for item in remove_list: + env.remove_path(x, item) + for item in append_list: + env.append_path(x, item) + for item in prepend_list: + env.prepend_path(x, item) + else: + # We just need to set the variable to the new value + env.set(x, value_after) + + return env + + +def concatenate_paths(paths, separator=os.pathsep): + """Concatenates an iterable of paths into a string of paths separated by + separator, defaulting to colon. + + Args: + paths: iterable of paths + separator: the separator to use, default ';' windows, ':' otherwise + + Returns: + string + """ + return separator.join(str(item) for item in paths) + + +def set_or_unset_not_first(variable, changes, errstream): + """Check if we are going to set or unset something after other + modifications have already been requested. + """ + indexes = [ + ii + for ii, item in enumerate(changes) + if ii != 0 and not item.args.get("force", False) and type(item) in [SetEnv, UnsetEnv] + ] + if indexes: + good = "\t \t{context} at {filename}:{lineno}" + nogood = "\t--->\t{context} at {filename}:{lineno}" + message = "Suspicious requests to set or unset '{var}' found" + errstream(message.format(var=variable)) + for ii, item in enumerate(changes): + print_format = nogood if ii in indexes else good + errstream(print_format.format(**item.args)) + + +def validate(env, errstream): + """Validates the environment modifications to check for the presence of + suspicious patterns. Prompts a warning for everything that was found. + + Current checks: + - set or unset variables after other changes on the same variable + + Args: + env: list of environment modifications + """ + if not env.traced: + return + modifications = env.group_by_name() + for variable, list_of_changes in sorted(modifications.items()): + set_or_unset_not_first(variable, list_of_changes, errstream) + + +def inspect_path(root, inspections, exclude=None): + """Inspects ``root`` to search for the subdirectories in ``inspections``. + Adds every path found to a list of prepend-path commands and returns it. + + Args: + root (str): absolute path where to search for subdirectories + + inspections (dict): maps relative paths to a list of environment + variables that will be modified if the path exists. The + modifications are not performed immediately, but stored in a + command object that is returned to client + + exclude (typing.Callable): optional callable. If present it must accept an + absolute path and return True if it should be excluded from the + inspection + + Examples: + + The following lines execute an inspection in ``/usr`` to search for + ``/usr/include`` and ``/usr/lib64``. If found we want to prepend + ``/usr/include`` to ``CPATH`` and ``/usr/lib64`` to ``MY_LIB64_PATH``. + + .. code-block:: python + + # Set up the dictionary containing the inspection + inspections = { + 'include': ['CPATH'], + 'lib64': ['MY_LIB64_PATH'] + } + + # Get back the list of command needed to modify the environment + env = inspect_path('/usr', inspections) + + # Eventually execute the commands + env.apply_modifications() + + Returns: + instance of EnvironmentModifications containing the requested + modifications + """ + if exclude is None: + exclude = lambda x: False + + env = EnvironmentModifications() + # Inspect the prefix to check for the existence of common directories + for relative_path, variables in inspections.items(): + expected = os.path.join(root, relative_path) + + if os.path.isdir(expected) and not exclude(expected): + for variable in variables: + env.prepend_path(variable, expected) + + return env + + +@contextlib.contextmanager +def preserve_environment(*variables): + """Ensures that the value of the environment variables passed as + arguments is the same before entering to the context manager and after + exiting it. + + Variables that are unset before entering the context manager will be + explicitly unset on exit. + + Args: + variables (list): list of environment variables to be preserved + """ + cache = {} + for var in variables: + # The environment variable to be preserved might not be there. + # In that case store None as a placeholder. + cache[var] = os.environ.get(var, None) + + yield + + for var in variables: + value = cache[var] + msg = "[PRESERVE_ENVIRONMENT]" + if value is not None: + # Print a debug statement if the value changed + if var not in os.environ: + msg += ' {0} was unset, will be reset to "{1}"' + tty.debug(msg.format(var, value)) + elif os.environ[var] != value: + msg += ' {0} was set to "{1}", will be reset to "{2}"' + tty.debug(msg.format(var, os.environ[var], value)) + os.environ[var] = value + elif var in os.environ: + msg += ' {0} was set to "{1}", will be unset' + tty.debug(msg.format(var, os.environ[var])) + del os.environ[var] + + +def environment_after_sourcing_files(*files, **kwargs): + """Returns a dictionary with the environment that one would have + after sourcing the files passed as argument. + + Args: + *files: each item can either be a string containing the path + of the file to be sourced or a sequence, where the first element + is the file to be sourced and the remaining are arguments to be + passed to the command line + + Keyword Args: + env (dict): the initial environment (default: current environment) + shell (str): the shell to use (default: ``/bin/bash``) + shell_options (str): options passed to the shell (default: ``-c``) + source_command (str): the command to run (default: ``source``) + suppress_output (str): redirect used to suppress output of command + (default: ``&> /dev/null``) + concatenate_on_success (str): operator used to execute a command + only when the previous command succeeds (default: ``&&``) + """ + # Set the shell executable that will be used to source files + shell_cmd = kwargs.get("shell", "/bin/bash") + shell_options = kwargs.get("shell_options", "-c") + source_command = kwargs.get("source_command", "source") + suppress_output = kwargs.get("suppress_output", "&> /dev/null") + concatenate_on_success = kwargs.get("concatenate_on_success", "&&") + + shell = executable.Executable(" ".join([shell_cmd, shell_options])) + + def _source_single_file(file_and_args, environment): + source_file = [source_command] + source_file.extend(x for x in file_and_args) + source_file = " ".join(source_file) + + # If the environment contains 'python' use it, if not + # go with sys.executable. Below we just need a working + # Python interpreter, not necessarily sys.executable. + python_cmd = executable.which("python3", "python", "python2") + python_cmd = python_cmd.path if python_cmd else sys.executable + + dump_cmd = "import os, json; print(json.dumps(dict(os.environ)))" + dump_environment = python_cmd + ' -E -c "{0}"'.format(dump_cmd) + + # Try to source the file + source_file_arguments = " ".join( + [ + source_file, + suppress_output, + concatenate_on_success, + dump_environment, + ] + ) + output = shell(source_file_arguments, output=str, env=environment, ignore_quotes=True) + return json.loads(output) + + current_environment = kwargs.get("env", dict(os.environ)) + for f in files: + # Normalize the input to the helper function + if isinstance(f, str): + f = [f] + + current_environment = _source_single_file(f, environment=current_environment) + + return current_environment + + +def sanitize(environment, exclude, include): + """Returns a copy of the input dictionary where all the keys that + match an excluded pattern and don't match an included pattern are + removed. + + Args: + environment (dict): input dictionary + exclude (list): literals or regex patterns to be excluded + include (list): literals or regex patterns to be included + """ + + def set_intersection(fullset, *args): + # A set intersection using string literals and regexs + meta = "[" + re.escape("[$()*?[]^{|}") + "]" + subset = fullset & set(args) # As literal + for name in args: + if re.search(meta, name): + pattern = re.compile(name) + for k in fullset: + if re.match(pattern, k): + subset.add(k) + return subset + + # Don't modify input, make a copy instead + environment = dict(environment) + + # include supersedes any excluded items + prune = set_intersection(set(environment), *exclude) + prune -= set_intersection(prune, *include) + for k in prune: + environment.pop(k, None) + + return environment diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 67d5d1f99e1..252cf99e6df 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,6 @@ from spack.util.executable import CommandNotFoundError, Executable, which - is_windows = _platform == "win32" if not is_windows: diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index 12777947196..bbaa732879d 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -14,7 +14,6 @@ import llnl.util.tty as tty from llnl.util.lang import pretty_seconds - if sys.platform != "win32": import fcntl diff --git a/lib/spack/spack/bootstrap/_common.py b/lib/spack/spack/bootstrap/_common.py index 0798c5a0b7b..d18f124cdf5 100644 --- a/lib/spack/spack/bootstrap/_common.py +++ b/lib/spack/spack/bootstrap/_common.py @@ -12,11 +12,11 @@ import archspec.cpu +import llnl.util.envmod import llnl.util.filesystem as fs from llnl.util import tty import spack.store -import spack.util.environment import spack.util.executable from .config import spec_for_current_python @@ -188,7 +188,7 @@ def _executables_in_store(executables, query_spec, query_info=None): and os.path.isdir(bin_dir) and spack.util.executable.which_string(*executables, path=bin_dir) ): - spack.util.environment.path_put_first("PATH", [bin_dir]) + llnl.util.envmod.path_put_first("PATH", [bin_dir]) if query_info is not None: query_info["command"] = spack.util.executable.which(*executables, path=bin_dir) query_info["spec"] = concrete_spec diff --git a/lib/spack/spack/bootstrap/core.py b/lib/spack/spack/bootstrap/core.py index f4b435deba4..9ec35e8e3a2 100644 --- a/lib/spack/spack/bootstrap/core.py +++ b/lib/spack/spack/bootstrap/core.py @@ -31,6 +31,7 @@ import uuid from typing import Callable, List, Optional +import llnl.util.envmod from llnl.util import tty from llnl.util.lang import GroupedExceptionHandler @@ -46,7 +47,6 @@ import spack.spec import spack.store import spack.user_environment -import spack.util.environment import spack.util.executable import spack.util.path import spack.util.spack_yaml @@ -439,7 +439,7 @@ def ensure_executables_in_path_or_raise( current_bootstrapper.last_search["spec"], current_bootstrapper.last_search["command"], ) - env_mods = spack.util.environment.EnvironmentModifications() + env_mods = llnl.util.envmod.EnvironmentModifications() for dep in concrete_spec.traverse( root=True, order="post", deptype=("link", "run") ): diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 67a40e65515..dd011668a3f 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -43,6 +43,16 @@ from typing import List, Tuple import llnl.util.tty as tty +from llnl.util.envmod import ( + EnvironmentModifications, + env_flag, + filter_system_paths, + get_path, + inspect_path, + is_system_path, + system_dirs, + validate, +) from llnl.util.lang import dedupe from llnl.util.string import plural from llnl.util.symlink import symlink @@ -68,16 +78,6 @@ from spack.error import NoHeadersError, NoLibrariesError from spack.installer import InstallError from spack.util.cpus import cpus_available -from spack.util.environment import ( - EnvironmentModifications, - env_flag, - filter_system_paths, - get_path, - inspect_path, - is_system_path, - system_dirs, - validate, -) from spack.util.executable import Executable from spack.util.log_parse import make_log_context, parse_log_events from spack.util.module_cmd import load_module, module, path_from_modules diff --git a/lib/spack/spack/build_systems/intel.py b/lib/spack/spack/build_systems/intel.py index 4abed70f21d..3158c002378 100644 --- a/lib/spack/spack/build_systems/intel.py +++ b/lib/spack/spack/build_systems/intel.py @@ -11,6 +11,7 @@ import xml.etree.ElementTree as ElementTree import llnl.util.tty as tty +from llnl.util.envmod import EnvironmentModifications from llnl.util.filesystem import ( HeaderList, LibraryList, @@ -25,7 +26,6 @@ import spack.error from spack.build_environment import dso_suffix from spack.package_base import InstallError -from spack.util.environment import EnvironmentModifications from spack.util.executable import Executable from spack.util.prefix import Prefix from spack.version import Version, ver diff --git a/lib/spack/spack/build_systems/oneapi.py b/lib/spack/spack/build_systems/oneapi.py index b7456a57754..cf1747e4925 100644 --- a/lib/spack/spack/build_systems/oneapi.py +++ b/lib/spack/spack/build_systems/oneapi.py @@ -8,10 +8,10 @@ import shutil from os.path import basename, dirname, isdir +from llnl.util.envmod import EnvironmentModifications from llnl.util.filesystem import find_headers, find_libraries, join_path from spack.directives import conflicts, variant -from spack.util.environment import EnvironmentModifications from spack.util.executable import Executable from .generic import Package diff --git a/lib/spack/spack/build_systems/racket.py b/lib/spack/spack/build_systems/racket.py index 1e42e3d199b..09096c9ccf0 100644 --- a/lib/spack/spack/build_systems/racket.py +++ b/lib/spack/spack/build_systems/racket.py @@ -8,12 +8,12 @@ import llnl.util.filesystem as fs import llnl.util.lang as lang import llnl.util.tty as tty +from llnl.util.envmod import env_flag import spack.builder from spack.build_environment import SPACK_NO_PARALLEL_MAKE, determine_number_of_jobs from spack.directives import build_system, extends from spack.package_base import PackageBase -from spack.util.environment import env_flag from spack.util.executable import Executable, ProcessError diff --git a/lib/spack/spack/builder.py b/lib/spack/spack/builder.py index 211d7e218e0..895f7428221 100644 --- a/lib/spack/spack/builder.py +++ b/lib/spack/spack/builder.py @@ -518,7 +518,7 @@ def setup_build_environment(self, env): Spack's store. Args: - env (spack.util.environment.EnvironmentModifications): environment + env (llnl.util.envmod.EnvironmentModifications): environment modifications to be applied when the package is built. Package authors can call methods on it to alter the build environment. """ @@ -546,7 +546,7 @@ def setup_dependent_build_environment(self, env, dependent_spec): variable. Args: - env (spack.util.environment.EnvironmentModifications): environment + env (llnl.util.envmod.EnvironmentModifications): environment modifications to be applied when the dependent package is built. Package authors can call methods on it to alter the build environment. diff --git a/lib/spack/spack/cmd/diff.py b/lib/spack/spack/cmd/diff.py index 013bb693dbc..66987ec68d6 100644 --- a/lib/spack/spack/cmd/diff.py +++ b/lib/spack/spack/cmd/diff.py @@ -13,7 +13,6 @@ import spack.cmd.common.arguments as arguments import spack.environment as ev import spack.solver.asp as asp -import spack.util.environment import spack.util.spack_json as sjson description = "compare two specs" diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index 81e6a62cb5d..d8e720499ba 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -13,6 +13,7 @@ import llnl.util.filesystem as fs import llnl.util.string as string import llnl.util.tty as tty +from llnl.util.envmod import EnvironmentModifications from llnl.util.tty.colify import colify from llnl.util.tty.color import colorize @@ -29,7 +30,6 @@ import spack.schema.env import spack.tengine import spack.traverse as traverse -from spack.util.environment import EnvironmentModifications description = "manage virtual environments" section = "environments" diff --git a/lib/spack/spack/cmd/external.py b/lib/spack/spack/cmd/external.py index af186bb2183..748e933fc55 100644 --- a/lib/spack/spack/cmd/external.py +++ b/lib/spack/spack/cmd/external.py @@ -18,7 +18,6 @@ import spack.cray_manifest as cray_manifest import spack.detection import spack.error -import spack.util.environment description = "manage external packages in Spack configuration" section = "config" diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py index 11ec2c5e25f..8a05d608eb8 100644 --- a/lib/spack/spack/cmd/load.py +++ b/lib/spack/spack/cmd/load.py @@ -5,13 +5,14 @@ import sys +import llnl.util.envmod + import spack.cmd import spack.cmd.common.arguments as arguments import spack.cmd.find import spack.environment as ev import spack.store import spack.user_environment as uenv -import spack.util.environment description = "add package to the user environment" section = "user environment" @@ -110,7 +111,7 @@ def load(parser, args): dep for spec in specs for dep in spec.traverse(root=include_roots, order="post") ] - env_mod = spack.util.environment.EnvironmentModifications() + env_mod = llnl.util.envmod.EnvironmentModifications() for spec in specs: env_mod.extend(uenv.environment_modifications_for_spec(spec)) env_mod.prepend_path(uenv.spack_loaded_hashes_var, spec.dag_hash()) diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py index 8cf7af8b9dc..72da5217036 100644 --- a/lib/spack/spack/cmd/unload.py +++ b/lib/spack/spack/cmd/unload.py @@ -6,11 +6,12 @@ import os import sys +import llnl.util.envmod + import spack.cmd import spack.cmd.common.arguments as arguments import spack.error import spack.user_environment as uenv -import spack.util.environment description = "remove package from the user environment" section = "user environment" @@ -82,7 +83,7 @@ def unload(parser, args): ) return 1 - env_mod = spack.util.environment.EnvironmentModifications() + env_mod = llnl.util.envmod.EnvironmentModifications() for spec in specs: env_mod.extend(uenv.environment_modifications_for_spec(spec).reversed()) env_mod.remove_path(uenv.spack_loaded_hashes_var, spec.dag_hash()) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1c169bb8aaf..1cd698a63b3 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -13,9 +13,11 @@ import tempfile from typing import List, Optional, Sequence +import llnl.util.envmod as envmod import llnl.util.lang import llnl.util.tty as tty from llnl.util.filesystem import path_contains_subdirectory, paths_containing_libs +from llnl.util.path import system_path_filter import spack.compilers import spack.error @@ -23,8 +25,6 @@ import spack.util.executable import spack.util.module_cmd import spack.version -from spack.util.environment import filter_system_paths -from llnl.util.path import system_path_filter __all__ = ["Compiler"] @@ -175,7 +175,7 @@ def _parse_non_system_link_dirs(string: str) -> List[str]: # system paths. Note that 'filter_system_paths' only checks for an # exact match, while 'in_system_subdirectory' checks if a path contains # a system directory as a subdirectory - link_dirs = filter_system_paths(link_dirs) + link_dirs = envmod.filter_system_paths(link_dirs) return list(p for p in link_dirs if not in_system_subdirectory(p)) @@ -658,7 +658,7 @@ def compiler_environment(self): spack.util.module_cmd.load_module(module) # apply other compiler environment changes - env = spack.util.environment.EnvironmentModifications() + env = llnl.util.envmod.EnvironmentModifications() env.extend(spack.schema.environment.parse(self.environment)) env.apply_modifications() diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 3df8c4b218d..50799e9c172 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -17,6 +17,7 @@ import llnl.util.filesystem as fs import llnl.util.lang import llnl.util.tty as tty +from llnl.util.envmod import get_path import spack.compiler import spack.config @@ -24,7 +25,6 @@ import spack.paths import spack.platforms import spack.spec -from spack.util.environment import get_path from spack.util.naming import mod_to_class _path_instance_vars = ["cc", "cxx", "f77", "fc"] diff --git a/lib/spack/spack/detection/common.py b/lib/spack/spack/detection/common.py index 4e5bf0efcc2..0cef7fb37bd 100644 --- a/lib/spack/spack/detection/common.py +++ b/lib/spack/spack/detection/common.py @@ -347,7 +347,7 @@ def find_win32_additional_install_paths(): windows_search_ext.extend( spack.config.get("config:additional_external_search_paths", default=[]) ) - windows_search_ext.extend(spack.util.environment.get_path("PATH")) + windows_search_ext.extend(llnl.util.envmod.get_path("PATH")) return windows_search_ext diff --git a/lib/spack/spack/detection/path.py b/lib/spack/spack/detection/path.py index ae297d86d26..02745cfaa5d 100644 --- a/lib/spack/spack/detection/path.py +++ b/lib/spack/spack/detection/path.py @@ -12,10 +12,10 @@ import sys import warnings +import llnl.util.envmod import llnl.util.filesystem import llnl.util.tty -import spack.util.environment import spack.util.ld_so_conf from .common import ( # find_windows_compiler_bundled_packages, @@ -83,9 +83,9 @@ def libraries_in_ld_and_system_library_path(path_hints=None): """ path_hints = ( path_hints - or spack.util.environment.get_path("LD_LIBRARY_PATH") - + spack.util.environment.get_path("DYLD_LIBRARY_PATH") - + spack.util.environment.get_path("DYLD_FALLBACK_LIBRARY_PATH") + or llnl.util.envmod.get_path("LD_LIBRARY_PATH") + + llnl.util.envmod.get_path("DYLD_LIBRARY_PATH") + + llnl.util.envmod.get_path("DYLD_FALLBACK_LIBRARY_PATH") + spack.util.ld_so_conf.host_dynamic_linker_search_paths() ) search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints) @@ -93,7 +93,7 @@ def libraries_in_ld_and_system_library_path(path_hints=None): def libraries_in_windows_paths(path_hints): - path_hints.extend(spack.util.environment.get_path("PATH")) + path_hints.extend(llnl.util.envmod.get_path("PATH")) search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints) # on Windows, some libraries (.dlls) are found in the bin directory or sometimes # at the search root. Add both of those options to the search scheme @@ -236,7 +236,7 @@ def by_executable(packages_to_check, path_hints=None): path_hints (list): list of paths to be searched. If None the list will be constructed based on the PATH environment variable. """ - path_hints = spack.util.environment.get_path("PATH") if path_hints is None else path_hints + path_hints = llnl.util.envmod.get_path("PATH") if path_hints is None else path_hints exe_pattern_to_pkgs = collections.defaultdict(list) for pkg in packages_to_check: if hasattr(pkg, "executables"): diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 34ebb7d93d7..ab07a96a209 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -20,6 +20,7 @@ import spack.spec import spack.util.spack_json as sjson from spack.error import SpackError +from spack.util.environment import get_host_environment_metadata is_windows = sys.platform == "win32" # Note: Posixpath is used here as opposed to @@ -120,8 +121,6 @@ def write_host_environment(self, spec): versioning. We use it in the case that an analysis later needs to easily access this information. """ - from spack.util.environment import get_host_environment_metadata - env_file = self.env_metadata_path(spec) environ = get_host_environment_metadata() with open(env_file, "w") as fd: diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index ea5728ad3c5..3695a9fe5f7 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -16,6 +16,7 @@ import ruamel.yaml as yaml +import llnl.util.envmod import llnl.util.filesystem as fs import llnl.util.tty as tty from llnl.util.lang import dedupe @@ -1535,7 +1536,7 @@ def check_views(self): ) def _env_modifications_for_default_view(self, reverse=False): - all_mods = spack.util.environment.EnvironmentModifications() + all_mods = llnl.util.envmod.EnvironmentModifications() visited = set() @@ -1576,7 +1577,7 @@ def add_default_view_to_env(self, env_mod): default view. Removes duplicate paths. Args: - env_mod (spack.util.environment.EnvironmentModifications): the environment + env_mod (llnl.util.envmod.EnvironmentModifications): the environment modifications object that is modified. """ if default_view_name not in self.views: @@ -1603,7 +1604,7 @@ def rm_default_view_from_env(self, env_mod): default view. Reverses the action of ``add_default_view_to_env``. Args: - env_mod (spack.util.environment.EnvironmentModifications): the environment + env_mod (llnl.util.envmod.EnvironmentModifications): the environment modifications object that is modified. """ if default_view_name not in self.views: diff --git a/lib/spack/spack/environment/shell.py b/lib/spack/spack/environment/shell.py index 4150c8d52e1..29d471ac37a 100644 --- a/lib/spack/spack/environment/shell.py +++ b/lib/spack/spack/environment/shell.py @@ -5,12 +5,12 @@ import os import llnl.util.tty as tty +from llnl.util.envmod import EnvironmentModifications from llnl.util.tty.color import colorize import spack.environment as ev import spack.repo import spack.store -from spack.util.environment import EnvironmentModifications def activate_header(env, shell, prompt=None): @@ -111,7 +111,7 @@ def activate(env, use_env_repo=False, add_view=True): add_view (bool): generate commands to add view to path variables Returns: - spack.util.environment.EnvironmentModifications: Environment variables + llnl.util.envmod.EnvironmentModifications: Environment variables modifications to activate environment. """ ev.activate(env, use_env_repo=use_env_repo) @@ -150,7 +150,7 @@ def deactivate(): after activation are not unloaded. Returns: - spack.util.environment.EnvironmentModifications: Environment variables + llnl.util.envmod.EnvironmentModifications: Environment variables modifications to activate environment. """ env_mods = EnvironmentModifications() diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 81074367e85..61c1e952df1 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -42,8 +42,8 @@ temp_rename, working_dir, ) -from llnl.util.symlink import symlink from llnl.util.string import comma_and, quote +from llnl.util.symlink import symlink import spack.config import spack.error diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py index 704c142c28d..048ad00c6cb 100644 --- a/lib/spack/spack/install_test.py +++ b/lib/spack/spack/install_test.py @@ -13,8 +13,8 @@ import spack.error import spack.paths -import spack.util.prefix import spack.util.path +import spack.util.prefix import spack.util.spack_json as sjson from spack.spec import Spec diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index eaf734a8161..1d3481f7a33 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -40,6 +40,7 @@ import llnl.util.filesystem as fs import llnl.util.lock as lk import llnl.util.tty as tty +from llnl.util.envmod import EnvironmentModifications from llnl.util.lang import pretty_seconds from llnl.util.tty.color import colorize from llnl.util.tty.log import log_output @@ -56,7 +57,7 @@ import spack.util.executable import spack.util.path import spack.util.timer as timer -from spack.util.environment import EnvironmentModifications, dump_environment +from spack.util.environment import dump_environment from spack.util.executable import which #: Counter to support unique spec sequencing that is used to ensure packages diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 6e279039aee..0b396d58b0a 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -26,6 +26,7 @@ import archspec.cpu +import llnl.util.envmod import llnl.util.lang import llnl.util.tty as tty import llnl.util.tty.colify @@ -44,7 +45,6 @@ import spack.spec import spack.store import spack.util.debug -import spack.util.environment import spack.util.git from spack.error import SpackError @@ -574,7 +574,7 @@ def setup_main_options(args): if args.debug: spack.util.debug.register_interrupt_handler() spack.config.set("config:debug", True, scope="command_line") - spack.util.environment.tracing_enabled = True + llnl.util.envmod.tracing_enabled = True if args.timestamp: tty.set_timestamp(True) diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index 060e33efb42..5c2781c6463 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -36,6 +36,7 @@ import re from typing import Optional +import llnl.util.envmod import llnl.util.filesystem import llnl.util.tty as tty from llnl.util.lang import dedupe @@ -51,7 +52,6 @@ import spack.schema.environment import spack.store import spack.tengine as tengine -import spack.util.environment import spack.util.file_permissions as fp import spack.util.path import spack.util.spack_yaml as syaml @@ -732,8 +732,8 @@ def environment_modifications(self): spec.prefix = view.get_projection_for_spec(spec) - env = spack.util.environment.inspect_path( - spec.prefix, prefix_inspections, exclude=spack.util.environment.is_system_path + env = llnl.util.envmod.inspect_path( + spec.prefix, prefix_inspections, exclude=llnl.util.envmod.is_system_path ) # Let the extendee/dependency modify their extensions/dependencies diff --git a/lib/spack/spack/modules/lmod.py b/lib/spack/spack/modules/lmod.py index 38d89f085e0..cbcd431c66f 100644 --- a/lib/spack/spack/modules/lmod.py +++ b/lib/spack/spack/modules/lmod.py @@ -9,6 +9,7 @@ import posixpath from typing import Any, Dict +import llnl.util.envmod import llnl.util.lang as lang import spack.compilers @@ -17,7 +18,6 @@ import spack.repo import spack.spec import spack.tengine as tengine -import spack.util.environment from .common import BaseConfiguration, BaseContext, BaseFileLayout, BaseModuleFileWriter @@ -71,7 +71,7 @@ def guess_core_compilers(name, store=False): # A compiler is considered to be a core compiler if any of the # C, C++ or Fortran compilers reside in a system directory is_system_compiler = any( - os.path.dirname(x) in spack.util.environment.system_dirs + os.path.dirname(x) in llnl.util.envmod.system_dirs for x in compiler["paths"].values() if x is not None ) diff --git a/lib/spack/spack/operating_systems/cray_frontend.py b/lib/spack/spack/operating_systems/cray_frontend.py index 1b1d7691492..97ea5e2ea83 100644 --- a/lib/spack/spack/operating_systems/cray_frontend.py +++ b/lib/spack/spack/operating_systems/cray_frontend.py @@ -10,8 +10,8 @@ import llnl.util.filesystem as fs import llnl.util.lang import llnl.util.tty as tty +from llnl.util.envmod import get_path -from spack.util.environment import get_path from spack.util.module_cmd import module from .linux_distro import LinuxDistro diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py index 3257ef455b7..367bcb96df9 100644 --- a/lib/spack/spack/package_base.py +++ b/lib/spack/spack/package_base.py @@ -29,9 +29,10 @@ import warnings from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type +import llnl.util.envmod import llnl.util.filesystem as fsys -import llnl.util.tty as tty import llnl.util.path +import llnl.util.tty as tty from llnl.util.lang import classproperty, memoized, nullcontext from llnl.util.link_tree import LinkTree @@ -52,7 +53,6 @@ import spack.spec import spack.store import spack.url -import spack.util.environment import spack.util.path import spack.util.web from spack.filesystem_view import YamlFilesystemView @@ -2102,7 +2102,7 @@ def setup_run_environment(self, env): """Sets up the run environment for a package. Args: - env (spack.util.environment.EnvironmentModifications): environment + env (llnl.util.envmod.EnvironmentModifications): environment modifications to be applied when the package is run. Package authors can call methods on it to alter the run environment. """ @@ -2119,7 +2119,7 @@ def setup_dependent_run_environment(self, env, dependent_spec): for dependencies. Args: - env (spack.util.environment.EnvironmentModifications): environment + env (llnl.util.envmod.EnvironmentModifications): environment modifications to be applied when the dependent package is run. Package authors can call methods on it to alter the build environment. diff --git a/lib/spack/spack/platforms/_functions.py b/lib/spack/spack/platforms/_functions.py index 4e469278472..a81858c34ef 100644 --- a/lib/spack/spack/platforms/_functions.py +++ b/lib/spack/spack/platforms/_functions.py @@ -4,10 +4,9 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import contextlib +import llnl.util.envmod import llnl.util.lang -import spack.util.environment - from .cray import Cray from .darwin import Darwin from .linux import Linux @@ -64,7 +63,7 @@ def prevent_cray_detection(): """Context manager that prevents the detection of the Cray platform""" reset() try: - with spack.util.environment.set_env(MODULEPATH=""): + with llnl.util.envmod.set_env(MODULEPATH=""): yield finally: reset() diff --git a/lib/spack/spack/schema/environment.py b/lib/spack/spack/schema/environment.py index b192ad72066..61e3e3cb143 100644 --- a/lib/spack/spack/schema/environment.py +++ b/lib/spack/spack/schema/environment.py @@ -40,9 +40,9 @@ def parse(config_obj): config_obj: a configuration dictionary conforming to the schema definition for environment modifications """ - import spack.util.environment as ev + import llnl.util.envmod as envmod - env = ev.EnvironmentModifications() + env = envmod.EnvironmentModifications() for command, variable in config_obj.items(): # Distinguish between commands that take only a name as argument # (e.g. unset) and commands that take a name and a value. diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 83f6d272ae0..68a76a2a72b 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -59,10 +59,10 @@ import ruamel.yaml as yaml -import llnl.util.string import llnl.util.filesystem as fs import llnl.util.lang as lang import llnl.util.path as pth +import llnl.util.string import llnl.util.tty as tty import llnl.util.tty.color as clr diff --git a/lib/spack/spack/test/build_environment.py b/lib/spack/spack/test/build_environment.py index da1159f6d42..1dc1289c1a0 100644 --- a/lib/spack/spack/test/build_environment.py +++ b/lib/spack/spack/test/build_environment.py @@ -10,6 +10,7 @@ import pytest +from llnl.util.envmod import EnvironmentModifications from llnl.util.filesystem import HeaderList, LibraryList from llnl.util.path import Path, convert_to_platform_path @@ -24,7 +25,6 @@ dso_suffix, ) from spack.paths import build_env_path -from spack.util.environment import EnvironmentModifications from spack.util.executable import Executable diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 410799a0113..3f50215a044 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -12,11 +12,12 @@ import pytest +from llnl.util.envmod import set_env, system_dirs + import spack.build_environment import spack.config import spack.spec from spack.paths import build_env_path -from spack.util.environment import set_env, system_dirs from spack.util.executable import Executable, ProcessError # diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 81e69ab568b..dc6d63266ea 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -638,7 +638,7 @@ def test_env_view_external_prefix(tmpdir_factory, mutable_database, mock_package e.install_all() e.write() - env_mod = spack.util.environment.EnvironmentModifications() + env_mod = llnl.util.envmod.EnvironmentModifications() e.add_default_view_to_env(env_mod) env_variables = {} env_mod.apply_modifications(env_variables) diff --git a/lib/spack/spack/test/compilers/basics.py b/lib/spack/spack/test/compilers/basics.py index 598e44f4fc7..544f109aabd 100644 --- a/lib/spack/spack/test/compilers/basics.py +++ b/lib/spack/spack/test/compilers/basics.py @@ -10,12 +10,12 @@ import pytest +import llnl.util.envmod import llnl.util.filesystem as fs import spack.compiler import spack.compilers as compilers import spack.spec -import spack.util.environment from spack.compiler import Compiler from spack.util.executable import ProcessError @@ -868,7 +868,7 @@ class MockPackage(object): "x86_64", ["/usr/bin/clang", "/usr/bin/clang++", None, None], ) - env = spack.util.environment.EnvironmentModifications() + env = llnl.util.envmod.EnvironmentModifications() # Check a package that doesn't use xcode and ensure we don't add changes # to the environment pkg = MockPackage() @@ -955,7 +955,7 @@ def test_xcode_not_available(xcode_select_output, mock_executable, monkeypatch): "x86_64", ["/usr/bin/clang", "/usr/bin/clang++", None, None], ) - env = spack.util.environment.EnvironmentModifications() + env = llnl.util.envmod.EnvironmentModifications() class MockPackage(object): use_xcode = True diff --git a/lib/spack/spack/test/environment_modifications.py b/lib/spack/spack/test/environment_modifications.py index dc6bc0d9a35..e09b39a60b5 100644 --- a/lib/spack/spack/test/environment_modifications.py +++ b/lib/spack/spack/test/environment_modifications.py @@ -8,18 +8,9 @@ import pytest -import spack.util.environment as environment +import llnl.util.envmod as envmod + from spack.paths import spack_root -from spack.util.environment import ( - AppendPath, - EnvironmentModifications, - PrependPath, - RemovePath, - SetEnv, - UnsetEnv, - filter_system_paths, - is_system_path, -) datadir = os.path.join(spack_root, "lib", "spack", "spack", "test", "data") @@ -43,7 +34,7 @@ def test_inspect_path(tmpdir): tmpdir.mkdir("lib") tmpdir.mkdir("include") - env = environment.inspect_path(str(tmpdir), inspections) + env = envmod.inspect_path(str(tmpdir), inspections) names = [item.name for item in env] assert "PATH" in names assert "LIBRARY_PATH" in names @@ -58,7 +49,7 @@ def test_exclude_paths_from_inspection(): "include": ["CPATH"], } - env = environment.inspect_path("/usr", inspections, exclude=is_system_path) + env = envmod.inspect_path("/usr", inspections, exclude=envmod.is_system_path) assert len(env) == 0 @@ -79,7 +70,7 @@ def prepare_environment_for_tests(working_env): @pytest.fixture def env(prepare_environment_for_tests): """Returns an empty EnvironmentModifications object.""" - return EnvironmentModifications() + return envmod.EnvironmentModifications() @pytest.fixture @@ -162,7 +153,7 @@ def test_unset(env): @pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)") def test_filter_system_paths(miscellaneous_paths): """Tests that the filtering of system paths works as expected.""" - filtered = filter_system_paths(miscellaneous_paths) + filtered = envmod.filter_system_paths(miscellaneous_paths) expected = [ "/usr/local/Cellar/gcc/5.3.0/lib", "/usr/local/opt/some-package/lib", @@ -246,7 +237,7 @@ def test_extend(env): """ env.set("A", "dummy value") env.set("B", 3) - copy_construct = EnvironmentModifications(env) + copy_construct = envmod.EnvironmentModifications(env) assert len(copy_construct) == 2 @@ -260,12 +251,12 @@ def test_source_files(files_to_be_sourced): """Tests the construction of a list of environment modifications that are the result of sourcing a file. """ - env = EnvironmentModifications() + env = envmod.EnvironmentModifications() for filename in files_to_be_sourced: if filename.endswith("sourceme_parameters.sh"): - env.extend(EnvironmentModifications.from_sourcing_file(filename, "intel64")) + env.extend(envmod.EnvironmentModifications.from_sourcing_file(filename, "intel64")) else: - env.extend(EnvironmentModifications.from_sourcing_file(filename)) + env.extend(envmod.EnvironmentModifications.from_sourcing_file(filename)) modifications = env.group_by_name() @@ -277,28 +268,28 @@ def test_source_files(files_to_be_sourced): # Set new variables assert len(modifications["NEW_VAR"]) == 1 - assert isinstance(modifications["NEW_VAR"][0], SetEnv) + assert isinstance(modifications["NEW_VAR"][0], envmod.SetEnv) assert modifications["NEW_VAR"][0].value == "new" assert len(modifications["FOO"]) == 1 - assert isinstance(modifications["FOO"][0], SetEnv) + assert isinstance(modifications["FOO"][0], envmod.SetEnv) assert modifications["FOO"][0].value == "intel64" # Unset variables assert len(modifications["EMPTY_PATH_LIST"]) == 1 - assert isinstance(modifications["EMPTY_PATH_LIST"][0], UnsetEnv) + assert isinstance(modifications["EMPTY_PATH_LIST"][0], envmod.UnsetEnv) # Modified variables assert len(modifications["UNSET_ME"]) == 1 - assert isinstance(modifications["UNSET_ME"][0], SetEnv) + assert isinstance(modifications["UNSET_ME"][0], envmod.SetEnv) assert modifications["UNSET_ME"][0].value == "overridden" assert len(modifications["PATH_LIST"]) == 3 - assert isinstance(modifications["PATH_LIST"][0], RemovePath) + assert isinstance(modifications["PATH_LIST"][0], envmod.RemovePath) assert modifications["PATH_LIST"][0].value == "/path/third" - assert isinstance(modifications["PATH_LIST"][1], AppendPath) + assert isinstance(modifications["PATH_LIST"][1], envmod.AppendPath) assert modifications["PATH_LIST"][1].value == "/path/fourth" - assert isinstance(modifications["PATH_LIST"][2], PrependPath) + assert isinstance(modifications["PATH_LIST"][2], envmod.PrependPath) assert modifications["PATH_LIST"][2].value == "/path/first" @@ -307,7 +298,7 @@ def test_preserve_environment(prepare_environment_for_tests): # UNSET_ME is defined, and will be unset in the context manager, # NOT_SET is not in the environment and will be set within the # context manager, PATH_LIST is set and will be changed. - with environment.preserve_environment("UNSET_ME", "NOT_SET", "PATH_LIST"): + with envmod.preserve_environment("UNSET_ME", "NOT_SET", "PATH_LIST"): os.environ["NOT_SET"] = "a" assert os.environ["NOT_SET"] == "a" @@ -363,7 +354,7 @@ def test_preserve_environment(prepare_environment_for_tests): @pytest.mark.usefixtures("prepare_environment_for_tests") def test_environment_from_sourcing_files(files, expected, deleted): - env = environment.environment_after_sourcing_files(*files) + env = envmod.environment_after_sourcing_files(*files) # Test that variables that have been modified are still there and contain # the expected output @@ -394,7 +385,7 @@ def test_clear(env): ) def test_sanitize_literals(env, exclude, include): - after = environment.sanitize(env, exclude, include) + after = envmod.sanitize(env, exclude, include) # Check that all the included variables are there assert all(x in after for x in include) @@ -431,7 +422,7 @@ def test_sanitize_literals(env, exclude, include): ) def test_sanitize_regex(env, exclude, include, expected, deleted): - after = environment.sanitize(env, exclude, include) + after = envmod.sanitize(env, exclude, include) assert all(x in after for x in expected) assert all(x not in after for x in deleted) @@ -442,39 +433,39 @@ def test_sanitize_regex(env, exclude, include, expected, deleted): "before,after,search_list", [ # Set environment variables - ({}, {"FOO": "foo"}, [environment.SetEnv("FOO", "foo")]), + ({}, {"FOO": "foo"}, [envmod.SetEnv("FOO", "foo")]), # Unset environment variables - ({"FOO": "foo"}, {}, [environment.UnsetEnv("FOO")]), + ({"FOO": "foo"}, {}, [envmod.UnsetEnv("FOO")]), # Append paths to an environment variable ( {"FOO_PATH": "/a/path"}, {"FOO_PATH": "/a/path:/b/path"}, - [environment.AppendPath("FOO_PATH", "/b/path")], + [envmod.AppendPath("FOO_PATH", "/b/path")], ), ( {}, {"FOO_PATH": "/a/path" + os.sep + "/b/path"}, - [environment.AppendPath("FOO_PATH", "/a/path" + os.sep + "/b/path")], + [envmod.AppendPath("FOO_PATH", "/a/path" + os.sep + "/b/path")], ), ( {"FOO_PATH": "/a/path:/b/path"}, {"FOO_PATH": "/b/path"}, - [environment.RemovePath("FOO_PATH", "/a/path")], + [envmod.RemovePath("FOO_PATH", "/a/path")], ), ( {"FOO_PATH": "/a/path:/b/path"}, {"FOO_PATH": "/a/path:/c/path"}, [ - environment.RemovePath("FOO_PATH", "/b/path"), - environment.AppendPath("FOO_PATH", "/c/path"), + envmod.RemovePath("FOO_PATH", "/b/path"), + envmod.AppendPath("FOO_PATH", "/c/path"), ], ), ( {"FOO_PATH": "/a/path:/b/path"}, {"FOO_PATH": "/c/path:/a/path"}, [ - environment.RemovePath("FOO_PATH", "/b/path"), - environment.PrependPath("FOO_PATH", "/c/path"), + envmod.RemovePath("FOO_PATH", "/b/path"), + envmod.PrependPath("FOO_PATH", "/c/path"), ], ), # Modify two variables in the same environment @@ -482,15 +473,15 @@ def test_sanitize_regex(env, exclude, include, expected, deleted): {"FOO": "foo", "BAR": "bar"}, {"FOO": "baz", "BAR": "baz"}, [ - environment.SetEnv("FOO", "baz"), - environment.SetEnv("BAR", "baz"), + envmod.SetEnv("FOO", "baz"), + envmod.SetEnv("BAR", "baz"), ], ), ], ) def test_from_environment_diff(before, after, search_list): - mod = environment.EnvironmentModifications.from_environment_diff(before, after) + mod = envmod.EnvironmentModifications.from_environment_diff(before, after) for item in search_list: assert item in mod @@ -501,7 +492,7 @@ def test_from_environment_diff(before, after, search_list): def test_exclude_lmod_variables(): # Construct the list of environment modifications file = os.path.join(datadir, "sourceme_lmod.sh") - env = EnvironmentModifications.from_sourcing_file(file) + env = envmod.EnvironmentModifications.from_sourcing_file(file) # Check that variables related to lmod are not in there modifications = env.group_by_name() diff --git a/lib/spack/spack/test/llnl/util/envmod.py b/lib/spack/spack/test/llnl/util/envmod.py new file mode 100644 index 00000000000..ddb145cf682 --- /dev/null +++ b/lib/spack/spack/test/llnl/util/envmod.py @@ -0,0 +1,138 @@ +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Test Spack's environment utility functions.""" +import os +import sys + +import pytest + +import llnl.util.envmod as envmod + +is_windows = sys.platform == "win32" + + +def test_is_system_path(): + sys_path = "C:\\Users" if is_windows else "/usr/bin" + assert envmod.is_system_path(sys_path) + assert not envmod.is_system_path("/nonsense_path/bin") + assert not envmod.is_system_path("") + assert not envmod.is_system_path(None) + + +if is_windows: + test_paths = [ + "C:\\Users", + "C:\\", + "C:\\ProgramData", + "C:\\nonsense_path", + "C:\\Program Files", + "C:\\nonsense_path\\extra\\bin", + ] +else: + test_paths = [ + "/usr/bin", + "/nonsense_path/lib", + "/usr/local/lib", + "/bin", + "/nonsense_path/extra/bin", + "/usr/lib64", + ] + + +def test_filter_system_paths(): + nonsense_prefix = "C:\\nonsense_path" if is_windows else "/nonsense_path" + expected = [p for p in test_paths if p.startswith(nonsense_prefix)] + filtered = envmod.filter_system_paths(test_paths) + assert expected == filtered + + +def deprioritize_system_paths(): + expected = [p for p in test_paths if p.startswith("/nonsense_path")] + expected.extend([p for p in test_paths if not p.startswith("/nonsense_path")]) + filtered = envmod.deprioritize_system_paths(test_paths) + assert expected == filtered + + +def test_prune_duplicate_paths(): + test_paths = ["/a/b", "/a/c", "/a/b", "/a/a", "/a/c", "/a/a/.."] + expected = ["/a/b", "/a/c", "/a/a", "/a/a/.."] + assert expected == envmod.prune_duplicate_paths(test_paths) + + +def test_get_path(working_env): + os.environ["TEST_ENV_VAR"] = os.pathsep.join(["/a", "/b", "/c/d"]) + expected = ["/a", "/b", "/c/d"] + assert envmod.get_path("TEST_ENV_VAR") == expected + + +def test_env_flag(working_env): + assert not envmod.env_flag("TEST_NO_ENV_VAR") + os.environ["TEST_ENV_VAR"] = "1" + assert envmod.env_flag("TEST_ENV_VAR") + os.environ["TEST_ENV_VAR"] = "TRUE" + assert envmod.env_flag("TEST_ENV_VAR") + os.environ["TEST_ENV_VAR"] = "True" + assert envmod.env_flag("TEST_ENV_VAR") + os.environ["TEST_ENV_VAR"] = "TRue" + assert envmod.env_flag("TEST_ENV_VAR") + os.environ["TEST_ENV_VAR"] = "true" + assert envmod.env_flag("TEST_ENV_VAR") + os.environ["TEST_ENV_VAR"] = "27" + assert not envmod.env_flag("TEST_ENV_VAR") + os.environ["TEST_ENV_VAR"] = "-2.3" + assert not envmod.env_flag("TEST_ENV_VAR") + os.environ["TEST_ENV_VAR"] = "0" + assert not envmod.env_flag("TEST_ENV_VAR") + os.environ["TEST_ENV_VAR"] = "False" + assert not envmod.env_flag("TEST_ENV_VAR") + os.environ["TEST_ENV_VAR"] = "false" + assert not envmod.env_flag("TEST_ENV_VAR") + os.environ["TEST_ENV_VAR"] = "garbage" + assert not envmod.env_flag("TEST_ENV_VAR") + + +def test_path_set(working_env): + envmod.path_set("TEST_ENV_VAR", ["/a", "/a/b", "/a/a"]) + assert os.environ["TEST_ENV_VAR"] == "/a" + os.pathsep + "/a/b" + os.pathsep + "/a/a" + + +def test_path_put_first(working_env): + envmod.path_set("TEST_ENV_VAR", test_paths) + expected = ["/usr/bin", "/new_nonsense_path/a/b"] + expected.extend([p for p in test_paths if p != "/usr/bin"]) + envmod.path_put_first("TEST_ENV_VAR", expected) + assert envmod.get_path("TEST_ENV_VAR") == expected + + +def test_reverse_environment_modifications(working_env): + start_env = { + "PREPEND_PATH": os.sep + os.path.join("path", "to", "prepend", "to"), + "APPEND_PATH": os.sep + os.path.join("path", "to", "append", "to"), + "UNSET": "var_to_unset", + "APPEND_FLAGS": "flags to append to", + } + + to_reverse = envmod.EnvironmentModifications() + + to_reverse.prepend_path("PREPEND_PATH", "/new/path/prepended") + to_reverse.append_path("APPEND_PATH", "/new/path/appended") + to_reverse.set_path("SET_PATH", ["/one/set/path", "/two/set/path"]) + to_reverse.set("SET", "a var") + to_reverse.unset("UNSET") + to_reverse.append_flags("APPEND_FLAGS", "more_flags") + + reversal = to_reverse.reversed() + + os.environ = start_env.copy() + + print(os.environ) + to_reverse.apply_modifications() + print(os.environ) + reversal.apply_modifications() + print(os.environ) + + start_env.pop("UNSET") + assert os.environ == start_env diff --git a/lib/spack/spack/test/llnl/util/path.py b/lib/spack/spack/test/llnl/util/path.py index 632dba5a6cf..2665f0f0205 100644 --- a/lib/spack/spack/test/llnl/util/path.py +++ b/lib/spack/spack/test/llnl/util/path.py @@ -3,15 +3,18 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import llnl.util.path as lup +import os +import sys + +import llnl.util.path def test_sanitze_file_path(tmpdir): """Test filtering illegal characters out of potential file paths""" # *nix illegal files characters are '/' and none others illegal_file_path = str(tmpdir) + "//" + "abcdefghi.txt" - if is_windows: + if sys.platform == "win32": # Windows has a larger set of illegal characters illegal_file_path = os.path.join(tmpdir, 'acd?e:f"g|h*i.txt') - real_path = lup.sanitize_file_path(illegal_file_path) + real_path = llnl.util.path.sanitize_file_path(illegal_file_path) assert real_path == os.path.join(str(tmpdir), "abcdefghi.txt") diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index efaff04de5e..d9b283065ec 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -16,8 +16,9 @@ import pytest +from llnl.util.envmod import path_put_first + from spack.build_environment import MakeExecutable -from spack.util.environment import path_put_first pytestmark = pytest.mark.skipif( sys.platform == "win32", diff --git a/lib/spack/spack/test/modules/lmod.py b/lib/spack/spack/test/modules/lmod.py index c0d228c2fea..33ee3e2ae3d 100644 --- a/lib/spack/spack/test/modules/lmod.py +++ b/lib/spack/spack/test/modules/lmod.py @@ -7,6 +7,8 @@ import pytest +import llnl.util.envmod + import spack.environment as ev import spack.main import spack.modules.lmod @@ -240,7 +242,7 @@ def test_guess_core_compilers(self, factory, module_configuration, monkeypatch): module_configuration("missing_core_compilers") # Our mock paths must be detected as system paths - monkeypatch.setattr(spack.util.environment, "system_dirs", ["/path/to"]) + monkeypatch.setattr(llnl.util.envmod, "system_dirs", ["/path/to"]) # We don't want to really write into user configuration # when running tests diff --git a/lib/spack/spack/test/util/environment.py b/lib/spack/spack/test/util/environment.py index fba126058f0..c65c43f284a 100644 --- a/lib/spack/spack/test/util/environment.py +++ b/lib/spack/spack/test/util/environment.py @@ -3,153 +3,15 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -"""Test Spack's environment utility functions.""" import os -import sys -import pytest - -import spack.util.environment as envutil - -is_windows = sys.platform == "win32" +from spack.util.environment import dump_environment -@pytest.fixture() -def prepare_environment_for_tests(): - if "TEST_ENV_VAR" in os.environ: - del os.environ["TEST_ENV_VAR"] - yield - del os.environ["TEST_ENV_VAR"] - - -def test_is_system_path(): - sys_path = "C:\\Users" if is_windows else "/usr/bin" - assert envutil.is_system_path(sys_path) - assert not envutil.is_system_path("/nonsense_path/bin") - assert not envutil.is_system_path("") - assert not envutil.is_system_path(None) - - -if is_windows: - test_paths = [ - "C:\\Users", - "C:\\", - "C:\\ProgramData", - "C:\\nonsense_path", - "C:\\Program Files", - "C:\\nonsense_path\\extra\\bin", - ] -else: - test_paths = [ - "/usr/bin", - "/nonsense_path/lib", - "/usr/local/lib", - "/bin", - "/nonsense_path/extra/bin", - "/usr/lib64", - ] - - -def test_filter_system_paths(): - nonsense_prefix = "C:\\nonsense_path" if is_windows else "/nonsense_path" - expected = [p for p in test_paths if p.startswith(nonsense_prefix)] - filtered = envutil.filter_system_paths(test_paths) - assert expected == filtered - - -def deprioritize_system_paths(): - expected = [p for p in test_paths if p.startswith("/nonsense_path")] - expected.extend([p for p in test_paths if not p.startswith("/nonsense_path")]) - filtered = envutil.deprioritize_system_paths(test_paths) - assert expected == filtered - - -def test_prune_duplicate_paths(): - test_paths = ["/a/b", "/a/c", "/a/b", "/a/a", "/a/c", "/a/a/.."] - expected = ["/a/b", "/a/c", "/a/a", "/a/a/.."] - assert expected == envutil.prune_duplicate_paths(test_paths) - - -def test_get_path(prepare_environment_for_tests): - os.environ["TEST_ENV_VAR"] = os.pathsep.join(["/a", "/b", "/c/d"]) - expected = ["/a", "/b", "/c/d"] - assert envutil.get_path("TEST_ENV_VAR") == expected - - -def test_env_flag(prepare_environment_for_tests): - assert not envutil.env_flag("TEST_NO_ENV_VAR") - os.environ["TEST_ENV_VAR"] = "1" - assert envutil.env_flag("TEST_ENV_VAR") - os.environ["TEST_ENV_VAR"] = "TRUE" - assert envutil.env_flag("TEST_ENV_VAR") - os.environ["TEST_ENV_VAR"] = "True" - assert envutil.env_flag("TEST_ENV_VAR") - os.environ["TEST_ENV_VAR"] = "TRue" - assert envutil.env_flag("TEST_ENV_VAR") - os.environ["TEST_ENV_VAR"] = "true" - assert envutil.env_flag("TEST_ENV_VAR") - os.environ["TEST_ENV_VAR"] = "27" - assert not envutil.env_flag("TEST_ENV_VAR") - os.environ["TEST_ENV_VAR"] = "-2.3" - assert not envutil.env_flag("TEST_ENV_VAR") - os.environ["TEST_ENV_VAR"] = "0" - assert not envutil.env_flag("TEST_ENV_VAR") - os.environ["TEST_ENV_VAR"] = "False" - assert not envutil.env_flag("TEST_ENV_VAR") - os.environ["TEST_ENV_VAR"] = "false" - assert not envutil.env_flag("TEST_ENV_VAR") - os.environ["TEST_ENV_VAR"] = "garbage" - assert not envutil.env_flag("TEST_ENV_VAR") - - -def test_path_set(prepare_environment_for_tests): - envutil.path_set("TEST_ENV_VAR", ["/a", "/a/b", "/a/a"]) - assert os.environ["TEST_ENV_VAR"] == "/a" + os.pathsep + "/a/b" + os.pathsep + "/a/a" - - -def test_path_put_first(prepare_environment_for_tests): - envutil.path_set("TEST_ENV_VAR", test_paths) - expected = ["/usr/bin", "/new_nonsense_path/a/b"] - expected.extend([p for p in test_paths if p != "/usr/bin"]) - envutil.path_put_first("TEST_ENV_VAR", expected) - assert envutil.get_path("TEST_ENV_VAR") == expected - - -def test_dump_environment(prepare_environment_for_tests, tmpdir): +def test_dump_environment(working_env, tmpdir): test_paths = "/a:/b/x:/b/c" os.environ["TEST_ENV_VAR"] = test_paths dumpfile_path = str(tmpdir.join("envdump.txt")) - envutil.dump_environment(dumpfile_path) + dump_environment(dumpfile_path) with open(dumpfile_path, "r") as dumpfile: assert "TEST_ENV_VAR={0}; export TEST_ENV_VAR\n".format(test_paths) in list(dumpfile) - - -def test_reverse_environment_modifications(working_env): - start_env = { - "PREPEND_PATH": os.sep + os.path.join("path", "to", "prepend", "to"), - "APPEND_PATH": os.sep + os.path.join("path", "to", "append", "to"), - "UNSET": "var_to_unset", - "APPEND_FLAGS": "flags to append to", - } - - to_reverse = envutil.EnvironmentModifications() - - to_reverse.prepend_path("PREPEND_PATH", "/new/path/prepended") - to_reverse.append_path("APPEND_PATH", "/new/path/appended") - to_reverse.set_path("SET_PATH", ["/one/set/path", "/two/set/path"]) - to_reverse.set("SET", "a var") - to_reverse.unset("UNSET") - to_reverse.append_flags("APPEND_FLAGS", "more_flags") - - reversal = to_reverse.reversed() - - os.environ = start_env.copy() - - print(os.environ) - to_reverse.apply_modifications() - print(os.environ) - reversal.apply_modifications() - print(os.environ) - - start_env.pop("UNSET") - assert os.environ == start_env diff --git a/lib/spack/spack/user_environment.py b/lib/spack/spack/user_environment.py index da5fd12e616..41b6fc3c3c6 100644 --- a/lib/spack/spack/user_environment.py +++ b/lib/spack/spack/user_environment.py @@ -5,9 +5,10 @@ import os import sys +import llnl.util.envmod as envmod + import spack.build_environment import spack.config -import spack.util.environment as environment import spack.util.prefix as prefix #: Environment variable name Spack uses to track individually loaded packages @@ -52,7 +53,7 @@ def unconditional_environment_modifications(view): """List of environment (shell) modifications to be processed for view. This list does not depend on the specs in this environment""" - env = environment.EnvironmentModifications() + env = envmod.EnvironmentModifications() for subdir, vars in prefix_inspections(sys.platform).items(): full_subdir = os.path.join(view.root, subdir) @@ -81,8 +82,8 @@ def environment_modifications_for_spec(spec, view=None, set_package_py_globals=T # generic environment modifications determined by inspecting the spec # prefix - env = environment.inspect_path( - spec.prefix, prefix_inspections(spec.platform), exclude=environment.is_system_path + env = envmod.inspect_path( + spec.prefix, prefix_inspections(spec.platform), exclude=envmod.is_system_path ) # Let the extendee/dependency modify their extensions/dependents diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index e46e2545c73..aa288579c91 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -2,122 +2,18 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -"""Utilities for setting and modifying environment variables.""" -import collections -import contextlib -import inspect -import json +"""Spack-specific environment metadata Utilities.""" import os -import os.path import pickle import platform import re import shlex import socket -import sys -import llnl.util.tty as tty -from llnl.util.lang import dedupe -from llnl.util.path import path_to_os_path, system_path_filter +from llnl.util.path import system_path_filter -import spack.config import spack.platforms import spack.spec -import spack.util.executable as executable - -is_windows = sys.platform == "win32" - -system_paths = ( - ["/", "/usr", "/usr/local"] - if not is_windows - else ["C:\\", "C:\\Program Files", "C:\\Program Files (x86)", "C:\\Users", "C:\\ProgramData"] -) -suffixes = ["bin", "bin64", "include", "lib", "lib64"] if not is_windows else [] -system_dirs = [os.path.join(p, s) for s in suffixes for p in system_paths] + system_paths - - -_shell_set_strings = { - "sh": "export {0}={1};\n", - "csh": "setenv {0} {1};\n", - "fish": "set -gx {0} {1};\n", - "bat": 'set "{0}={1}"\n', -} - - -_shell_unset_strings = { - "sh": "unset {0};\n", - "csh": "unsetenv {0};\n", - "fish": "set -e {0};\n", - "bat": 'set "{0}="\n', -} - - -tracing_enabled = False - - -def is_system_path(path): - """Predicate that given a path returns True if it is a system path, - False otherwise. - - Args: - path (str): path to a directory - - Returns: - True or False - """ - return path and os.path.normpath(path) in system_dirs - - -def filter_system_paths(paths): - """Return only paths that are not system paths.""" - return [p for p in paths if not is_system_path(p)] - - -def deprioritize_system_paths(paths): - """Put system paths at the end of paths, otherwise preserving order.""" - filtered_paths = filter_system_paths(paths) - fp = set(filtered_paths) - return filtered_paths + [p for p in paths if p not in fp] - - -def prune_duplicate_paths(paths): - """Returns the paths with duplicates removed, order preserved.""" - return list(dedupe(paths)) - - -def get_path(name): - path = os.environ.get(name, "").strip() - if path: - return path.split(os.pathsep) - else: - return [] - - -def env_flag(name): - if name in os.environ: - value = os.environ[name].lower() - return value == "true" or value == "1" - return False - - -def path_set(var_name, directories): - path_str = os.pathsep.join(str(dir) for dir in directories) - os.environ[var_name] = path_str - - -def path_put_first(var_name, directories): - """Puts the provided directories first in the path, adding them - if they're not already there. - """ - path = os.environ.get(var_name, "").split(os.pathsep) - - for dir in directories: - if dir in path: - path.remove(dir) - - new_path = tuple(directories) + tuple(path) - path_set(var_name, new_path) - bash_function_finder = re.compile(r"BASH_FUNC_(.*?)\(\)") @@ -189,872 +85,3 @@ def get_host_environment(): "arch_str": str(arch_spec), "hostname": socket.gethostname(), } - - -@contextlib.contextmanager -def set_env(**kwargs): - """Temporarily sets and restores environment variables. - - Variables can be set as keyword arguments to this function. - """ - saved = {} - for var, value in kwargs.items(): - if var in os.environ: - saved[var] = os.environ[var] - - if value is None: - if var in os.environ: - del os.environ[var] - else: - os.environ[var] = value - - yield - - for var, value in kwargs.items(): - if var in saved: - os.environ[var] = saved[var] - else: - if var in os.environ: - del os.environ[var] - - -class NameModifier(object): - def __init__(self, name, **kwargs): - self.name = name - self.separator = kwargs.get("separator", os.pathsep) - self.args = {"name": name, "separator": self.separator} - - self.args.update(kwargs) - - def __eq__(self, other): - if not isinstance(other, NameModifier): - return False - return self.name == other.name - - def update_args(self, **kwargs): - self.__dict__.update(kwargs) - self.args.update(kwargs) - - -class NameValueModifier(object): - def __init__(self, name, value, **kwargs): - self.name = name - self.value = value - self.separator = kwargs.get("separator", os.pathsep) - self.args = {"name": name, "value": value, "separator": self.separator} - self.args.update(kwargs) - - def __eq__(self, other): - if not isinstance(other, NameValueModifier): - return False - return ( - self.name == other.name - and self.value == other.value - and self.separator == other.separator - ) - - def update_args(self, **kwargs): - self.__dict__.update(kwargs) - self.args.update(kwargs) - - -class SetEnv(NameValueModifier): - def execute(self, env): - tty.debug("SetEnv: {0}={1}".format(self.name, str(self.value)), level=3) - env[self.name] = str(self.value) - - -class AppendFlagsEnv(NameValueModifier): - def execute(self, env): - tty.debug("AppendFlagsEnv: {0}={1}".format(self.name, str(self.value)), level=3) - if self.name in env and env[self.name]: - env[self.name] += self.separator + str(self.value) - else: - env[self.name] = str(self.value) - - -class UnsetEnv(NameModifier): - def execute(self, env): - tty.debug("UnsetEnv: {0}".format(self.name), level=3) - # Avoid throwing if the variable was not set - env.pop(self.name, None) - - -class RemoveFlagsEnv(NameValueModifier): - def execute(self, env): - tty.debug("RemoveFlagsEnv: {0}-{1}".format(self.name, str(self.value)), level=3) - environment_value = env.get(self.name, "") - flags = environment_value.split(self.separator) if environment_value else [] - flags = [f for f in flags if f != self.value] - env[self.name] = self.separator.join(flags) - - -class SetPath(NameValueModifier): - def execute(self, env): - string_path = concatenate_paths(self.value, separator=self.separator) - tty.debug("SetPath: {0}={1}".format(self.name, string_path), level=3) - env[self.name] = string_path - - -class AppendPath(NameValueModifier): - def execute(self, env): - tty.debug("AppendPath: {0}+{1}".format(self.name, str(self.value)), level=3) - environment_value = env.get(self.name, "") - directories = environment_value.split(self.separator) if environment_value else [] - directories.append(path_to_os_path(os.path.normpath(self.value)).pop()) - env[self.name] = self.separator.join(directories) - - -class PrependPath(NameValueModifier): - def execute(self, env): - tty.debug("PrependPath: {0}+{1}".format(self.name, str(self.value)), level=3) - environment_value = env.get(self.name, "") - directories = environment_value.split(self.separator) if environment_value else [] - directories = [path_to_os_path(os.path.normpath(self.value)).pop()] + directories - env[self.name] = self.separator.join(directories) - - -class RemovePath(NameValueModifier): - def execute(self, env): - tty.debug("RemovePath: {0}-{1}".format(self.name, str(self.value)), level=3) - environment_value = env.get(self.name, "") - directories = environment_value.split(self.separator) if environment_value else [] - directories = [ - path_to_os_path(os.path.normpath(x)).pop() - for x in directories - if x != path_to_os_path(os.path.normpath(self.value)).pop() - ] - env[self.name] = self.separator.join(directories) - - -class DeprioritizeSystemPaths(NameModifier): - def execute(self, env): - tty.debug("DeprioritizeSystemPaths: {0}".format(self.name), level=3) - environment_value = env.get(self.name, "") - directories = environment_value.split(self.separator) if environment_value else [] - directories = deprioritize_system_paths( - [path_to_os_path(os.path.normpath(x)).pop() for x in directories] - ) - env[self.name] = self.separator.join(directories) - - -class PruneDuplicatePaths(NameModifier): - def execute(self, env): - tty.debug("PruneDuplicatePaths: {0}".format(self.name), level=3) - environment_value = env.get(self.name, "") - directories = environment_value.split(self.separator) if environment_value else [] - directories = prune_duplicate_paths( - [path_to_os_path(os.path.normpath(x)).pop() for x in directories] - ) - env[self.name] = self.separator.join(directories) - - -class EnvironmentModifications(object): - """Keeps track of requests to modify the current environment. - - Each call to a method to modify the environment stores the extra - information on the caller in the request: - - * 'filename' : filename of the module where the caller is defined - * 'lineno': line number where the request occurred - * 'context' : line of code that issued the request that failed - """ - - def __init__(self, other=None, traced=None): - """Initializes a new instance, copying commands from 'other' - if it is not None. - - Args: - other (EnvironmentModifications): list of environment modifications - to be extended (optional) - traced (bool): enable or disable stack trace inspection to log the origin - of the environment modifications. - """ - self.traced = tracing_enabled if traced is None else bool(traced) - self.env_modifications = [] - if other is not None: - self.extend(other) - - def __iter__(self): - return iter(self.env_modifications) - - def __len__(self): - return len(self.env_modifications) - - def extend(self, other): - self._check_other(other) - self.env_modifications.extend(other.env_modifications) - - @staticmethod - def _check_other(other): - if not isinstance(other, EnvironmentModifications): - raise TypeError("other must be an instance of EnvironmentModifications") - - def _maybe_trace(self, kwargs): - """Provide the modification with stack trace info so that we can track its - origin to find issues in packages. This is very slow and expensive.""" - if not self.traced: - return - - stack = inspect.stack() - try: - _, filename, lineno, _, context, index = stack[2] - context = context[index].strip() - except Exception: - filename = "unknown file" - lineno = "unknown line" - context = "unknown context" - kwargs.update({"filename": filename, "lineno": lineno, "context": context}) - - def set(self, name, value, **kwargs): - """Stores a request to set an environment variable. - - Args: - name: name of the environment variable to be set - value: value of the environment variable - """ - self._maybe_trace(kwargs) - item = SetEnv(name, value, **kwargs) - self.env_modifications.append(item) - - def append_flags(self, name, value, sep=" ", **kwargs): - """ - Stores in the current object a request to append to an env variable - - Args: - name: name of the environment variable to be appended to - value: value to append to the environment variable - Appends with spaces separating different additions to the variable - """ - self._maybe_trace(kwargs) - kwargs.update({"separator": sep}) - item = AppendFlagsEnv(name, value, **kwargs) - self.env_modifications.append(item) - - def unset(self, name, **kwargs): - """Stores a request to unset an environment variable. - - Args: - name: name of the environment variable to be unset - """ - self._maybe_trace(kwargs) - item = UnsetEnv(name, **kwargs) - self.env_modifications.append(item) - - def remove_flags(self, name, value, sep=" ", **kwargs): - """ - Stores in the current object a request to remove flags from an - env variable - - Args: - name: name of the environment variable to be removed from - value: value to remove to the environment variable - sep: separator to assume for environment variable - """ - self._maybe_trace(kwargs) - kwargs.update({"separator": sep}) - item = RemoveFlagsEnv(name, value, **kwargs) - self.env_modifications.append(item) - - def set_path(self, name, elements, **kwargs): - """Stores a request to set a path generated from a list. - - Args: - name: name o the environment variable to be set. - elements: elements of the path to set. - """ - self._maybe_trace(kwargs) - item = SetPath(name, elements, **kwargs) - self.env_modifications.append(item) - - def append_path(self, name, path, **kwargs): - """Stores a request to append a path to a path list. - - Args: - name: name of the path list in the environment - path: path to be appended - """ - self._maybe_trace(kwargs) - item = AppendPath(name, path, **kwargs) - self.env_modifications.append(item) - - def prepend_path(self, name, path, **kwargs): - """Same as `append_path`, but the path is pre-pended. - - Args: - name: name of the path list in the environment - path: path to be pre-pended - """ - self._maybe_trace(kwargs) - item = PrependPath(name, path, **kwargs) - self.env_modifications.append(item) - - def remove_path(self, name, path, **kwargs): - """Stores a request to remove a path from a path list. - - Args: - name: name of the path list in the environment - path: path to be removed - """ - self._maybe_trace(kwargs) - item = RemovePath(name, path, **kwargs) - self.env_modifications.append(item) - - def deprioritize_system_paths(self, name, **kwargs): - """Stores a request to deprioritize system paths in a path list, - otherwise preserving the order. - - Args: - name: name of the path list in the environment. - """ - self._maybe_trace(kwargs) - item = DeprioritizeSystemPaths(name, **kwargs) - self.env_modifications.append(item) - - def prune_duplicate_paths(self, name, **kwargs): - """Stores a request to remove duplicates from a path list, otherwise - preserving the order. - - Args: - name: name of the path list in the environment. - """ - self._maybe_trace(kwargs) - item = PruneDuplicatePaths(name, **kwargs) - self.env_modifications.append(item) - - def group_by_name(self): - """Returns a dict of the modifications grouped by variable name. - - Returns: - dict mapping the environment variable name to the modifications to - be done on it - """ - modifications = collections.defaultdict(list) - for item in self: - modifications[item.name].append(item) - return modifications - - def is_unset(self, var_name): - modifications = self.group_by_name() - var_updates = modifications.get(var_name, None) - if not var_updates: - # We did not explicitly unset it - return False - - # The last modification must unset the variable for it to be considered - # unset - return type(var_updates[-1]) == UnsetEnv - - def clear(self): - """ - Clears the current list of modifications - """ - self.env_modifications = [] - - def reversed(self): - """ - Returns the EnvironmentModifications object that will reverse self - - Only creates reversals for additions to the environment, as reversing - ``unset`` and ``remove_path`` modifications is impossible. - - Reversable operations are set(), prepend_path(), append_path(), - set_path(), and append_flags(). - """ - rev = EnvironmentModifications() - - for envmod in reversed(self.env_modifications): - if type(envmod) == SetEnv: - tty.debug("Reversing `Set` environment operation may lose " "original value") - rev.unset(envmod.name) - elif type(envmod) == AppendPath: - rev.remove_path(envmod.name, envmod.value) - elif type(envmod) == PrependPath: - rev.remove_path(envmod.name, envmod.value) - elif type(envmod) == SetPath: - tty.debug("Reversing `SetPath` environment operation may lose " "original value") - rev.unset(envmod.name) - elif type(envmod) == AppendFlagsEnv: - rev.remove_flags(envmod.name, envmod.value) - else: - # This is an un-reversable operation - tty.warn( - "Skipping reversal of unreversable operation" - "%s %s" % (type(envmod), envmod.name) - ) - - return rev - - def apply_modifications(self, env=None): - """Applies the modifications and clears the list.""" - # Use os.environ if not specified - # Do not copy, we want to modify it in place - if env is None: - env = os.environ - - modifications = self.group_by_name() - # Apply modifications one variable at a time - for name, actions in sorted(modifications.items()): - for x in actions: - x.execute(env) - - def shell_modifications(self, shell="sh", explicit=False, env=None): - """Return shell code to apply the modifications and clears the list.""" - modifications = self.group_by_name() - - if env is None: - env = os.environ - - new_env = env.copy() - - for name, actions in sorted(modifications.items()): - for x in actions: - x.execute(new_env) - - if "MANPATH" in new_env and not new_env.get("MANPATH").endswith(":"): - new_env["MANPATH"] += ":" - - cmds = "" - - for name in sorted(set(modifications)): - new = new_env.get(name, None) - old = env.get(name, None) - if explicit or new != old: - if new is None: - cmds += _shell_unset_strings[shell].format(name) - else: - if sys.platform != "win32": - cmd = _shell_set_strings[shell].format(name, shlex.quote(new_env[name])) - else: - cmd = _shell_set_strings[shell].format(name, new_env[name]) - cmds += cmd - return cmds - - @staticmethod - def from_sourcing_file(filename, *arguments, **kwargs): - """Constructs an instance of a - :py:class:`spack.util.environment.EnvironmentModifications` object - that has the same effect as sourcing a file. - - Args: - filename (str): the file to be sourced - *arguments (list): arguments to pass on the command line - - Keyword Args: - shell (str): the shell to use (default: ``bash``) - shell_options (str): options passed to the shell (default: ``-c``) - source_command (str): the command to run (default: ``source``) - suppress_output (str): redirect used to suppress output of command - (default: ``&> /dev/null``) - concatenate_on_success (str): operator used to execute a command - only when the previous command succeeds (default: ``&&``) - exclude ([str or re]): ignore any modifications of these - variables (default: []) - include ([str or re]): always respect modifications of these - variables (default: []). Supersedes any excluded variables. - clean (bool): in addition to removing empty entries, - also remove duplicate entries (default: False). - """ - tty.debug("EnvironmentModifications.from_sourcing_file: {0}".format(filename)) - # Check if the file actually exists - if not os.path.isfile(filename): - msg = "Trying to source non-existing file: {0}".format(filename) - raise RuntimeError(msg) - - # Prepare include and exclude lists of environment variable names - exclude = kwargs.get("exclude", []) - include = kwargs.get("include", []) - clean = kwargs.get("clean", False) - - # Other variables unrelated to sourcing a file - exclude.extend( - [ - # Bash internals - "SHLVL", - "_", - "PWD", - "OLDPWD", - "PS1", - "PS2", - "ENV", - # Environment modules v4 - "LOADEDMODULES", - "_LMFILES_", - "BASH_FUNC_module()", - "MODULEPATH", - "MODULES_(.*)", - r"(\w*)_mod(quar|share)", - # Lmod configuration - r"LMOD_(.*)", - "MODULERCFILE", - ] - ) - - # Compute the environments before and after sourcing - before = sanitize( - environment_after_sourcing_files(os.devnull, **kwargs), - exclude=exclude, - include=include, - ) - file_and_args = (filename,) + arguments - after = sanitize( - environment_after_sourcing_files(file_and_args, **kwargs), - exclude=exclude, - include=include, - ) - - # Delegate to the other factory - return EnvironmentModifications.from_environment_diff(before, after, clean) - - @staticmethod - def from_environment_diff(before, after, clean=False): - """Constructs an instance of a - :py:class:`spack.util.environment.EnvironmentModifications` object - from the diff of two dictionaries. - - Args: - before (dict): environment before the modifications are applied - after (dict): environment after the modifications are applied - clean (bool): in addition to removing empty entries, also remove - duplicate entries - """ - # Fill the EnvironmentModifications instance - env = EnvironmentModifications() - # New variables - new_variables = list(set(after) - set(before)) - # Variables that have been unset - unset_variables = list(set(before) - set(after)) - # Variables that have been modified - common_variables = set(before).intersection(set(after)) - modified_variables = [x for x in common_variables if before[x] != after[x]] - # Consistent output order - looks nicer, easier comparison... - new_variables.sort() - unset_variables.sort() - modified_variables.sort() - - def return_separator_if_any(*args): - separators = ":", ";" - for separator in separators: - for arg in args: - if separator in arg: - return separator - return None - - # Add variables to env. - # Assume that variables with 'PATH' in the name or that contain - # separators like ':' or ';' are more likely to be paths - for x in new_variables: - sep = return_separator_if_any(after[x]) - if sep: - env.prepend_path(x, after[x], separator=sep) - elif "PATH" in x: - env.prepend_path(x, after[x]) - else: - # We just need to set the variable to the new value - env.set(x, after[x]) - - for x in unset_variables: - env.unset(x) - - for x in modified_variables: - value_before = before[x] - value_after = after[x] - sep = return_separator_if_any(value_before, value_after) - if sep: - before_list = value_before.split(sep) - after_list = value_after.split(sep) - - # Filter out empty strings - before_list = list(filter(None, before_list)) - after_list = list(filter(None, after_list)) - - # Remove duplicate entries (worse matching, bloats env) - if clean: - before_list = list(dedupe(before_list)) - after_list = list(dedupe(after_list)) - # The reassembled cleaned entries - value_before = sep.join(before_list) - value_after = sep.join(after_list) - - # Paths that have been removed - remove_list = [ii for ii in before_list if ii not in after_list] - # Check that nothing has been added in the middle of - # before_list - remaining_list = [ii for ii in before_list if ii in after_list] - try: - start = after_list.index(remaining_list[0]) - end = after_list.index(remaining_list[-1]) - search = sep.join(after_list[start : end + 1]) - except IndexError: - env.prepend_path(x, value_after) - continue - - if search not in value_before: - # We just need to set the variable to the new value - env.prepend_path(x, value_after) - else: - try: - prepend_list = after_list[:start] - prepend_list.reverse() # Preserve order after prepend - except KeyError: - prepend_list = [] - try: - append_list = after_list[end + 1 :] - except KeyError: - append_list = [] - - for item in remove_list: - env.remove_path(x, item) - for item in append_list: - env.append_path(x, item) - for item in prepend_list: - env.prepend_path(x, item) - else: - # We just need to set the variable to the new value - env.set(x, value_after) - - return env - - -def concatenate_paths(paths, separator=os.pathsep): - """Concatenates an iterable of paths into a string of paths separated by - separator, defaulting to colon. - - Args: - paths: iterable of paths - separator: the separator to use, default ';' windows, ':' otherwise - - Returns: - string - """ - return separator.join(str(item) for item in paths) - - -def set_or_unset_not_first(variable, changes, errstream): - """Check if we are going to set or unset something after other - modifications have already been requested. - """ - indexes = [ - ii - for ii, item in enumerate(changes) - if ii != 0 and not item.args.get("force", False) and type(item) in [SetEnv, UnsetEnv] - ] - if indexes: - good = "\t \t{context} at {filename}:{lineno}" - nogood = "\t--->\t{context} at {filename}:{lineno}" - message = "Suspicious requests to set or unset '{var}' found" - errstream(message.format(var=variable)) - for ii, item in enumerate(changes): - print_format = nogood if ii in indexes else good - errstream(print_format.format(**item.args)) - - -def validate(env, errstream): - """Validates the environment modifications to check for the presence of - suspicious patterns. Prompts a warning for everything that was found. - - Current checks: - - set or unset variables after other changes on the same variable - - Args: - env: list of environment modifications - """ - if not env.traced: - return - modifications = env.group_by_name() - for variable, list_of_changes in sorted(modifications.items()): - set_or_unset_not_first(variable, list_of_changes, errstream) - - -def inspect_path(root, inspections, exclude=None): - """Inspects ``root`` to search for the subdirectories in ``inspections``. - Adds every path found to a list of prepend-path commands and returns it. - - Args: - root (str): absolute path where to search for subdirectories - - inspections (dict): maps relative paths to a list of environment - variables that will be modified if the path exists. The - modifications are not performed immediately, but stored in a - command object that is returned to client - - exclude (typing.Callable): optional callable. If present it must accept an - absolute path and return True if it should be excluded from the - inspection - - Examples: - - The following lines execute an inspection in ``/usr`` to search for - ``/usr/include`` and ``/usr/lib64``. If found we want to prepend - ``/usr/include`` to ``CPATH`` and ``/usr/lib64`` to ``MY_LIB64_PATH``. - - .. code-block:: python - - # Set up the dictionary containing the inspection - inspections = { - 'include': ['CPATH'], - 'lib64': ['MY_LIB64_PATH'] - } - - # Get back the list of command needed to modify the environment - env = inspect_path('/usr', inspections) - - # Eventually execute the commands - env.apply_modifications() - - Returns: - instance of EnvironmentModifications containing the requested - modifications - """ - if exclude is None: - exclude = lambda x: False - - env = EnvironmentModifications() - # Inspect the prefix to check for the existence of common directories - for relative_path, variables in inspections.items(): - expected = os.path.join(root, relative_path) - - if os.path.isdir(expected) and not exclude(expected): - for variable in variables: - env.prepend_path(variable, expected) - - return env - - -@contextlib.contextmanager -def preserve_environment(*variables): - """Ensures that the value of the environment variables passed as - arguments is the same before entering to the context manager and after - exiting it. - - Variables that are unset before entering the context manager will be - explicitly unset on exit. - - Args: - variables (list): list of environment variables to be preserved - """ - cache = {} - for var in variables: - # The environment variable to be preserved might not be there. - # In that case store None as a placeholder. - cache[var] = os.environ.get(var, None) - - yield - - for var in variables: - value = cache[var] - msg = "[PRESERVE_ENVIRONMENT]" - if value is not None: - # Print a debug statement if the value changed - if var not in os.environ: - msg += ' {0} was unset, will be reset to "{1}"' - tty.debug(msg.format(var, value)) - elif os.environ[var] != value: - msg += ' {0} was set to "{1}", will be reset to "{2}"' - tty.debug(msg.format(var, os.environ[var], value)) - os.environ[var] = value - elif var in os.environ: - msg += ' {0} was set to "{1}", will be unset' - tty.debug(msg.format(var, os.environ[var])) - del os.environ[var] - - -def environment_after_sourcing_files(*files, **kwargs): - """Returns a dictionary with the environment that one would have - after sourcing the files passed as argument. - - Args: - *files: each item can either be a string containing the path - of the file to be sourced or a sequence, where the first element - is the file to be sourced and the remaining are arguments to be - passed to the command line - - Keyword Args: - env (dict): the initial environment (default: current environment) - shell (str): the shell to use (default: ``/bin/bash``) - shell_options (str): options passed to the shell (default: ``-c``) - source_command (str): the command to run (default: ``source``) - suppress_output (str): redirect used to suppress output of command - (default: ``&> /dev/null``) - concatenate_on_success (str): operator used to execute a command - only when the previous command succeeds (default: ``&&``) - """ - # Set the shell executable that will be used to source files - shell_cmd = kwargs.get("shell", "/bin/bash") - shell_options = kwargs.get("shell_options", "-c") - source_command = kwargs.get("source_command", "source") - suppress_output = kwargs.get("suppress_output", "&> /dev/null") - concatenate_on_success = kwargs.get("concatenate_on_success", "&&") - - shell = executable.Executable(" ".join([shell_cmd, shell_options])) - - def _source_single_file(file_and_args, environment): - source_file = [source_command] - source_file.extend(x for x in file_and_args) - source_file = " ".join(source_file) - - # If the environment contains 'python' use it, if not - # go with sys.executable. Below we just need a working - # Python interpreter, not necessarily sys.executable. - python_cmd = executable.which("python3", "python", "python2") - python_cmd = python_cmd.path if python_cmd else sys.executable - - dump_cmd = "import os, json; print(json.dumps(dict(os.environ)))" - dump_environment = python_cmd + ' -E -c "{0}"'.format(dump_cmd) - - # Try to source the file - source_file_arguments = " ".join( - [ - source_file, - suppress_output, - concatenate_on_success, - dump_environment, - ] - ) - output = shell(source_file_arguments, output=str, env=environment, ignore_quotes=True) - return json.loads(output) - - current_environment = kwargs.get("env", dict(os.environ)) - for f in files: - # Normalize the input to the helper function - if isinstance(f, str): - f = [f] - - current_environment = _source_single_file(f, environment=current_environment) - - return current_environment - - -def sanitize(environment, exclude, include): - """Returns a copy of the input dictionary where all the keys that - match an excluded pattern and don't match an included pattern are - removed. - - Args: - environment (dict): input dictionary - exclude (list): literals or regex patterns to be excluded - include (list): literals or regex patterns to be included - """ - - def set_intersection(fullset, *args): - # A set intersection using string literals and regexs - meta = "[" + re.escape("[$()*?[]^{|}") + "]" - subset = fullset & set(args) # As literal - for name in args: - if re.search(meta, name): - pattern = re.compile(name) - for k in fullset: - if re.match(pattern, k): - subset.add(k) - return subset - - # Don't modify input, make a copy instead - environment = dict(environment) - - # include supersedes any excluded items - prune = set_intersection(set(environment), *exclude) - prune -= set_intersection(prune, *include) - for k in prune: - environment.pop(k, None) - - return environment diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 5ca3749ec8d..df9aca4eaf1 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -9,6 +9,7 @@ import subprocess import sys +import llnl.util.envmod as envmod import llnl.util.tty as tty from llnl.util.path import Path, format_os_path, path_to_os_path, system_path_filter @@ -27,9 +28,8 @@ def __init__(self, name): # filter back to platform dependent path self.exe = path_to_os_path(*self.exe) self.default_env = {} - from spack.util.environment import EnvironmentModifications # no cycle - self.default_envmod = EnvironmentModifications() + self.default_envmod = envmod.EnvironmentModifications() self.returncode = None if not self.exe: @@ -130,17 +130,15 @@ def __call__(self, *args, **kwargs): self.default_envmod.apply_modifications(env) env.update(self.default_env) - from spack.util.environment import EnvironmentModifications # no cycle - # Apply env argument - if isinstance(env_arg, EnvironmentModifications): + if isinstance(env_arg, envmod.EnvironmentModifications): env_arg.apply_modifications(env) elif env_arg: env.update(env_arg) # Apply extra env extra_env = kwargs.get("extra_env", {}) - if isinstance(extra_env, EnvironmentModifications): + if isinstance(extra_env, envmod.EnvironmentModifications): extra_env.apply_modifications(env) else: env.update(extra_env) diff --git a/var/spack/repos/builtin/packages/anaconda2/package.py b/var/spack/repos/builtin/packages/anaconda2/package.py index d46498f7d01..5b1b9b69352 100644 --- a/var/spack/repos/builtin/packages/anaconda2/package.py +++ b/var/spack/repos/builtin/packages/anaconda2/package.py @@ -6,7 +6,7 @@ from os.path import split from spack.package import * -from spack.util.environment import EnvironmentModifications +from llnl.util.envmod import EnvironmentModifications class Anaconda2(Package): diff --git a/var/spack/repos/builtin/packages/anaconda3/package.py b/var/spack/repos/builtin/packages/anaconda3/package.py index 21de5862167..8bd12459823 100644 --- a/var/spack/repos/builtin/packages/anaconda3/package.py +++ b/var/spack/repos/builtin/packages/anaconda3/package.py @@ -6,7 +6,7 @@ from os.path import split from spack.package import * -from spack.util.environment import EnvironmentModifications +from llnl.util.envmod import EnvironmentModifications class Anaconda3(Package): diff --git a/var/spack/repos/builtin/packages/cdo/package.py b/var/spack/repos/builtin/packages/cdo/package.py index 47a93450b65..7faaf96bc18 100644 --- a/var/spack/repos/builtin/packages/cdo/package.py +++ b/var/spack/repos/builtin/packages/cdo/package.py @@ -6,7 +6,7 @@ from collections import defaultdict from spack.package import * -from spack.util.environment import is_system_path +from llnl.util.envmod import is_system_path class Cdo(AutotoolsPackage): diff --git a/var/spack/repos/builtin/packages/conda4aarch64/package.py b/var/spack/repos/builtin/packages/conda4aarch64/package.py index 6d916829b27..765fb4986f2 100644 --- a/var/spack/repos/builtin/packages/conda4aarch64/package.py +++ b/var/spack/repos/builtin/packages/conda4aarch64/package.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.package import * -from spack.util.environment import EnvironmentModifications +from llnl.util.envmod import EnvironmentModifications class Conda4aarch64(Package): diff --git a/var/spack/repos/builtin/packages/cp2k/package.py b/var/spack/repos/builtin/packages/cp2k/package.py index 452fd8c5525..6b17372950e 100644 --- a/var/spack/repos/builtin/packages/cp2k/package.py +++ b/var/spack/repos/builtin/packages/cp2k/package.py @@ -6,7 +6,7 @@ import os import os.path -import spack.util.environment +import llnl.util.envmod from spack.package import * @@ -730,7 +730,7 @@ def build(self, spec, prefix): # Apparently the Makefile bases its paths on PWD # so we need to set PWD = self.build_directory - with spack.util.environment.set_env(PWD=self.build_directory): + with llnl.util.envmod.set_env(PWD=self.build_directory): super(Cp2k, self).build(spec, prefix) with working_dir(self.build_directory): @@ -782,6 +782,6 @@ def check(self): # CP2K < 7 still uses $PWD to detect the current working dir # and Makefile is in a subdir, account for both facts here: - with spack.util.environment.set_env(CP2K_DATA_DIR=data_dir, PWD=self.build_directory): + with llnl.util.envmod.set_env(CP2K_DATA_DIR=data_dir, PWD=self.build_directory): with working_dir(self.build_directory): make("test", *self.build_targets) diff --git a/var/spack/repos/builtin/packages/foam-extend/package.py b/var/spack/repos/builtin/packages/foam-extend/package.py index 1754c18a459..f8e4123c65a 100644 --- a/var/spack/repos/builtin/packages/foam-extend/package.py +++ b/var/spack/repos/builtin/packages/foam-extend/package.py @@ -42,7 +42,7 @@ rewrite_environ_files, write_environ, ) -from spack.util.environment import EnvironmentModifications +from llnl.util.envmod import EnvironmentModifications class FoamExtend(Package): diff --git a/var/spack/repos/builtin/packages/fsl/package.py b/var/spack/repos/builtin/packages/fsl/package.py index 9dea6e93863..dff199ac91f 100644 --- a/var/spack/repos/builtin/packages/fsl/package.py +++ b/var/spack/repos/builtin/packages/fsl/package.py @@ -7,7 +7,7 @@ import os from spack.package import * -from spack.util.environment import EnvironmentModifications +from llnl.util.envmod import EnvironmentModifications class Fsl(Package, CudaPackage): @@ -180,7 +180,7 @@ def postinstall(self): # the post install script does not get confused. vars_to_unset = ["PYTHONPATH", "PYTHONHOME"] - with spack.util.environment.preserve_environment(*vars_to_unset): + with llnl.util.envmod.preserve_environment(*vars_to_unset): for v in vars_to_unset: del os.environ[v] diff --git a/var/spack/repos/builtin/packages/gdal/package.py b/var/spack/repos/builtin/packages/gdal/package.py index ce6f26ef3ae..debe6d5a8d2 100644 --- a/var/spack/repos/builtin/packages/gdal/package.py +++ b/var/spack/repos/builtin/packages/gdal/package.py @@ -9,7 +9,7 @@ from spack.build_systems.autotools import AutotoolsBuilder from spack.build_systems.cmake import CMakeBuilder from spack.package import * -from spack.util.environment import filter_system_paths +from llnl.util.envmod import filter_system_paths class Gdal(CMakePackage, AutotoolsPackage, PythonExtension): diff --git a/var/spack/repos/builtin/packages/heasoft/package.py b/var/spack/repos/builtin/packages/heasoft/package.py index ba98fe14930..1fb9f83520d 100644 --- a/var/spack/repos/builtin/packages/heasoft/package.py +++ b/var/spack/repos/builtin/packages/heasoft/package.py @@ -8,7 +8,7 @@ import llnl.util.tty as tty from spack.package import * -from spack.util.environment import EnvironmentModifications +from llnl.util.envmod import EnvironmentModifications class Heasoft(AutotoolsPackage): diff --git a/var/spack/repos/builtin/packages/magma/package.py b/var/spack/repos/builtin/packages/magma/package.py index 0799cdb8eaf..03101ec7cf2 100644 --- a/var/spack/repos/builtin/packages/magma/package.py +++ b/var/spack/repos/builtin/packages/magma/package.py @@ -166,7 +166,7 @@ def test(self): test_dir = join_path(self.test_suite.current_test_cache_dir, self.test_src_dir) with working_dir(test_dir, create=False): pkg_config_path = "{0}/lib/pkgconfig".format(self.prefix) - with spack.util.environment.set_env(PKG_CONFIG_PATH=pkg_config_path): + with llnl.util.envmod.set_env(PKG_CONFIG_PATH=pkg_config_path): make("c") self.run_test("./example_sparse", purpose="MAGMA smoke test - sparse solver") self.run_test( diff --git a/var/spack/repos/builtin/packages/miniconda2/package.py b/var/spack/repos/builtin/packages/miniconda2/package.py index 04520ddfe14..729a186365b 100644 --- a/var/spack/repos/builtin/packages/miniconda2/package.py +++ b/var/spack/repos/builtin/packages/miniconda2/package.py @@ -6,7 +6,7 @@ from os.path import split from spack.package import * -from spack.util.environment import EnvironmentModifications +from llnl.util.envmod import EnvironmentModifications class Miniconda2(Package): diff --git a/var/spack/repos/builtin/packages/miniconda3/package.py b/var/spack/repos/builtin/packages/miniconda3/package.py index b9e3dce3804..f7ff5378dfd 100644 --- a/var/spack/repos/builtin/packages/miniconda3/package.py +++ b/var/spack/repos/builtin/packages/miniconda3/package.py @@ -7,7 +7,7 @@ from os.path import split from spack.package import * -from spack.util.environment import EnvironmentModifications +from llnl.util.envmod import EnvironmentModifications _versions = { "4.10.3": { diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py index 85d3a62a4c5..c0b15ebfb4d 100644 --- a/var/spack/repos/builtin/packages/octave/package.py +++ b/var/spack/repos/builtin/packages/octave/package.py @@ -8,7 +8,7 @@ import sys import tempfile -import spack.util.environment +import llnl.util.envmod from spack.package import * @@ -141,7 +141,7 @@ def check_mkoctfile_works_outside_of_build_env(self): # Spack's build environment when running tests vars_to_unset = ["CC", "CXX", "F77", "FC"] - with spack.util.environment.preserve_environment(*vars_to_unset): + with llnl.util.envmod.preserve_environment(*vars_to_unset): # Delete temporarily the environment variables that point # to Spack compiler wrappers for v in vars_to_unset: diff --git a/var/spack/repos/builtin/packages/openfoam-org/package.py b/var/spack/repos/builtin/packages/openfoam-org/package.py index af7b88e5c3e..ce8e66f1b8e 100644 --- a/var/spack/repos/builtin/packages/openfoam-org/package.py +++ b/var/spack/repos/builtin/packages/openfoam-org/package.py @@ -49,7 +49,7 @@ rewrite_environ_files, write_environ, ) -from spack.util.environment import EnvironmentModifications +from llnl.util.envmod import EnvironmentModifications class OpenfoamOrg(Package): diff --git a/var/spack/repos/builtin/packages/openfoam/package.py b/var/spack/repos/builtin/packages/openfoam/package.py index f6c8ff34efa..8e1860f1476 100644 --- a/var/spack/repos/builtin/packages/openfoam/package.py +++ b/var/spack/repos/builtin/packages/openfoam/package.py @@ -48,7 +48,7 @@ from spack.package import * from spack.pkg.builtin.boost import Boost -from spack.util.environment import EnvironmentModifications +from llnl.util.envmod import EnvironmentModifications # Not the nice way of doing things, but is a start for refactoring __all__ = [ diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 3d3e9550248..2a5d0071f71 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -22,7 +22,7 @@ from spack.build_environment import dso_suffix, stat_suffix from spack.package import * -from spack.util.environment import is_system_path +from llnl.util.envmod import is_system_path from spack.util.prefix import Prefix is_windows = sys.platform == "win32" diff --git a/var/spack/repos/builtin/packages/root/package.py b/var/spack/repos/builtin/packages/root/package.py index 262bea2ca1e..fa68eb0e106 100644 --- a/var/spack/repos/builtin/packages/root/package.py +++ b/var/spack/repos/builtin/packages/root/package.py @@ -8,7 +8,7 @@ from spack.operating_systems.mac_os import macos_version from spack.package import * -from spack.util.environment import is_system_path +from llnl.util.envmod import is_system_path class Root(CMakePackage): diff --git a/var/spack/repos/builtin/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py index cbd197ad5d8..90bdfeb0b8c 100644 --- a/var/spack/repos/builtin/packages/silo/package.py +++ b/var/spack/repos/builtin/packages/silo/package.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.package import * -from spack.util.environment import is_system_path +from llnl.util.envmod import is_system_path class Silo(AutotoolsPackage): diff --git a/var/spack/repos/builtin/packages/strumpack/package.py b/var/spack/repos/builtin/packages/strumpack/package.py index 9bc99b4737b..8a87a8b4488 100644 --- a/var/spack/repos/builtin/packages/strumpack/package.py +++ b/var/spack/repos/builtin/packages/strumpack/package.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.package import * -from spack.util.environment import set_env +from llnl.util.envmod import set_env class Strumpack(CMakePackage, CudaPackage, ROCmPackage): diff --git a/var/spack/repos/builtin/packages/tcl/package.py b/var/spack/repos/builtin/packages/tcl/package.py index 14da1480153..711a23d98a8 100644 --- a/var/spack/repos/builtin/packages/tcl/package.py +++ b/var/spack/repos/builtin/packages/tcl/package.py @@ -6,7 +6,7 @@ import os from spack.package import * -from spack.util.environment import is_system_path +from llnl.util.envmod import is_system_path class Tcl(AutotoolsPackage, SourceforgePackage):