From 3f2e77e5fab6c1b85cfb14e749450453f17885b3 Mon Sep 17 00:00:00 2001 From: psakiev Date: Mon, 26 Sep 2022 22:31:29 -0600 Subject: [PATCH] Allow users to specify root env dir Environments managed by spack have some advantages over anonymous Environments but they are tucked away inside spack's directory tree. This PR gives users the ability to specify where the environments should live. See #32823 --- etc/spack/defaults/config.yaml | 4 ++++ lib/spack/spack/environment/environment.py | 13 +++++++---- lib/spack/spack/schema/config.py | 1 + lib/spack/spack/test/config.py | 19 +++++++++++++++ lib/spack/spack/test/conftest.py | 8 ++++--- lib/spack/spack/util/path.py | 27 +++++++++++----------- 6 files changed, 51 insertions(+), 21 deletions(-) diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml index d3e7e4ce686..ba8f9879bab 100644 --- a/etc/spack/defaults/config.yaml +++ b/etc/spack/defaults/config.yaml @@ -76,6 +76,10 @@ config: source_cache: $spack/var/spack/cache + ## Directory where spack managed environments are created and stored + environments_root: $spack/var/spack/environments + + # Cache directory for miscellaneous files, like the package index. # This can be purged with `spack clean --misc-cache` misc_cache: $user_cache_path/cache diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index a27977c9560..2f89548f8d2 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -63,7 +63,10 @@ #: path where environments are stored in the spack tree -env_path = os.path.join(spack.paths.var_path, "environments") +# circular dependency to use canonicalize_path but we want to ignore an active env anyways +# env_path = spack.config.get(os.path.expandvars(os.path.expanduser("config:environments_root"))) +def env_path(): + return spack.util.path.canonicalize_path(spack.config.get("config:environments_root"), False) #: Name of the input yaml file for an environment @@ -205,7 +208,7 @@ def active_environment(): def _root(name): """Non-validating version of root(), to be used internally.""" - return os.path.join(env_path, name) + return os.path.join(env_path(), name) def root(name): @@ -257,10 +260,10 @@ def all_environment_names(): """List the names of environments that currently exist.""" # just return empty if the env path does not exist. A read-only # operation like list should not try to create a directory. - if not os.path.exists(env_path): + if not os.path.exists(env_path()): return [] - candidates = sorted(os.listdir(env_path)) + candidates = sorted(os.listdir(env_path())) names = [] for candidate in candidates: yaml_path = os.path.join(_root(candidate), manifest_name) @@ -842,7 +845,7 @@ def clear(self, re_read=False): @property def internal(self): """Whether this environment is managed by Spack.""" - return self.path.startswith(env_path) + return self.path.startswith(env_path()) @property def name(self): diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 4ed6e7fc2d2..8c874f7f9ea 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -52,6 +52,7 @@ "license_dir": {"type": "string"}, "source_cache": {"type": "string"}, "misc_cache": {"type": "string"}, + "environments_root": {"type": "string"}, "connect_timeout": {"type": "integer", "minimum": 0}, "verify_ssl": {"type": "boolean"}, "suppress_gpg_warnings": {"type": "boolean"}, diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index f4d0b570fda..e457be3b688 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -28,6 +28,9 @@ import spack.schema.repos import spack.util.path as spack_path import spack.util.spack_yaml as syaml +from spack.main import SpackCommand + +env = SpackCommand("env") # sample config data config_low = { @@ -1005,6 +1008,7 @@ def test_good_env_yaml(tmpdir): config: verify_ssl: False dirty: False + environtments_dir: ~/my/env/location repos: - ~/my/repo/location mirrors: @@ -1431,3 +1435,18 @@ def test_config_file_read_invalid_yaml(tmpdir, mutable_empty_config): with pytest.raises(spack.config.ConfigFileError, match="parsing yaml"): spack.config.read_config_file(filename) + + +def test_environment_created_in_users_location(mutable_config, tmpdir): + """Test that an environment is created in a location based on the config""" + spack.config.set("config:environments_root", str(tmpdir.join("envs"))) + env_dir = spack.config.get("config:environments_root") + assert tmpdir.strpath in env_dir + assert not os.path.isdir(env_dir) + os.makedirs(env_dir) + env('create', 'test') + out = env('list') + assert 'test' in out + + assert env_dir in ev.root('test') + assert os.path.isdir(os.path.join(env_dir, 'test')) diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index ed8dc0f9bd4..845abae6454 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1493,11 +1493,13 @@ def get_rev(): @pytest.fixture() def mutable_mock_env_path(tmpdir_factory): """Fixture for mocking the internal spack environments directory.""" - saved_path = ev.environment.env_path + saved_path_fun = ev.environment.env_path mock_path = tmpdir_factory.mktemp("mock-env-path") - ev.environment.env_path = str(mock_path) + def mock_fun(): + return str(mock_path) + ev.environment.env_path = mock_fun yield mock_path - ev.environment.env_path = saved_path + ev.environment.env_path = saved_path_fun @pytest.fixture() diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index 981a6b672d9..48b647d02e7 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -226,7 +226,7 @@ def convert_to_platform_path(path): return format_os_path(path, mode=Path.platform_path) -def substitute_config_variables(path): +def substitute_config_variables(path, allow_env=True): """Substitute placeholders into paths. Spack allows paths in configs to have some placeholders, as follows: @@ -242,15 +242,16 @@ def substitute_config_variables(path): replaced if there is an active environment, and should only be used in environment yaml files. """ - import spack.environment as ev # break circular - _replacements = replacements() - env = ev.active_environment() - if env: - _replacements.update({"env": env.path}) - else: - # If a previous invocation added env, remove it - _replacements.pop("env", None) + if allow_env: + import spack.environment as ev # break circular + + env = ev.active_environment() + if env: + _replacements.update({"env": env.path}) + else: + # If a previous invocation added env, remove it + _replacements.pop("env", None) # Look up replacements def repl(match): @@ -261,9 +262,9 @@ def repl(match): return re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) -def substitute_path_variables(path): +def substitute_path_variables(path, allow_env = True): """Substitute config vars, expand environment vars, expand user home.""" - path = substitute_config_variables(path) + path = substitute_config_variables(path, allow_env) path = os.path.expandvars(path) path = os.path.expanduser(path) return path @@ -305,7 +306,7 @@ def add_padding(path, length): return os.path.join(path, padding) -def canonicalize_path(path): +def canonicalize_path(path, allow_env = True): """Same as substitute_path_variables, but also take absolute path. Arguments: @@ -321,7 +322,7 @@ def canonicalize_path(path): filename = os.path.dirname(path._start_mark.name) assert path._start_mark.name == path._end_mark.name - path = substitute_path_variables(path) + path = substitute_path_variables(path, allow_env) if not os.path.isabs(path): if filename: path = os.path.join(filename, path)