zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Isolate util/path

Browse Source
This commit is contained in:
Douglas Jacobsen 2023-11-06 11:41:37 -07:00
parent 425d3ba8a6
commit 1f0a8755c7
29 changed files with 267 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/spack/spack/binary_distribution.py
View File

@ -44,6 +44,7 @@
import spack.oci.image
import spack.oci.oci
import spack.oci.opener
import spack.paths
import spack.platforms
import spack.relocate as relocate
import spack.repo
@ -532,7 +533,8 @@ def _fetch_and_cache_index(self, mirror_url, cache_entry={}):
def binary_index_location():
"""Set up a BinaryCacheIndex for remote buildcache dbs in the user's homedir."""
cache_root = os.path.join(misc_cache_location(), "indices")
return spack.util.path.canonicalize_path(cache_root)
return spack.util.path.canonicalize_path(cache_root,
replacements=spack.paths.path_replacements())
#: Default binary cache index instance

9
lib/spack/spack/bootstrap/config.py
View File

@ -45,7 +45,8 @@ def spec_for_current_python() -> str:
def root_path() -> str:
"""Root of all the bootstrap related folders"""
return spack.util.path.canonicalize_path(
spack.config.get("bootstrap:root", spack.paths.default_user_bootstrap_path)
spack.config.get("bootstrap:root", spack.paths.default_user_bootstrap_path),
replacements=spack.paths.path_replacements()
)
@ -79,12 +80,14 @@ def spack_python_interpreter() -> Generator:
def _store_path() -> str:
bootstrap_root_path = root_path()
return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "store"))
return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "store"),
replacements=spack.paths.path_replacements())
def _config_path() -> str:
bootstrap_root_path = root_path()
return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "config"))
return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "config"),
replacements=spack.paths.path_replacements())
@contextlib.contextmanager

10
lib/spack/spack/bootstrap/core.py
View File

@ -92,7 +92,10 @@ class Bootstrapper:
def __init__(self, conf: ConfigDictionary) -> None:
self.conf = conf
self.name = conf["name"]
self.metadata_dir = spack.util.path.canonicalize_path(conf["metadata"])
self.metadata_dir = spack.util.path.canonicalize_path(
conf["metadata"],
replacements=spack.paths.path_replacements()
)
# Promote (relative) paths to file urls
url = conf["info"]["url"]
@ -585,7 +588,10 @@ def bootstrapping_sources(scope: Optional[str] = None):
list_of_sources = []
for entry in source_configs:
current = copy.copy(entry)
metadata_dir = spack.util.path.canonicalize_path(entry["metadata"])
metadata_dir = spack.util.path.canonicalize_path(
entry["metadata"],
replacements=spack.paths.path_replacements()
)
metadata_yaml = os.path.join(metadata_dir, METADATA_YAML_FILENAME)
with open(metadata_yaml, encoding="utf-8") as stream:
current.update(spack.util.spack_yaml.load(stream))

4
lib/spack/spack/bootstrap/environment.py
View File

@ -16,6 +16,7 @@
from llnl.util import tty
import spack.environment
import spack.paths
import spack.tengine
import spack.util.cpus
import spack.util.executable
@ -50,7 +51,8 @@ def environment_root(cls) -> pathlib.Path:
environment_dir = f"{python_part}-{arch_part}-{interpreter_part}"
return pathlib.Path(
spack.util.path.canonicalize_path(
os.path.join(bootstrap_root_path, "environments", environment_dir)
os.path.join(bootstrap_root_path, "environments", environment_dir),
replacements=spack.paths.path_replacements()
)
)

6
lib/spack/spack/caches.py
View File

@ -26,7 +26,8 @@ def misc_cache_location():
providers and for which packages provide which tags.
"""
path = spack.config.get("config:misc_cache", spack.paths.default_misc_cache_path)
return spack.util.path.canonicalize_path(path)
return spack.util.path.canonicalize_path(path,
replacements=spack.paths.path_replacements())
def _misc_cache():
@ -49,7 +50,8 @@ def fetch_cache_location():
path = spack.config.get("config:source_cache")
if not path:
path = spack.paths.default_fetch_cache_path
path = spack.util.path.canonicalize_path(path)
path = spack.util.path.canonicalize_path(path,
replacements=spack.paths.path_replacements())
return path

28
lib/spack/spack/cmd/bootstrap.py
View File

@ -18,6 +18,7 @@
import spack.config
import spack.main
import spack.mirror
import spack.paths
import spack.spec
import spack.stage
import spack.util.path
@ -191,7 +192,8 @@ def _root(args):
root = spack.config.get("bootstrap:root", default=None, scope=args.scope)
if root:
root = spack.util.path.canonicalize_path(root)
root = spack.util.path.canonicalize_path(root,
replacements=spack.paths.path_replacements())
print(root)
@ -335,7 +337,8 @@ def _add(args):
raise RuntimeError(msg.format(args.name))
# Check that the metadata file exists
metadata_dir = spack.util.path.canonicalize_path(args.metadata_dir)
metadata_dir = spack.util.path.canonicalize_path(args.metadata_dir,
replacements=spack.paths.path_replacements())
if not os.path.exists(metadata_dir) or not os.path.isdir(metadata_dir):
raise RuntimeError('the directory "{0}" does not exist'.format(args.metadata_dir))
@ -384,7 +387,8 @@ def _remove(args):
def _mirror(args):
mirror_dir = spack.util.path.canonicalize_path(os.path.join(args.root_dir, LOCAL_MIRROR_DIR))
mirror_dir = spack.util.path.canonicalize_path(os.path.join(args.root_dir, LOCAL_MIRROR_DIR),
replacements=spack.paths.path_replacements())
# TODO: Here we are adding gnuconfig manually, but this can be fixed
# TODO: as soon as we have an option to add to a mirror all the possible
@ -433,9 +437,21 @@ def write_metadata(subdir, metadata):
instructions += cmd.format("local-sources", rel_directory)
if args.binary_packages:
abs_directory, rel_directory = write_metadata(subdir="binaries", metadata=BINARY_METADATA)
shutil.copy(spack.util.path.canonicalize_path(CLINGO_JSON), abs_directory)
shutil.copy(spack.util.path.canonicalize_path(GNUPG_JSON), abs_directory)
shutil.copy(spack.util.path.canonicalize_path(PATCHELF_JSON), abs_directory)
shutil.copy(
spack.util.path.canonicalize_path(CLINGO_JSON,
replacements=spack.paths.path_replacements()),
abs_directory
)
shutil.copy(
spack.util.path.canonicalize_path(GNUPG_JSON,
replacements=spack.paths.path_replacements()),
abs_directory
)
shutil.copy(
spack.util.path.canonicalize_path(PATCHELF_JSON,
replacements=spack.paths.path_replacements()),
abs_directory
)
instructions += cmd.format("local-binaries", rel_directory)
print(instructions)

6
lib/spack/spack/cmd/clean.py
View File

@ -14,6 +14,7 @@
import spack.caches
import spack.cmd.test
import spack.config
import spack.paths
import spack.repo
import spack.stage
import spack.store
@ -133,7 +134,10 @@ def clean(parser, args):
remove_python_cache()
if args.bootstrap:
bootstrap_prefix = spack.util.path.canonicalize_path(spack.config.get("bootstrap:root"))
bootstrap_prefix = spack.util.path.canonicalize_path(
spack.config.get("bootstrap:root"),
replacements=spack.paths.path_replacements()
)
msg = 'Removing bootstrapped software and configuration in "{0}"'
tty.msg(msg.format(bootstrap_prefix))
llnl.util.filesystem.remove_directory_contents(bootstrap_prefix)

9
lib/spack/spack/cmd/develop.py
View File

@ -8,6 +8,7 @@
import llnl.util.tty as tty
import spack.cmd
import spack.paths
import spack.spec
import spack.util.path
import spack.version
@ -55,7 +56,10 @@ def develop(parser, args):
# download all dev specs
for name, entry in env.dev_specs.items():
path = entry.get("path", name)
abspath = spack.util.path.canonicalize_path(path, default_wd=env.path)
abspath = spack.util.path.canonicalize_path(
path, default_wd=env.path,
replacements=spack.paths.path_replacements()
)
if os.path.exists(abspath):
msg = "Skipping developer download of %s" % entry["spec"]
@ -86,7 +90,8 @@ def develop(parser, args):
# default path is relative path to spec.name
path = args.path or spec.name
abspath = spack.util.path.canonicalize_path(path, default_wd=env.path)
abspath = spack.util.path.canonicalize_path(path, default_wd=env.path,
replacements=spack.paths.path_replacements())
# clone default: only if the path doesn't exist
clone = args.clone

12
lib/spack/spack/cmd/repo.py
View File

@ -9,6 +9,7 @@
import llnl.util.tty as tty
import spack.config
import spack.paths
import spack.repo
import spack.util.path
from spack.cmd.common import arguments
@ -83,7 +84,8 @@ def repo_add(args):
path = args.path
# real_path is absolute and handles substitution.
canon_path = spack.util.path.canonicalize_path(path)
canon_path = spack.util.path.canonicalize_path(path,
replacements=spack.paths.path_replacements())
# check if the path exists
if not os.path.exists(canon_path):
@ -115,9 +117,13 @@ def repo_remove(args):
namespace_or_path = args.namespace_or_path
# If the argument is a path, remove that repository from config.
canon_path = spack.util.path.canonicalize_path(namespace_or_path)
canon_path = spack.util.path.canonicalize_path(namespace_or_path,
replacements=spack.paths.path_replacements())
for repo_path in repos:
repo_canon_path = spack.util.path.canonicalize_path(repo_path)
repo_canon_path = spack.util.path.canonicalize_path(
repo_path,
replacements=spack.paths.path_replacements()
)
if canon_path == repo_canon_path:
repos.remove(repo_path)
spack.config.set("repos", repos, args.scope)

4
lib/spack/spack/concretize.py
View File

@ -31,6 +31,7 @@
import spack.config
import spack.environment
import spack.error
import spack.paths
import spack.platforms
import spack.repo
import spack.spec
@ -91,7 +92,8 @@ def concretize_develop(self, spec):
if not dev_info:
return False
path = spack.util.path.canonicalize_path(dev_info["path"], default_wd=env.path)
path = spack.util.path.canonicalize_path(dev_info["path"], default_wd=env.path,
replacements=spack.paths.path_replacements())
if "dev_path" in spec.variants:
assert spec.variants["dev_path"].value == path

21
lib/spack/spack/environment/environment.py
View File

@ -90,7 +90,8 @@
def env_root_path():
"""Override default root path if the user specified it"""
return spack.util.path.canonicalize_path(
spack.config.get("config:environments_root", default=default_env_path)
spack.config.get("config:environments_root", default=default_env_path),
replacements=spack.paths.path_replacements()
)
@ -478,7 +479,10 @@ def __init__(
):
self.base = base_path
self.raw_root = root
self.root = spack.util.path.canonicalize_path(root, default_wd=base_path)
self.root = spack.util.path.canonicalize_path(
root, default_wd=base_path,
replacements=spack.paths.path_replacements()
)
self.projections = projections
self.select = select
self.exclude = exclude
@ -493,7 +497,10 @@ def exclude_fn(self, spec):
def update_root(self, new_path):
self.raw_root = new_path
self.root = spack.util.path.canonicalize_path(new_path, default_wd=self.base)
self.root = spack.util.path.canonicalize_path(
new_path, default_wd=self.base,
replacements=spack.paths.path_replacements()
)
def __eq__(self, other):
return all(
@ -985,7 +992,8 @@ def included_config_scopes(self):
missing = []
for i, config_path in enumerate(reversed(includes)):
# allow paths to contain spack config/environment variables, etc.
config_path = substitute_path_variables(config_path)
config_path = substitute_path_variables(config_path,
replacements=spack.paths.path_replacements())
include_url = urllib.parse.urlparse(config_path)
@ -1296,7 +1304,10 @@ def develop(self, spec: Spec, path: str, clone: bool = False) -> bool:
# to be created, then copy it afterwards somewhere else. It would be
# better if we can create the `source_path` directly into its final
# destination.
abspath = spack.util.path.canonicalize_path(path, default_wd=self.path)
abspath = spack.util.path.canonicalize_path(
path, default_wd=self.path,
replacements=spack.paths.path_replacements()
)
pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)
# We construct a package class ourselves, rather than asking for
# Spec.package, since Spec only allows this when it is concrete

4
lib/spack/spack/extensions.py
View File

@ -16,6 +16,7 @@
import spack.config
import spack.error
import spack.paths
import spack.util.path
_extension_regexp = re.compile(r"spack-(\w[-\w]*)$")
@ -109,7 +110,8 @@ def ensure_package_creation(name):
def get_extension_paths():
"""Return the list of canonicalized extension paths from config:extensions."""
extension_paths = spack.config.get("config:extensions") or []
paths = [spack.util.path.canonicalize_path(p) for p in extension_paths]
r = spack.paths.path_replacements()
paths = [spack.util.path.canonicalize_path(p, replacements=r) for p in extension_paths]
return paths

3
lib/spack/spack/install_test.py
View File

@ -91,7 +91,8 @@ def get_test_stage_dir():
the default test stage path
"""
return spack.util.path.canonicalize_path(
spack.config.get("config:test_stage", spack.paths.default_test_path)
spack.config.get("config:test_stage", spack.paths.default_test_path),
replacements=spack.paths.path_replacements()
)

6
lib/spack/spack/mirror.py
View File

@ -30,6 +30,7 @@
import spack.fetch_strategy
import spack.mirror
import spack.oci.image
import spack.paths
import spack.spec
import spack.util.path
import spack.util.spack_json as sjson
@ -51,7 +52,10 @@ def _url_or_path_to_url(url_or_path: str) -> str:
return url_or_path
# Otherwise we interpret it as path, and we should promote it to file:// URL.
return url_util.path_to_file_url(spack.util.path.canonicalize_path(url_or_path))
return url_util.path_to_file_url(
spack.util.path.canonicalize_path(url_or_path,
replacements=spack.paths.path_replacements())
)
class Mirror:

3
lib/spack/spack/modules/common.py
View File

@ -225,7 +225,8 @@ def root_path(name, module_set_name):
roots = spack.config.merge_yaml(defaults, roots)
path = roots.get(name, os.path.join(spack.paths.share_path, name))
return spack.util.path.canonicalize_path(path)
return spack.util.path.canonicalize_path(path,
replacements=spack.paths.path_replacements())
def generate_module_index(root, modules, overwrite=False):

3
lib/spack/spack/package_base.py
View File

@ -829,7 +829,8 @@ def name(cls):
@classproperty
def global_license_dir(cls):
"""Returns the directory where license files for all packages are stored."""
return spack.util.path.canonicalize_path(spack.config.get("config:license_dir"))
return spack.util.path.canonicalize_path(spack.config.get("config:license_dir"),
replacements=spack.paths.path_replacements())
@property
def global_license_file(self):

4
lib/spack/spack/package_prefs.py
View File

@ -7,6 +7,7 @@
import spack.error
import spack.repo
import spack.paths
from spack.config import ConfigError
from spack.util.path import canonicalize_path
from spack.version import Version
@ -177,7 +178,8 @@ def _package(maybe_abstract_spec):
spec_str = entry["spec"]
external_path = entry.get("prefix", None)
if external_path:
external_path = canonicalize_path(external_path)
external_path = canonicalize_path(external_path,
replacements=spack.paths.path_replacements())
external_modules = entry.get("modules", None)
external_spec = spack.spec.Spec.from_detection(
spack.spec.Spec(

53
lib/spack/spack/paths.py
View File

@ -10,10 +10,14 @@
dependencies.
"""
import os
import tempfile
from datetime import date
from pathlib import PurePath
import llnl.util.filesystem
import spack.util.path
#: This file lives in $prefix/lib/spack/spack/__file__
prefix = str(PurePath(llnl.util.filesystem.ancestor(__file__, 4)))
@ -136,3 +140,52 @@ def _get_system_config_path():
#: System configuration location
system_config_path = _get_system_config_path()
def architecture():
# break circular import
import spack.platforms
import spack.spec
host_platform = spack.platforms.host()
host_os = host_platform.operating_system("default_os")
host_target = host_platform.target("default_target")
return spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))
def get_user():
# User pwd where available because it accounts for effective uids when using ksu and similar
try:
# user pwd for unix systems
import pwd
return pwd.getpwuid(os.geteuid()).pw_name
except ImportError:
# fallback on getpass
return getpass.getuser()
def path_replacements():
# break circular imports
import spack.environment as ev
import spack.paths
arch = architecture()
return {
"spack": lambda: spack.paths.prefix,
"user": lambda: get_user(),
"tempdir": lambda: tempfile.gettempdir(),
"user_cache_path": lambda: spack.paths.user_cache_path,
"architecture": lambda: arch,
"arch": lambda: arch,
"platform": lambda: arch.platform,
"operating_system": lambda: arch.os,
"os": lambda: arch.os,
"target": lambda: arch.target,
"target_family": lambda: arch.target.microarchitecture.family,
"date": lambda: date.today().strftime("%Y-%m-%d"),
"env": lambda: ev.active_environment().path if ev.active_environment() else \
spack.util.path.NOMATCH,
}

9
lib/spack/spack/repo.py
View File

@ -37,6 +37,7 @@
import spack.config
import spack.error
import spack.patch
import spack.paths
import spack.provider_index
import spack.spec
import spack.tag
@ -928,7 +929,10 @@ def __init__(self, root, cache=None):
"""
# Root directory, containing _repo.yaml and package dirs
# Allow roots to by spack-relative by starting with '$spack'
self.root = spack.util.path.canonicalize_path(root)
self.root = spack.util.path.canonicalize_path(
root,
replacements=spack.paths.path_replacements()
)
# check and raise BadRepoError on fail.
def check(condition, msg):
@ -1327,7 +1331,8 @@ def create_repo(root, namespace=None, subdir=packages_dir_name):
If the namespace is not provided, use basename of root.
Return the canonicalized path and namespace of the created repository.
"""
root = spack.util.path.canonicalize_path(root)
root = spack.util.path.canonicalize_path(root,
replacements=spack.paths.path_replacements())
if not namespace:
namespace = os.path.basename(root)

9
lib/spack/spack/solver/asp.py
View File

@ -41,6 +41,7 @@
import spack.error
import spack.package_base
import spack.package_prefs
import spack.paths
import spack.platforms
import spack.repo
import spack.spec
@ -2602,7 +2603,10 @@ def setup(
dev_specs = tuple(
spack.spec.Spec(info["spec"]).constrained(
"dev_path=%s"
% spack.util.path.canonicalize_path(info["path"], default_wd=env.path)
% spack.util.path.canonicalize_path(
info["path"], default_wd=env.path,
replacements=spack.paths.path_replacements()
)
)
for name, info in env.dev_specs.items()
)
@ -3119,7 +3123,8 @@ def _develop_specs_from_env(spec, env):
if not dev_info:
return
path = spack.util.path.canonicalize_path(dev_info["path"], default_wd=env.path)
path = spack.util.path.canonicalize_path(dev_info["path"], default_wd=env.path,
replacements=spack.paths.path_replacements())
if "dev_path" in spec.variants:
error_msg = (

12
lib/spack/spack/stage.py
View File

@ -150,7 +150,8 @@ def _resolve_paths(candidates):
Adjustments involve removing extra $user from $tempdir if $tempdir includes
$user and appending $user if it is not present in the path.
"""
temp_path = sup.canonicalize_path("$tempdir")
temp_path = sup.canonicalize_path("$tempdir",
replacements=spack.paths.path_replacements())
user = getpass.getuser()
tmp_has_usr = user in temp_path.split(os.path.sep)
@ -162,7 +163,8 @@ def _resolve_paths(candidates):
path = path.replace("/$user", "", 1)
# Ensure the path is unique per user.
can_path = sup.canonicalize_path(path)
can_path = sup.canonicalize_path(path,
replacements=spack.paths.path_replacements())
# When multiple users share a stage root, we can avoid conflicts between
# them by adding a per-user subdirectory.
# Avoid doing this on Windows to keep stage absolute path as short as possible.
@ -199,9 +201,11 @@ def get_stage_root():
def _mirror_roots():
mirrors = spack.config.get("mirrors")
return [
sup.substitute_path_variables(root)
sup.substitute_path_variables(root,
replacements=spack.paths.path_replacements())
if root.endswith(os.sep)
else sup.substitute_path_variables(root) + os.sep
else sup.substitute_path_variables(root,
replacemnts=spack.paths.path_replacements()) + os.sep
for root in mirrors.values()
]

13
lib/spack/spack/store.py
View File

@ -77,7 +77,10 @@ def parse_install_tree(config_dict):
if isinstance(install_tree, str):
tty.warn("Using deprecated format for configuring install_tree")
unpadded_root = install_tree
unpadded_root = spack.util.path.canonicalize_path(unpadded_root)
unpadded_root = spack.util.path.canonicalize_path(
unpadded_root,
replacements=spack.paths.path_replacements()
)
# construct projection from previous values for backwards compatibility
all_projection = config_dict.get(
"install_path_scheme", spack.directory_layout.default_projections["all"]
@ -86,7 +89,10 @@ def parse_install_tree(config_dict):
projections = {"all": all_projection}
else:
unpadded_root = install_tree.get("root", DEFAULT_INSTALL_TREE_ROOT)
unpadded_root = spack.util.path.canonicalize_path(unpadded_root)
unpadded_root = spack.util.path.canonicalize_path(
unpadded_root,
replacements=spack.paths.path_replacements()
)
padded_length = install_tree.get("padded_length", False)
if padded_length is True:
@ -267,7 +273,8 @@ def _construct_upstream_dbs_from_install_roots(
for install_root in reversed(install_roots):
upstream_dbs = list(accumulated_upstream_dbs)
next_db = spack.database.Database(
spack.util.path.canonicalize_path(install_root),
spack.util.path.canonicalize_path(install_root,
replacements=spack.paths.path_replacements()),
is_upstream=True,
upstream_dbs=upstream_dbs,
)

5
lib/spack/spack/tengine.py
View File

@ -10,6 +10,7 @@
import spack.config
import spack.extensions
import spack.paths
from spack.util.path import canonicalize_path
@ -76,7 +77,9 @@ def make_environment(dirs: Optional[Tuple[str, ...]] = None):
# Default directories where to search for templates
builtins = spack.config.get("config:template_dirs", ["$spack/share/spack/templates"])
extensions = spack.extensions.get_template_dirs()
dirs = tuple(canonicalize_path(d) for d in itertools.chain(builtins, extensions))
r = spack.paths.path_replacements()
dirs = tuple(canonicalize_path(d, replacements=r)
for d in itertools.chain(builtins, extensions))
# Loader for the templates
loader = jinja2.FileSystemLoader(dirs)

6
lib/spack/spack/test/bootstrap.py
View File

@ -10,6 +10,7 @@
import spack.bootstrap.core
import spack.compilers
import spack.environment
import spack.paths
import spack.store
import spack.util.path
@ -81,7 +82,10 @@ def test_store_path_customization(config_value, expected, mutable_config):
# Check the store path
current = spack.bootstrap.config.store_path()
assert current == spack.util.path.canonicalize_path(expected)
assert current == spack.util.path.canonicalize_path(
expected,
replacements=spack.paths.path_replacements()
)
def test_raising_exception_if_bootstrap_disabled(mutable_config):

11
lib/spack/spack/test/cmd/develop.py
View File

@ -10,6 +10,7 @@
import llnl.util.filesystem as fs
import spack.environment as ev
import spack.paths
import spack.spec
from spack.main import SpackCommand
@ -106,7 +107,10 @@ def test_develop_canonicalize_path(self, monkeypatch, config):
env("create", "test")
with ev.read("test") as e:
path = "../$user"
abspath = spack.util.path.canonicalize_path(path, e.path)
abspath = spack.util.path.canonicalize_path(
path, e.path,
replacements=spack.paths.path_replacements()
)
def check_path(stage, dest):
assert dest == abspath
@ -123,7 +127,10 @@ def test_develop_canonicalize_path_no_args(self, monkeypatch, config):
env("create", "test")
with ev.read("test") as e:
path = "$user"
abspath = spack.util.path.canonicalize_path(path, e.path)
abspath = spack.util.path.canonicalize_path(
path, e.path,
replacements=spack.paths.path_replacements()
)
def check_path(stage, dest):
assert dest == abspath

55
lib/spack/spack/test/config.py
View File

@ -338,48 +338,59 @@ def __init__(self, path):
def test_substitute_config_variables(mock_low_high_config, monkeypatch):
prefix = spack.paths.prefix.lstrip("/")
r = spack.paths.path_replacements()
assert cross_plat_join(
os.sep + os.path.join("foo", "bar", "baz"), prefix
) == spack_path.canonicalize_path("/foo/bar/baz/$spack")
) == spack_path.canonicalize_path("/foo/bar/baz/$spack",
replacements=r)
assert cross_plat_join(
spack.paths.prefix, os.path.join("foo", "bar", "baz")
) == spack_path.canonicalize_path("$spack/foo/bar/baz/")
) == spack_path.canonicalize_path("$spack/foo/bar/baz/",
replacements=r)
assert cross_plat_join(
os.sep + os.path.join("foo", "bar", "baz"), prefix, os.path.join("foo", "bar", "baz")
) == spack_path.canonicalize_path("/foo/bar/baz/$spack/foo/bar/baz/")
) == spack_path.canonicalize_path("/foo/bar/baz/$spack/foo/bar/baz/",
replacements=r)
assert cross_plat_join(
os.sep + os.path.join("foo", "bar", "baz"), prefix
) == spack_path.canonicalize_path("/foo/bar/baz/${spack}")
) == spack_path.canonicalize_path("/foo/bar/baz/${spack}",
replacements=r)
assert cross_plat_join(
spack.paths.prefix, os.path.join("foo", "bar", "baz")
) == spack_path.canonicalize_path("${spack}/foo/bar/baz/")
) == spack_path.canonicalize_path("${spack}/foo/bar/baz/",
replacements=r)
assert cross_plat_join(
os.sep + os.path.join("foo", "bar", "baz"), prefix, os.path.join("foo", "bar", "baz")
) == spack_path.canonicalize_path("/foo/bar/baz/${spack}/foo/bar/baz/")
) == spack_path.canonicalize_path("/foo/bar/baz/${spack}/foo/bar/baz/",
replacements=r)
assert cross_plat_join(
os.sep + os.path.join("foo", "bar", "baz"), prefix, os.path.join("foo", "bar", "baz")
) != spack_path.canonicalize_path("/foo/bar/baz/${spack/foo/bar/baz/")
) != spack_path.canonicalize_path("/foo/bar/baz/${spack/foo/bar/baz/",
replacements=r)
# $env replacement is a no-op when no environment is active
assert spack_path.canonicalize_path(
os.sep + os.path.join("foo", "bar", "baz", "$env")
os.sep + os.path.join("foo", "bar", "baz", "$env"),
replacements=r
) == os.sep + os.path.join("foo", "bar", "baz", "$env")
# Fake an active environment and $env is replaced properly
fake_env_path = os.sep + os.path.join("quux", "quuux")
monkeypatch.setattr(ev, "active_environment", lambda: MockEnv(fake_env_path))
assert spack_path.canonicalize_path("$env/foo/bar/baz") == os.path.join(
assert spack_path.canonicalize_path("$env/foo/bar/baz",
replacements=r) == os.path.join(
fake_env_path, os.path.join("foo", "bar", "baz")
)
# relative paths without source information are relative to cwd
assert spack_path.canonicalize_path(os.path.join("foo", "bar", "baz")) == os.path.abspath(
assert spack_path.canonicalize_path(os.path.join("foo", "bar", "baz"),
replacements=r) == os.path.abspath(
os.path.join("foo", "bar", "baz")
)
@ -389,19 +400,22 @@ def test_substitute_config_variables(mock_low_high_config, monkeypatch):
)
spack.config.CONFIG.clear_caches()
path = spack.config.get("modules:default:roots:lmod")
assert spack_path.canonicalize_path(path) == os.path.normpath(
assert spack_path.canonicalize_path(path,
replacements=r) == os.path.normpath(
os.path.join(mock_low_high_config.scopes["low"].path, os.path.join("foo", "bar", "baz"))
)
# test architecture information is in replacements
assert spack_path.canonicalize_path(
os.path.join("foo", "$platform", "bar")
os.path.join("foo", "$platform", "bar"),
replacements=r
) == os.path.abspath(os.path.join("foo", "test", "bar"))
host_target = spack.platforms.host().target("default_target")
host_target_family = str(host_target.microarchitecture.family)
assert spack_path.canonicalize_path(
os.path.join("foo", "$target_family", "bar")
os.path.join("foo", "$target_family", "bar"),
replacements=r
) == os.path.abspath(os.path.join("foo", host_target_family, "bar"))
@ -438,28 +452,33 @@ def test_substitute_user(mock_low_high_config):
assert os.sep + os.path.join(
"foo", "bar"
) + os.sep + user + os.sep + "baz" == spack_path.canonicalize_path(
os.sep + os.path.join("foo", "bar", "$user", "baz")
os.sep + os.path.join("foo", "bar", "$user", "baz"),
replacements=spack.paths.path_replacements()
)
def test_substitute_user_cache(mock_low_high_config):
user_cache_path = spack.paths.user_cache_path
assert user_cache_path + os.sep + "baz" == spack_path.canonicalize_path(
os.path.join("$user_cache_path", "baz")
os.path.join("$user_cache_path", "baz"),
replacements=spack.paths.path_replacements()
)
def test_substitute_tempdir(mock_low_high_config):
tempdir = tempfile.gettempdir()
assert tempdir == spack_path.canonicalize_path("$tempdir")
assert tempdir == spack_path.canonicalize_path("$tempdir",
replacements=spack.paths.path_replacements())
assert tempdir + os.sep + os.path.join("foo", "bar", "baz") == spack_path.canonicalize_path(
os.path.join("$tempdir", "foo", "bar", "baz")
os.path.join("$tempdir", "foo", "bar", "baz"),
replacements=spack.paths.path_replacements()
)
def test_substitute_date(mock_low_high_config):
test_path = os.path.join("hello", "world", "on", "$date")
new_path = spack_path.canonicalize_path(test_path)
new_path = spack_path.canonicalize_path(test_path,
replacements=spack.paths.path_replacements())
assert "$date" in test_path
assert date.today().strftime("%Y-%m-%d") in new_path

6
lib/spack/spack/test/stage.py
View File

@ -734,7 +734,8 @@ def test_resolve_paths(self):
assert spack.stage._resolve_paths(paths) == paths
tempdir = "$tempdir"
can_tempdir = canonicalize_path(tempdir)
can_tempdir = canonicalize_path(tempdir,
replacements=spack.paths.path_replacements())
user = getpass.getuser()
temp_has_user = user in can_tempdir.split(os.sep)
paths = [
@ -744,7 +745,8 @@ def test_resolve_paths(self):
os.path.join(tempdir, "$user", "stage", "$user"),
]
res_paths = [canonicalize_path(p) for p in paths]
r = spack.paths.path_replacements()
res_paths = [canonicalize_path(p, replacements=r) for p in paths]
if temp_has_user:
res_paths[1] = can_tempdir
res_paths[2] = os.path.join(can_tempdir, user)

4
lib/spack/spack/test/tengine.py
View File

@ -7,6 +7,7 @@
import pytest
import spack.config
import spack.paths
import spack.tengine as tengine
from spack.util.path import canonicalize_path
@ -70,8 +71,9 @@ class TestTengineEnvironment:
def test_template_retrieval(self):
"""Tests the template retrieval mechanism hooked into config files"""
# Check the directories are correct
r = spack.paths.path_replacements()
template_dirs = spack.config.get("config:template_dirs")
template_dirs = tuple([canonicalize_path(x) for x in template_dirs])
template_dirs = tuple([canonicalize_path(x, replacements=r) for x in template_dirs])
assert len(template_dirs) == 3
env = tengine.make_environment(template_dirs)

76
lib/spack/spack/util/path.py
View File

@ -21,62 +21,12 @@
import spack.util.spack_yaml as syaml
__all__ = ["substitute_config_variables", "substitute_path_variables", "canonicalize_path"]
def architecture():
# break circular import
import spack.platforms
import spack.spec
host_platform = spack.platforms.host()
host_os = host_platform.operating_system("default_os")
host_target = host_platform.target("default_target")
return spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))
def get_user():
# User pwd where available because it accounts for effective uids when using ksu and similar
try:
# user pwd for unix systems
import pwd
return pwd.getpwuid(os.geteuid()).pw_name
except ImportError:
# fallback on getpass
return getpass.getuser()
__all__ = ["substitute_config_variables", "substitute_path_variables", "canonicalize_path", "NOMATCH"]
# return value for replacements with no match
NOMATCH = object()
# Substitutions to perform
def replacements():
# break circular imports
import spack.environment as ev
import spack.paths
arch = architecture()
return {
"spack": lambda: spack.paths.prefix,
"user": lambda: get_user(),
"tempdir": lambda: tempfile.gettempdir(),
"user_cache_path": lambda: spack.paths.user_cache_path,
"architecture": lambda: arch,
"arch": lambda: arch,
"platform": lambda: arch.platform,
"operating_system": lambda: arch.os,
"os": lambda: arch.os,
"target": lambda: arch.target,
"target_family": lambda: arch.target.microarchitecture.family,
"date": lambda: date.today().strftime("%Y-%m-%d"),
"env": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH,
}
# This is intended to be longer than the part of the install path
# spack generates from the root path we give it. Included in the
# estimate:
@ -144,7 +94,7 @@ def get_system_path_max():
return sys_max_path_length
def substitute_config_variables(path):
def substitute_config_variables(path, replacements={}):
"""Substitute placeholders into paths.
Spack allows paths in configs to have some placeholders, as follows:
@ -168,22 +118,20 @@ def substitute_config_variables(path):
replaced if there is an active environment, and should only be used in
environment yaml files.
"""
_replacements = replacements()
# Look up replacements
def repl(match):
m = match.group(0)
key = m.strip("${}").lower()
repl = _replacements.get(key, lambda: m)()
repl = replacements.get(key, lambda: m)()
return m if repl is NOMATCH else str(repl)
# Replace $var or ${var}.
return re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path)
def substitute_path_variables(path):
def substitute_path_variables(path, replacements={}):
"""Substitute config vars, expand environment vars, expand user home."""
path = substitute_config_variables(path)
path = substitute_config_variables(path, replacements=replacements)
path = os.path.expandvars(path)
path = os.path.expanduser(path)
return path
@ -225,7 +173,7 @@ def add_padding(path, length):
return os.path.join(path, padding)
def canonicalize_path(path, default_wd=None):
def canonicalize_path(path, default_wd=None, replacements=None):
"""Same as substitute_path_variables, but also take absolute path.
If the string is a yaml object with file annotations, make absolute paths
@ -234,6 +182,7 @@ def canonicalize_path(path, default_wd=None):
Arguments:
path (str): path being converted as needed
replacements (dict): dictionary of replacements to use
Returns:
(str): An absolute path with path variable substitution
@ -245,7 +194,16 @@ def canonicalize_path(path, default_wd=None):
filename = os.path.dirname(path._start_mark.name)
assert path._start_mark.name == path._end_mark.name
path = substitute_path_variables(path)
if replacements is None:
_replacements = {}
else:
_replacements = replacements
if not isinstance(_replacements, dict):
tty.die("Replacements returned by replacements func are of type"
f"{type(replacements)} and not of the expected type of dict.")
path = substitute_path_variables(path, replacements=_replacements)
if not os.path.isabs(path):
if filename:
path = os.path.join(filename, path)