Isolate util/path

This commit is contained in:
Douglas Jacobsen 2023-11-06 11:41:37 -07:00
parent 425d3ba8a6
commit 1f0a8755c7
29 changed files with 267 additions and 128 deletions

View File

@ -44,6 +44,7 @@
import spack.oci.image import spack.oci.image
import spack.oci.oci import spack.oci.oci
import spack.oci.opener import spack.oci.opener
import spack.paths
import spack.platforms import spack.platforms
import spack.relocate as relocate import spack.relocate as relocate
import spack.repo import spack.repo
@ -532,7 +533,8 @@ def _fetch_and_cache_index(self, mirror_url, cache_entry={}):
def binary_index_location(): def binary_index_location():
"""Set up a BinaryCacheIndex for remote buildcache dbs in the user's homedir.""" """Set up a BinaryCacheIndex for remote buildcache dbs in the user's homedir."""
cache_root = os.path.join(misc_cache_location(), "indices") cache_root = os.path.join(misc_cache_location(), "indices")
return spack.util.path.canonicalize_path(cache_root) return spack.util.path.canonicalize_path(cache_root,
replacements=spack.paths.path_replacements())
#: Default binary cache index instance #: Default binary cache index instance

View File

@ -45,7 +45,8 @@ def spec_for_current_python() -> str:
def root_path() -> str: def root_path() -> str:
"""Root of all the bootstrap related folders""" """Root of all the bootstrap related folders"""
return spack.util.path.canonicalize_path( return spack.util.path.canonicalize_path(
spack.config.get("bootstrap:root", spack.paths.default_user_bootstrap_path) spack.config.get("bootstrap:root", spack.paths.default_user_bootstrap_path),
replacements=spack.paths.path_replacements()
) )
@ -79,12 +80,14 @@ def spack_python_interpreter() -> Generator:
def _store_path() -> str: def _store_path() -> str:
bootstrap_root_path = root_path() bootstrap_root_path = root_path()
return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "store")) return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "store"),
replacements=spack.paths.path_replacements())
def _config_path() -> str: def _config_path() -> str:
bootstrap_root_path = root_path() bootstrap_root_path = root_path()
return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "config")) return spack.util.path.canonicalize_path(os.path.join(bootstrap_root_path, "config"),
replacements=spack.paths.path_replacements())
@contextlib.contextmanager @contextlib.contextmanager

View File

@ -92,7 +92,10 @@ class Bootstrapper:
def __init__(self, conf: ConfigDictionary) -> None: def __init__(self, conf: ConfigDictionary) -> None:
self.conf = conf self.conf = conf
self.name = conf["name"] self.name = conf["name"]
self.metadata_dir = spack.util.path.canonicalize_path(conf["metadata"]) self.metadata_dir = spack.util.path.canonicalize_path(
conf["metadata"],
replacements=spack.paths.path_replacements()
)
# Promote (relative) paths to file urls # Promote (relative) paths to file urls
url = conf["info"]["url"] url = conf["info"]["url"]
@ -585,7 +588,10 @@ def bootstrapping_sources(scope: Optional[str] = None):
list_of_sources = [] list_of_sources = []
for entry in source_configs: for entry in source_configs:
current = copy.copy(entry) current = copy.copy(entry)
metadata_dir = spack.util.path.canonicalize_path(entry["metadata"]) metadata_dir = spack.util.path.canonicalize_path(
entry["metadata"],
replacements=spack.paths.path_replacements()
)
metadata_yaml = os.path.join(metadata_dir, METADATA_YAML_FILENAME) metadata_yaml = os.path.join(metadata_dir, METADATA_YAML_FILENAME)
with open(metadata_yaml, encoding="utf-8") as stream: with open(metadata_yaml, encoding="utf-8") as stream:
current.update(spack.util.spack_yaml.load(stream)) current.update(spack.util.spack_yaml.load(stream))

View File

@ -16,6 +16,7 @@
from llnl.util import tty from llnl.util import tty
import spack.environment import spack.environment
import spack.paths
import spack.tengine import spack.tengine
import spack.util.cpus import spack.util.cpus
import spack.util.executable import spack.util.executable
@ -50,7 +51,8 @@ def environment_root(cls) -> pathlib.Path:
environment_dir = f"{python_part}-{arch_part}-{interpreter_part}" environment_dir = f"{python_part}-{arch_part}-{interpreter_part}"
return pathlib.Path( return pathlib.Path(
spack.util.path.canonicalize_path( spack.util.path.canonicalize_path(
os.path.join(bootstrap_root_path, "environments", environment_dir) os.path.join(bootstrap_root_path, "environments", environment_dir),
replacements=spack.paths.path_replacements()
) )
) )

View File

@ -26,7 +26,8 @@ def misc_cache_location():
providers and for which packages provide which tags. providers and for which packages provide which tags.
""" """
path = spack.config.get("config:misc_cache", spack.paths.default_misc_cache_path) path = spack.config.get("config:misc_cache", spack.paths.default_misc_cache_path)
return spack.util.path.canonicalize_path(path) return spack.util.path.canonicalize_path(path,
replacements=spack.paths.path_replacements())
def _misc_cache(): def _misc_cache():
@ -49,7 +50,8 @@ def fetch_cache_location():
path = spack.config.get("config:source_cache") path = spack.config.get("config:source_cache")
if not path: if not path:
path = spack.paths.default_fetch_cache_path path = spack.paths.default_fetch_cache_path
path = spack.util.path.canonicalize_path(path) path = spack.util.path.canonicalize_path(path,
replacements=spack.paths.path_replacements())
return path return path

View File

@ -18,6 +18,7 @@
import spack.config import spack.config
import spack.main import spack.main
import spack.mirror import spack.mirror
import spack.paths
import spack.spec import spack.spec
import spack.stage import spack.stage
import spack.util.path import spack.util.path
@ -191,7 +192,8 @@ def _root(args):
root = spack.config.get("bootstrap:root", default=None, scope=args.scope) root = spack.config.get("bootstrap:root", default=None, scope=args.scope)
if root: if root:
root = spack.util.path.canonicalize_path(root) root = spack.util.path.canonicalize_path(root,
replacements=spack.paths.path_replacements())
print(root) print(root)
@ -335,7 +337,8 @@ def _add(args):
raise RuntimeError(msg.format(args.name)) raise RuntimeError(msg.format(args.name))
# Check that the metadata file exists # Check that the metadata file exists
metadata_dir = spack.util.path.canonicalize_path(args.metadata_dir) metadata_dir = spack.util.path.canonicalize_path(args.metadata_dir,
replacements=spack.paths.path_replacements())
if not os.path.exists(metadata_dir) or not os.path.isdir(metadata_dir): if not os.path.exists(metadata_dir) or not os.path.isdir(metadata_dir):
raise RuntimeError('the directory "{0}" does not exist'.format(args.metadata_dir)) raise RuntimeError('the directory "{0}" does not exist'.format(args.metadata_dir))
@ -384,7 +387,8 @@ def _remove(args):
def _mirror(args): def _mirror(args):
mirror_dir = spack.util.path.canonicalize_path(os.path.join(args.root_dir, LOCAL_MIRROR_DIR)) mirror_dir = spack.util.path.canonicalize_path(os.path.join(args.root_dir, LOCAL_MIRROR_DIR),
replacements=spack.paths.path_replacements())
# TODO: Here we are adding gnuconfig manually, but this can be fixed # TODO: Here we are adding gnuconfig manually, but this can be fixed
# TODO: as soon as we have an option to add to a mirror all the possible # TODO: as soon as we have an option to add to a mirror all the possible
@ -433,9 +437,21 @@ def write_metadata(subdir, metadata):
instructions += cmd.format("local-sources", rel_directory) instructions += cmd.format("local-sources", rel_directory)
if args.binary_packages: if args.binary_packages:
abs_directory, rel_directory = write_metadata(subdir="binaries", metadata=BINARY_METADATA) abs_directory, rel_directory = write_metadata(subdir="binaries", metadata=BINARY_METADATA)
shutil.copy(spack.util.path.canonicalize_path(CLINGO_JSON), abs_directory) shutil.copy(
shutil.copy(spack.util.path.canonicalize_path(GNUPG_JSON), abs_directory) spack.util.path.canonicalize_path(CLINGO_JSON,
shutil.copy(spack.util.path.canonicalize_path(PATCHELF_JSON), abs_directory) replacements=spack.paths.path_replacements()),
abs_directory
)
shutil.copy(
spack.util.path.canonicalize_path(GNUPG_JSON,
replacements=spack.paths.path_replacements()),
abs_directory
)
shutil.copy(
spack.util.path.canonicalize_path(PATCHELF_JSON,
replacements=spack.paths.path_replacements()),
abs_directory
)
instructions += cmd.format("local-binaries", rel_directory) instructions += cmd.format("local-binaries", rel_directory)
print(instructions) print(instructions)

View File

@ -14,6 +14,7 @@
import spack.caches import spack.caches
import spack.cmd.test import spack.cmd.test
import spack.config import spack.config
import spack.paths
import spack.repo import spack.repo
import spack.stage import spack.stage
import spack.store import spack.store
@ -133,7 +134,10 @@ def clean(parser, args):
remove_python_cache() remove_python_cache()
if args.bootstrap: if args.bootstrap:
bootstrap_prefix = spack.util.path.canonicalize_path(spack.config.get("bootstrap:root")) bootstrap_prefix = spack.util.path.canonicalize_path(
spack.config.get("bootstrap:root"),
replacements=spack.paths.path_replacements()
)
msg = 'Removing bootstrapped software and configuration in "{0}"' msg = 'Removing bootstrapped software and configuration in "{0}"'
tty.msg(msg.format(bootstrap_prefix)) tty.msg(msg.format(bootstrap_prefix))
llnl.util.filesystem.remove_directory_contents(bootstrap_prefix) llnl.util.filesystem.remove_directory_contents(bootstrap_prefix)

View File

@ -8,6 +8,7 @@
import llnl.util.tty as tty import llnl.util.tty as tty
import spack.cmd import spack.cmd
import spack.paths
import spack.spec import spack.spec
import spack.util.path import spack.util.path
import spack.version import spack.version
@ -55,7 +56,10 @@ def develop(parser, args):
# download all dev specs # download all dev specs
for name, entry in env.dev_specs.items(): for name, entry in env.dev_specs.items():
path = entry.get("path", name) path = entry.get("path", name)
abspath = spack.util.path.canonicalize_path(path, default_wd=env.path) abspath = spack.util.path.canonicalize_path(
path, default_wd=env.path,
replacements=spack.paths.path_replacements()
)
if os.path.exists(abspath): if os.path.exists(abspath):
msg = "Skipping developer download of %s" % entry["spec"] msg = "Skipping developer download of %s" % entry["spec"]
@ -86,7 +90,8 @@ def develop(parser, args):
# default path is relative path to spec.name # default path is relative path to spec.name
path = args.path or spec.name path = args.path or spec.name
abspath = spack.util.path.canonicalize_path(path, default_wd=env.path) abspath = spack.util.path.canonicalize_path(path, default_wd=env.path,
replacements=spack.paths.path_replacements())
# clone default: only if the path doesn't exist # clone default: only if the path doesn't exist
clone = args.clone clone = args.clone

View File

@ -9,6 +9,7 @@
import llnl.util.tty as tty import llnl.util.tty as tty
import spack.config import spack.config
import spack.paths
import spack.repo import spack.repo
import spack.util.path import spack.util.path
from spack.cmd.common import arguments from spack.cmd.common import arguments
@ -83,7 +84,8 @@ def repo_add(args):
path = args.path path = args.path
# real_path is absolute and handles substitution. # real_path is absolute and handles substitution.
canon_path = spack.util.path.canonicalize_path(path) canon_path = spack.util.path.canonicalize_path(path,
replacements=spack.paths.path_replacements())
# check if the path exists # check if the path exists
if not os.path.exists(canon_path): if not os.path.exists(canon_path):
@ -115,9 +117,13 @@ def repo_remove(args):
namespace_or_path = args.namespace_or_path namespace_or_path = args.namespace_or_path
# If the argument is a path, remove that repository from config. # If the argument is a path, remove that repository from config.
canon_path = spack.util.path.canonicalize_path(namespace_or_path) canon_path = spack.util.path.canonicalize_path(namespace_or_path,
replacements=spack.paths.path_replacements())
for repo_path in repos: for repo_path in repos:
repo_canon_path = spack.util.path.canonicalize_path(repo_path) repo_canon_path = spack.util.path.canonicalize_path(
repo_path,
replacements=spack.paths.path_replacements()
)
if canon_path == repo_canon_path: if canon_path == repo_canon_path:
repos.remove(repo_path) repos.remove(repo_path)
spack.config.set("repos", repos, args.scope) spack.config.set("repos", repos, args.scope)

View File

@ -31,6 +31,7 @@
import spack.config import spack.config
import spack.environment import spack.environment
import spack.error import spack.error
import spack.paths
import spack.platforms import spack.platforms
import spack.repo import spack.repo
import spack.spec import spack.spec
@ -91,7 +92,8 @@ def concretize_develop(self, spec):
if not dev_info: if not dev_info:
return False return False
path = spack.util.path.canonicalize_path(dev_info["path"], default_wd=env.path) path = spack.util.path.canonicalize_path(dev_info["path"], default_wd=env.path,
replacements=spack.paths.path_replacements())
if "dev_path" in spec.variants: if "dev_path" in spec.variants:
assert spec.variants["dev_path"].value == path assert spec.variants["dev_path"].value == path

View File

@ -90,7 +90,8 @@
def env_root_path(): def env_root_path():
"""Override default root path if the user specified it""" """Override default root path if the user specified it"""
return spack.util.path.canonicalize_path( return spack.util.path.canonicalize_path(
spack.config.get("config:environments_root", default=default_env_path) spack.config.get("config:environments_root", default=default_env_path),
replacements=spack.paths.path_replacements()
) )
@ -478,7 +479,10 @@ def __init__(
): ):
self.base = base_path self.base = base_path
self.raw_root = root self.raw_root = root
self.root = spack.util.path.canonicalize_path(root, default_wd=base_path) self.root = spack.util.path.canonicalize_path(
root, default_wd=base_path,
replacements=spack.paths.path_replacements()
)
self.projections = projections self.projections = projections
self.select = select self.select = select
self.exclude = exclude self.exclude = exclude
@ -493,7 +497,10 @@ def exclude_fn(self, spec):
def update_root(self, new_path): def update_root(self, new_path):
self.raw_root = new_path self.raw_root = new_path
self.root = spack.util.path.canonicalize_path(new_path, default_wd=self.base) self.root = spack.util.path.canonicalize_path(
new_path, default_wd=self.base,
replacements=spack.paths.path_replacements()
)
def __eq__(self, other): def __eq__(self, other):
return all( return all(
@ -985,7 +992,8 @@ def included_config_scopes(self):
missing = [] missing = []
for i, config_path in enumerate(reversed(includes)): for i, config_path in enumerate(reversed(includes)):
# allow paths to contain spack config/environment variables, etc. # allow paths to contain spack config/environment variables, etc.
config_path = substitute_path_variables(config_path) config_path = substitute_path_variables(config_path,
replacements=spack.paths.path_replacements())
include_url = urllib.parse.urlparse(config_path) include_url = urllib.parse.urlparse(config_path)
@ -1296,7 +1304,10 @@ def develop(self, spec: Spec, path: str, clone: bool = False) -> bool:
# to be created, then copy it afterwards somewhere else. It would be # to be created, then copy it afterwards somewhere else. It would be
# better if we can create the `source_path` directly into its final # better if we can create the `source_path` directly into its final
# destination. # destination.
abspath = spack.util.path.canonicalize_path(path, default_wd=self.path) abspath = spack.util.path.canonicalize_path(
path, default_wd=self.path,
replacements=spack.paths.path_replacements()
)
pkg_cls = spack.repo.PATH.get_pkg_class(spec.name) pkg_cls = spack.repo.PATH.get_pkg_class(spec.name)
# We construct a package class ourselves, rather than asking for # We construct a package class ourselves, rather than asking for
# Spec.package, since Spec only allows this when it is concrete # Spec.package, since Spec only allows this when it is concrete

View File

@ -16,6 +16,7 @@
import spack.config import spack.config
import spack.error import spack.error
import spack.paths
import spack.util.path import spack.util.path
_extension_regexp = re.compile(r"spack-(\w[-\w]*)$") _extension_regexp = re.compile(r"spack-(\w[-\w]*)$")
@ -109,7 +110,8 @@ def ensure_package_creation(name):
def get_extension_paths(): def get_extension_paths():
"""Return the list of canonicalized extension paths from config:extensions.""" """Return the list of canonicalized extension paths from config:extensions."""
extension_paths = spack.config.get("config:extensions") or [] extension_paths = spack.config.get("config:extensions") or []
paths = [spack.util.path.canonicalize_path(p) for p in extension_paths] r = spack.paths.path_replacements()
paths = [spack.util.path.canonicalize_path(p, replacements=r) for p in extension_paths]
return paths return paths

View File

@ -91,7 +91,8 @@ def get_test_stage_dir():
the default test stage path the default test stage path
""" """
return spack.util.path.canonicalize_path( return spack.util.path.canonicalize_path(
spack.config.get("config:test_stage", spack.paths.default_test_path) spack.config.get("config:test_stage", spack.paths.default_test_path),
replacements=spack.paths.path_replacements()
) )

View File

@ -30,6 +30,7 @@
import spack.fetch_strategy import spack.fetch_strategy
import spack.mirror import spack.mirror
import spack.oci.image import spack.oci.image
import spack.paths
import spack.spec import spack.spec
import spack.util.path import spack.util.path
import spack.util.spack_json as sjson import spack.util.spack_json as sjson
@ -51,7 +52,10 @@ def _url_or_path_to_url(url_or_path: str) -> str:
return url_or_path return url_or_path
# Otherwise we interpret it as path, and we should promote it to file:// URL. # Otherwise we interpret it as path, and we should promote it to file:// URL.
return url_util.path_to_file_url(spack.util.path.canonicalize_path(url_or_path)) return url_util.path_to_file_url(
spack.util.path.canonicalize_path(url_or_path,
replacements=spack.paths.path_replacements())
)
class Mirror: class Mirror:

View File

@ -225,7 +225,8 @@ def root_path(name, module_set_name):
roots = spack.config.merge_yaml(defaults, roots) roots = spack.config.merge_yaml(defaults, roots)
path = roots.get(name, os.path.join(spack.paths.share_path, name)) path = roots.get(name, os.path.join(spack.paths.share_path, name))
return spack.util.path.canonicalize_path(path) return spack.util.path.canonicalize_path(path,
replacements=spack.paths.path_replacements())
def generate_module_index(root, modules, overwrite=False): def generate_module_index(root, modules, overwrite=False):

View File

@ -829,7 +829,8 @@ def name(cls):
@classproperty @classproperty
def global_license_dir(cls): def global_license_dir(cls):
"""Returns the directory where license files for all packages are stored.""" """Returns the directory where license files for all packages are stored."""
return spack.util.path.canonicalize_path(spack.config.get("config:license_dir")) return spack.util.path.canonicalize_path(spack.config.get("config:license_dir"),
replacements=spack.paths.path_replacements())
@property @property
def global_license_file(self): def global_license_file(self):

View File

@ -7,6 +7,7 @@
import spack.error import spack.error
import spack.repo import spack.repo
import spack.paths
from spack.config import ConfigError from spack.config import ConfigError
from spack.util.path import canonicalize_path from spack.util.path import canonicalize_path
from spack.version import Version from spack.version import Version
@ -177,7 +178,8 @@ def _package(maybe_abstract_spec):
spec_str = entry["spec"] spec_str = entry["spec"]
external_path = entry.get("prefix", None) external_path = entry.get("prefix", None)
if external_path: if external_path:
external_path = canonicalize_path(external_path) external_path = canonicalize_path(external_path,
replacements=spack.paths.path_replacements())
external_modules = entry.get("modules", None) external_modules = entry.get("modules", None)
external_spec = spack.spec.Spec.from_detection( external_spec = spack.spec.Spec.from_detection(
spack.spec.Spec( spack.spec.Spec(

View File

@ -10,10 +10,14 @@
dependencies. dependencies.
""" """
import os import os
import tempfile
from datetime import date
from pathlib import PurePath from pathlib import PurePath
import llnl.util.filesystem import llnl.util.filesystem
import spack.util.path
#: This file lives in $prefix/lib/spack/spack/__file__ #: This file lives in $prefix/lib/spack/spack/__file__
prefix = str(PurePath(llnl.util.filesystem.ancestor(__file__, 4))) prefix = str(PurePath(llnl.util.filesystem.ancestor(__file__, 4)))
@ -136,3 +140,52 @@ def _get_system_config_path():
#: System configuration location #: System configuration location
system_config_path = _get_system_config_path() system_config_path = _get_system_config_path()
def architecture():
# break circular import
import spack.platforms
import spack.spec
host_platform = spack.platforms.host()
host_os = host_platform.operating_system("default_os")
host_target = host_platform.target("default_target")
return spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))
def get_user():
# User pwd where available because it accounts for effective uids when using ksu and similar
try:
# user pwd for unix systems
import pwd
return pwd.getpwuid(os.geteuid()).pw_name
except ImportError:
# fallback on getpass
return getpass.getuser()
def path_replacements():
# break circular imports
import spack.environment as ev
import spack.paths
arch = architecture()
return {
"spack": lambda: spack.paths.prefix,
"user": lambda: get_user(),
"tempdir": lambda: tempfile.gettempdir(),
"user_cache_path": lambda: spack.paths.user_cache_path,
"architecture": lambda: arch,
"arch": lambda: arch,
"platform": lambda: arch.platform,
"operating_system": lambda: arch.os,
"os": lambda: arch.os,
"target": lambda: arch.target,
"target_family": lambda: arch.target.microarchitecture.family,
"date": lambda: date.today().strftime("%Y-%m-%d"),
"env": lambda: ev.active_environment().path if ev.active_environment() else \
spack.util.path.NOMATCH,
}

View File

@ -37,6 +37,7 @@
import spack.config import spack.config
import spack.error import spack.error
import spack.patch import spack.patch
import spack.paths
import spack.provider_index import spack.provider_index
import spack.spec import spack.spec
import spack.tag import spack.tag
@ -928,7 +929,10 @@ def __init__(self, root, cache=None):
""" """
# Root directory, containing _repo.yaml and package dirs # Root directory, containing _repo.yaml and package dirs
# Allow roots to by spack-relative by starting with '$spack' # Allow roots to by spack-relative by starting with '$spack'
self.root = spack.util.path.canonicalize_path(root) self.root = spack.util.path.canonicalize_path(
root,
replacements=spack.paths.path_replacements()
)
# check and raise BadRepoError on fail. # check and raise BadRepoError on fail.
def check(condition, msg): def check(condition, msg):
@ -1327,7 +1331,8 @@ def create_repo(root, namespace=None, subdir=packages_dir_name):
If the namespace is not provided, use basename of root. If the namespace is not provided, use basename of root.
Return the canonicalized path and namespace of the created repository. Return the canonicalized path and namespace of the created repository.
""" """
root = spack.util.path.canonicalize_path(root) root = spack.util.path.canonicalize_path(root,
replacements=spack.paths.path_replacements())
if not namespace: if not namespace:
namespace = os.path.basename(root) namespace = os.path.basename(root)

View File

@ -41,6 +41,7 @@
import spack.error import spack.error
import spack.package_base import spack.package_base
import spack.package_prefs import spack.package_prefs
import spack.paths
import spack.platforms import spack.platforms
import spack.repo import spack.repo
import spack.spec import spack.spec
@ -2602,7 +2603,10 @@ def setup(
dev_specs = tuple( dev_specs = tuple(
spack.spec.Spec(info["spec"]).constrained( spack.spec.Spec(info["spec"]).constrained(
"dev_path=%s" "dev_path=%s"
% spack.util.path.canonicalize_path(info["path"], default_wd=env.path) % spack.util.path.canonicalize_path(
info["path"], default_wd=env.path,
replacements=spack.paths.path_replacements()
)
) )
for name, info in env.dev_specs.items() for name, info in env.dev_specs.items()
) )
@ -3119,7 +3123,8 @@ def _develop_specs_from_env(spec, env):
if not dev_info: if not dev_info:
return return
path = spack.util.path.canonicalize_path(dev_info["path"], default_wd=env.path) path = spack.util.path.canonicalize_path(dev_info["path"], default_wd=env.path,
replacements=spack.paths.path_replacements())
if "dev_path" in spec.variants: if "dev_path" in spec.variants:
error_msg = ( error_msg = (

View File

@ -150,7 +150,8 @@ def _resolve_paths(candidates):
Adjustments involve removing extra $user from $tempdir if $tempdir includes Adjustments involve removing extra $user from $tempdir if $tempdir includes
$user and appending $user if it is not present in the path. $user and appending $user if it is not present in the path.
""" """
temp_path = sup.canonicalize_path("$tempdir") temp_path = sup.canonicalize_path("$tempdir",
replacements=spack.paths.path_replacements())
user = getpass.getuser() user = getpass.getuser()
tmp_has_usr = user in temp_path.split(os.path.sep) tmp_has_usr = user in temp_path.split(os.path.sep)
@ -162,7 +163,8 @@ def _resolve_paths(candidates):
path = path.replace("/$user", "", 1) path = path.replace("/$user", "", 1)
# Ensure the path is unique per user. # Ensure the path is unique per user.
can_path = sup.canonicalize_path(path) can_path = sup.canonicalize_path(path,
replacements=spack.paths.path_replacements())
# When multiple users share a stage root, we can avoid conflicts between # When multiple users share a stage root, we can avoid conflicts between
# them by adding a per-user subdirectory. # them by adding a per-user subdirectory.
# Avoid doing this on Windows to keep stage absolute path as short as possible. # Avoid doing this on Windows to keep stage absolute path as short as possible.
@ -199,9 +201,11 @@ def get_stage_root():
def _mirror_roots(): def _mirror_roots():
mirrors = spack.config.get("mirrors") mirrors = spack.config.get("mirrors")
return [ return [
sup.substitute_path_variables(root) sup.substitute_path_variables(root,
replacements=spack.paths.path_replacements())
if root.endswith(os.sep) if root.endswith(os.sep)
else sup.substitute_path_variables(root) + os.sep else sup.substitute_path_variables(root,
replacemnts=spack.paths.path_replacements()) + os.sep
for root in mirrors.values() for root in mirrors.values()
] ]

View File

@ -77,7 +77,10 @@ def parse_install_tree(config_dict):
if isinstance(install_tree, str): if isinstance(install_tree, str):
tty.warn("Using deprecated format for configuring install_tree") tty.warn("Using deprecated format for configuring install_tree")
unpadded_root = install_tree unpadded_root = install_tree
unpadded_root = spack.util.path.canonicalize_path(unpadded_root) unpadded_root = spack.util.path.canonicalize_path(
unpadded_root,
replacements=spack.paths.path_replacements()
)
# construct projection from previous values for backwards compatibility # construct projection from previous values for backwards compatibility
all_projection = config_dict.get( all_projection = config_dict.get(
"install_path_scheme", spack.directory_layout.default_projections["all"] "install_path_scheme", spack.directory_layout.default_projections["all"]
@ -86,7 +89,10 @@ def parse_install_tree(config_dict):
projections = {"all": all_projection} projections = {"all": all_projection}
else: else:
unpadded_root = install_tree.get("root", DEFAULT_INSTALL_TREE_ROOT) unpadded_root = install_tree.get("root", DEFAULT_INSTALL_TREE_ROOT)
unpadded_root = spack.util.path.canonicalize_path(unpadded_root) unpadded_root = spack.util.path.canonicalize_path(
unpadded_root,
replacements=spack.paths.path_replacements()
)
padded_length = install_tree.get("padded_length", False) padded_length = install_tree.get("padded_length", False)
if padded_length is True: if padded_length is True:
@ -267,7 +273,8 @@ def _construct_upstream_dbs_from_install_roots(
for install_root in reversed(install_roots): for install_root in reversed(install_roots):
upstream_dbs = list(accumulated_upstream_dbs) upstream_dbs = list(accumulated_upstream_dbs)
next_db = spack.database.Database( next_db = spack.database.Database(
spack.util.path.canonicalize_path(install_root), spack.util.path.canonicalize_path(install_root,
replacements=spack.paths.path_replacements()),
is_upstream=True, is_upstream=True,
upstream_dbs=upstream_dbs, upstream_dbs=upstream_dbs,
) )

View File

@ -10,6 +10,7 @@
import spack.config import spack.config
import spack.extensions import spack.extensions
import spack.paths
from spack.util.path import canonicalize_path from spack.util.path import canonicalize_path
@ -76,7 +77,9 @@ def make_environment(dirs: Optional[Tuple[str, ...]] = None):
# Default directories where to search for templates # Default directories where to search for templates
builtins = spack.config.get("config:template_dirs", ["$spack/share/spack/templates"]) builtins = spack.config.get("config:template_dirs", ["$spack/share/spack/templates"])
extensions = spack.extensions.get_template_dirs() extensions = spack.extensions.get_template_dirs()
dirs = tuple(canonicalize_path(d) for d in itertools.chain(builtins, extensions)) r = spack.paths.path_replacements()
dirs = tuple(canonicalize_path(d, replacements=r)
for d in itertools.chain(builtins, extensions))
# Loader for the templates # Loader for the templates
loader = jinja2.FileSystemLoader(dirs) loader = jinja2.FileSystemLoader(dirs)

View File

@ -10,6 +10,7 @@
import spack.bootstrap.core import spack.bootstrap.core
import spack.compilers import spack.compilers
import spack.environment import spack.environment
import spack.paths
import spack.store import spack.store
import spack.util.path import spack.util.path
@ -81,7 +82,10 @@ def test_store_path_customization(config_value, expected, mutable_config):
# Check the store path # Check the store path
current = spack.bootstrap.config.store_path() current = spack.bootstrap.config.store_path()
assert current == spack.util.path.canonicalize_path(expected) assert current == spack.util.path.canonicalize_path(
expected,
replacements=spack.paths.path_replacements()
)
def test_raising_exception_if_bootstrap_disabled(mutable_config): def test_raising_exception_if_bootstrap_disabled(mutable_config):

View File

@ -10,6 +10,7 @@
import llnl.util.filesystem as fs import llnl.util.filesystem as fs
import spack.environment as ev import spack.environment as ev
import spack.paths
import spack.spec import spack.spec
from spack.main import SpackCommand from spack.main import SpackCommand
@ -106,7 +107,10 @@ def test_develop_canonicalize_path(self, monkeypatch, config):
env("create", "test") env("create", "test")
with ev.read("test") as e: with ev.read("test") as e:
path = "../$user" path = "../$user"
abspath = spack.util.path.canonicalize_path(path, e.path) abspath = spack.util.path.canonicalize_path(
path, e.path,
replacements=spack.paths.path_replacements()
)
def check_path(stage, dest): def check_path(stage, dest):
assert dest == abspath assert dest == abspath
@ -123,7 +127,10 @@ def test_develop_canonicalize_path_no_args(self, monkeypatch, config):
env("create", "test") env("create", "test")
with ev.read("test") as e: with ev.read("test") as e:
path = "$user" path = "$user"
abspath = spack.util.path.canonicalize_path(path, e.path) abspath = spack.util.path.canonicalize_path(
path, e.path,
replacements=spack.paths.path_replacements()
)
def check_path(stage, dest): def check_path(stage, dest):
assert dest == abspath assert dest == abspath

View File

@ -338,48 +338,59 @@ def __init__(self, path):
def test_substitute_config_variables(mock_low_high_config, monkeypatch): def test_substitute_config_variables(mock_low_high_config, monkeypatch):
prefix = spack.paths.prefix.lstrip("/") prefix = spack.paths.prefix.lstrip("/")
r = spack.paths.path_replacements()
assert cross_plat_join( assert cross_plat_join(
os.sep + os.path.join("foo", "bar", "baz"), prefix os.sep + os.path.join("foo", "bar", "baz"), prefix
) == spack_path.canonicalize_path("/foo/bar/baz/$spack") ) == spack_path.canonicalize_path("/foo/bar/baz/$spack",
replacements=r)
assert cross_plat_join( assert cross_plat_join(
spack.paths.prefix, os.path.join("foo", "bar", "baz") spack.paths.prefix, os.path.join("foo", "bar", "baz")
) == spack_path.canonicalize_path("$spack/foo/bar/baz/") ) == spack_path.canonicalize_path("$spack/foo/bar/baz/",
replacements=r)
assert cross_plat_join( assert cross_plat_join(
os.sep + os.path.join("foo", "bar", "baz"), prefix, os.path.join("foo", "bar", "baz") os.sep + os.path.join("foo", "bar", "baz"), prefix, os.path.join("foo", "bar", "baz")
) == spack_path.canonicalize_path("/foo/bar/baz/$spack/foo/bar/baz/") ) == spack_path.canonicalize_path("/foo/bar/baz/$spack/foo/bar/baz/",
replacements=r)
assert cross_plat_join( assert cross_plat_join(
os.sep + os.path.join("foo", "bar", "baz"), prefix os.sep + os.path.join("foo", "bar", "baz"), prefix
) == spack_path.canonicalize_path("/foo/bar/baz/${spack}") ) == spack_path.canonicalize_path("/foo/bar/baz/${spack}",
replacements=r)
assert cross_plat_join( assert cross_plat_join(
spack.paths.prefix, os.path.join("foo", "bar", "baz") spack.paths.prefix, os.path.join("foo", "bar", "baz")
) == spack_path.canonicalize_path("${spack}/foo/bar/baz/") ) == spack_path.canonicalize_path("${spack}/foo/bar/baz/",
replacements=r)
assert cross_plat_join( assert cross_plat_join(
os.sep + os.path.join("foo", "bar", "baz"), prefix, os.path.join("foo", "bar", "baz") os.sep + os.path.join("foo", "bar", "baz"), prefix, os.path.join("foo", "bar", "baz")
) == spack_path.canonicalize_path("/foo/bar/baz/${spack}/foo/bar/baz/") ) == spack_path.canonicalize_path("/foo/bar/baz/${spack}/foo/bar/baz/",
replacements=r)
assert cross_plat_join( assert cross_plat_join(
os.sep + os.path.join("foo", "bar", "baz"), prefix, os.path.join("foo", "bar", "baz") os.sep + os.path.join("foo", "bar", "baz"), prefix, os.path.join("foo", "bar", "baz")
) != spack_path.canonicalize_path("/foo/bar/baz/${spack/foo/bar/baz/") ) != spack_path.canonicalize_path("/foo/bar/baz/${spack/foo/bar/baz/",
replacements=r)
# $env replacement is a no-op when no environment is active # $env replacement is a no-op when no environment is active
assert spack_path.canonicalize_path( assert spack_path.canonicalize_path(
os.sep + os.path.join("foo", "bar", "baz", "$env") os.sep + os.path.join("foo", "bar", "baz", "$env"),
replacements=r
) == os.sep + os.path.join("foo", "bar", "baz", "$env") ) == os.sep + os.path.join("foo", "bar", "baz", "$env")
# Fake an active environment and $env is replaced properly # Fake an active environment and $env is replaced properly
fake_env_path = os.sep + os.path.join("quux", "quuux") fake_env_path = os.sep + os.path.join("quux", "quuux")
monkeypatch.setattr(ev, "active_environment", lambda: MockEnv(fake_env_path)) monkeypatch.setattr(ev, "active_environment", lambda: MockEnv(fake_env_path))
assert spack_path.canonicalize_path("$env/foo/bar/baz") == os.path.join( assert spack_path.canonicalize_path("$env/foo/bar/baz",
replacements=r) == os.path.join(
fake_env_path, os.path.join("foo", "bar", "baz") fake_env_path, os.path.join("foo", "bar", "baz")
) )
# relative paths without source information are relative to cwd # relative paths without source information are relative to cwd
assert spack_path.canonicalize_path(os.path.join("foo", "bar", "baz")) == os.path.abspath( assert spack_path.canonicalize_path(os.path.join("foo", "bar", "baz"),
replacements=r) == os.path.abspath(
os.path.join("foo", "bar", "baz") os.path.join("foo", "bar", "baz")
) )
@ -389,19 +400,22 @@ def test_substitute_config_variables(mock_low_high_config, monkeypatch):
) )
spack.config.CONFIG.clear_caches() spack.config.CONFIG.clear_caches()
path = spack.config.get("modules:default:roots:lmod") path = spack.config.get("modules:default:roots:lmod")
assert spack_path.canonicalize_path(path) == os.path.normpath( assert spack_path.canonicalize_path(path,
replacements=r) == os.path.normpath(
os.path.join(mock_low_high_config.scopes["low"].path, os.path.join("foo", "bar", "baz")) os.path.join(mock_low_high_config.scopes["low"].path, os.path.join("foo", "bar", "baz"))
) )
# test architecture information is in replacements # test architecture information is in replacements
assert spack_path.canonicalize_path( assert spack_path.canonicalize_path(
os.path.join("foo", "$platform", "bar") os.path.join("foo", "$platform", "bar"),
replacements=r
) == os.path.abspath(os.path.join("foo", "test", "bar")) ) == os.path.abspath(os.path.join("foo", "test", "bar"))
host_target = spack.platforms.host().target("default_target") host_target = spack.platforms.host().target("default_target")
host_target_family = str(host_target.microarchitecture.family) host_target_family = str(host_target.microarchitecture.family)
assert spack_path.canonicalize_path( assert spack_path.canonicalize_path(
os.path.join("foo", "$target_family", "bar") os.path.join("foo", "$target_family", "bar"),
replacements=r
) == os.path.abspath(os.path.join("foo", host_target_family, "bar")) ) == os.path.abspath(os.path.join("foo", host_target_family, "bar"))
@ -438,28 +452,33 @@ def test_substitute_user(mock_low_high_config):
assert os.sep + os.path.join( assert os.sep + os.path.join(
"foo", "bar" "foo", "bar"
) + os.sep + user + os.sep + "baz" == spack_path.canonicalize_path( ) + os.sep + user + os.sep + "baz" == spack_path.canonicalize_path(
os.sep + os.path.join("foo", "bar", "$user", "baz") os.sep + os.path.join("foo", "bar", "$user", "baz"),
replacements=spack.paths.path_replacements()
) )
def test_substitute_user_cache(mock_low_high_config): def test_substitute_user_cache(mock_low_high_config):
user_cache_path = spack.paths.user_cache_path user_cache_path = spack.paths.user_cache_path
assert user_cache_path + os.sep + "baz" == spack_path.canonicalize_path( assert user_cache_path + os.sep + "baz" == spack_path.canonicalize_path(
os.path.join("$user_cache_path", "baz") os.path.join("$user_cache_path", "baz"),
replacements=spack.paths.path_replacements()
) )
def test_substitute_tempdir(mock_low_high_config): def test_substitute_tempdir(mock_low_high_config):
tempdir = tempfile.gettempdir() tempdir = tempfile.gettempdir()
assert tempdir == spack_path.canonicalize_path("$tempdir") assert tempdir == spack_path.canonicalize_path("$tempdir",
replacements=spack.paths.path_replacements())
assert tempdir + os.sep + os.path.join("foo", "bar", "baz") == spack_path.canonicalize_path( assert tempdir + os.sep + os.path.join("foo", "bar", "baz") == spack_path.canonicalize_path(
os.path.join("$tempdir", "foo", "bar", "baz") os.path.join("$tempdir", "foo", "bar", "baz"),
replacements=spack.paths.path_replacements()
) )
def test_substitute_date(mock_low_high_config): def test_substitute_date(mock_low_high_config):
test_path = os.path.join("hello", "world", "on", "$date") test_path = os.path.join("hello", "world", "on", "$date")
new_path = spack_path.canonicalize_path(test_path) new_path = spack_path.canonicalize_path(test_path,
replacements=spack.paths.path_replacements())
assert "$date" in test_path assert "$date" in test_path
assert date.today().strftime("%Y-%m-%d") in new_path assert date.today().strftime("%Y-%m-%d") in new_path

View File

@ -734,7 +734,8 @@ def test_resolve_paths(self):
assert spack.stage._resolve_paths(paths) == paths assert spack.stage._resolve_paths(paths) == paths
tempdir = "$tempdir" tempdir = "$tempdir"
can_tempdir = canonicalize_path(tempdir) can_tempdir = canonicalize_path(tempdir,
replacements=spack.paths.path_replacements())
user = getpass.getuser() user = getpass.getuser()
temp_has_user = user in can_tempdir.split(os.sep) temp_has_user = user in can_tempdir.split(os.sep)
paths = [ paths = [
@ -744,7 +745,8 @@ def test_resolve_paths(self):
os.path.join(tempdir, "$user", "stage", "$user"), os.path.join(tempdir, "$user", "stage", "$user"),
] ]
res_paths = [canonicalize_path(p) for p in paths] r = spack.paths.path_replacements()
res_paths = [canonicalize_path(p, replacements=r) for p in paths]
if temp_has_user: if temp_has_user:
res_paths[1] = can_tempdir res_paths[1] = can_tempdir
res_paths[2] = os.path.join(can_tempdir, user) res_paths[2] = os.path.join(can_tempdir, user)

View File

@ -7,6 +7,7 @@
import pytest import pytest
import spack.config import spack.config
import spack.paths
import spack.tengine as tengine import spack.tengine as tengine
from spack.util.path import canonicalize_path from spack.util.path import canonicalize_path
@ -70,8 +71,9 @@ class TestTengineEnvironment:
def test_template_retrieval(self): def test_template_retrieval(self):
"""Tests the template retrieval mechanism hooked into config files""" """Tests the template retrieval mechanism hooked into config files"""
# Check the directories are correct # Check the directories are correct
r = spack.paths.path_replacements()
template_dirs = spack.config.get("config:template_dirs") template_dirs = spack.config.get("config:template_dirs")
template_dirs = tuple([canonicalize_path(x) for x in template_dirs]) template_dirs = tuple([canonicalize_path(x, replacements=r) for x in template_dirs])
assert len(template_dirs) == 3 assert len(template_dirs) == 3
env = tengine.make_environment(template_dirs) env = tengine.make_environment(template_dirs)

View File

@ -21,62 +21,12 @@
import spack.util.spack_yaml as syaml import spack.util.spack_yaml as syaml
__all__ = ["substitute_config_variables", "substitute_path_variables", "canonicalize_path"] __all__ = ["substitute_config_variables", "substitute_path_variables", "canonicalize_path", "NOMATCH"]
def architecture():
# break circular import
import spack.platforms
import spack.spec
host_platform = spack.platforms.host()
host_os = host_platform.operating_system("default_os")
host_target = host_platform.target("default_target")
return spack.spec.ArchSpec((str(host_platform), str(host_os), str(host_target)))
def get_user():
# User pwd where available because it accounts for effective uids when using ksu and similar
try:
# user pwd for unix systems
import pwd
return pwd.getpwuid(os.geteuid()).pw_name
except ImportError:
# fallback on getpass
return getpass.getuser()
# return value for replacements with no match # return value for replacements with no match
NOMATCH = object() NOMATCH = object()
# Substitutions to perform
def replacements():
# break circular imports
import spack.environment as ev
import spack.paths
arch = architecture()
return {
"spack": lambda: spack.paths.prefix,
"user": lambda: get_user(),
"tempdir": lambda: tempfile.gettempdir(),
"user_cache_path": lambda: spack.paths.user_cache_path,
"architecture": lambda: arch,
"arch": lambda: arch,
"platform": lambda: arch.platform,
"operating_system": lambda: arch.os,
"os": lambda: arch.os,
"target": lambda: arch.target,
"target_family": lambda: arch.target.microarchitecture.family,
"date": lambda: date.today().strftime("%Y-%m-%d"),
"env": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH,
}
# This is intended to be longer than the part of the install path # This is intended to be longer than the part of the install path
# spack generates from the root path we give it. Included in the # spack generates from the root path we give it. Included in the
# estimate: # estimate:
@ -144,7 +94,7 @@ def get_system_path_max():
return sys_max_path_length return sys_max_path_length
def substitute_config_variables(path): def substitute_config_variables(path, replacements={}):
"""Substitute placeholders into paths. """Substitute placeholders into paths.
Spack allows paths in configs to have some placeholders, as follows: Spack allows paths in configs to have some placeholders, as follows:
@ -168,22 +118,20 @@ def substitute_config_variables(path):
replaced if there is an active environment, and should only be used in replaced if there is an active environment, and should only be used in
environment yaml files. environment yaml files.
""" """
_replacements = replacements()
# Look up replacements # Look up replacements
def repl(match): def repl(match):
m = match.group(0) m = match.group(0)
key = m.strip("${}").lower() key = m.strip("${}").lower()
repl = _replacements.get(key, lambda: m)() repl = replacements.get(key, lambda: m)()
return m if repl is NOMATCH else str(repl) return m if repl is NOMATCH else str(repl)
# Replace $var or ${var}. # Replace $var or ${var}.
return re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path) return re.sub(r"(\$\w+\b|\$\{\w+\})", repl, path)
def substitute_path_variables(path): def substitute_path_variables(path, replacements={}):
"""Substitute config vars, expand environment vars, expand user home.""" """Substitute config vars, expand environment vars, expand user home."""
path = substitute_config_variables(path) path = substitute_config_variables(path, replacements=replacements)
path = os.path.expandvars(path) path = os.path.expandvars(path)
path = os.path.expanduser(path) path = os.path.expanduser(path)
return path return path
@ -225,7 +173,7 @@ def add_padding(path, length):
return os.path.join(path, padding) return os.path.join(path, padding)
def canonicalize_path(path, default_wd=None): def canonicalize_path(path, default_wd=None, replacements=None):
"""Same as substitute_path_variables, but also take absolute path. """Same as substitute_path_variables, but also take absolute path.
If the string is a yaml object with file annotations, make absolute paths If the string is a yaml object with file annotations, make absolute paths
@ -234,6 +182,7 @@ def canonicalize_path(path, default_wd=None):
Arguments: Arguments:
path (str): path being converted as needed path (str): path being converted as needed
replacements (dict): dictionary of replacements to use
Returns: Returns:
(str): An absolute path with path variable substitution (str): An absolute path with path variable substitution
@ -245,7 +194,16 @@ def canonicalize_path(path, default_wd=None):
filename = os.path.dirname(path._start_mark.name) filename = os.path.dirname(path._start_mark.name)
assert path._start_mark.name == path._end_mark.name assert path._start_mark.name == path._end_mark.name
path = substitute_path_variables(path) if replacements is None:
_replacements = {}
else:
_replacements = replacements
if not isinstance(_replacements, dict):
tty.die("Replacements returned by replacements func are of type"
f"{type(replacements)} and not of the expected type of dict.")
path = substitute_path_variables(path, replacements=_replacements)
if not os.path.isabs(path): if not os.path.isabs(path):
if filename: if filename:
path = os.path.join(filename, path) path = os.path.join(filename, path)