Merge pull request #2789 from paulhopkins/features/allow_package_installation_directory_to_be_configured
Allow installation directory layout to be configured
This commit is contained in:
scheibelp
GitHub
commit
9e50d16f13
@ -42,6 +42,43 @@ or with braces to distinguish the variable from surrounding characters:
|
||||
The location where Spack will install packages and their dependencies.
|
||||
Default is ``$spack/opt/spack``.
|
||||
|
||||
---------------------------------------------------
|
||||
``install_hash_length`` and ``install_path_scheme``
|
||||
---------------------------------------------------
|
||||
|
||||
The default Spack installation path can be very long and can create
|
||||
problems for scripts with hardcoded shebangs. There are two parameters
|
||||
to help with that. Firstly, the ``install_hash_length`` parameter can
|
||||
set the length of the hash in the installation path from 1 to 32. The
|
||||
default path uses the full 32 characters.
|
||||
|
||||
Secondly, it is
|
||||
also possible to modify the entire installation scheme. By default
|
||||
Spack uses
|
||||
``${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}``
|
||||
where the tokens that are available for use in this directive are the
|
||||
same as those understood by the ``Spec.format`` method. Using this parameter it
|
||||
is possible to use a different package layout or reduce the depth of
|
||||
the installation paths. For example
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
config:
|
||||
install_path_scheme: '${PACKAGE}/${VERSION}/${HASH:7}'
|
||||
|
||||
would install packages into sub-directories using only the package
|
||||
name, version and a hash length of 7 characters.
|
||||
|
||||
When using either parameter to set the hash length it only affects the
|
||||
representation of the hash in the installation directory. You
|
||||
should be aware that the smaller the hash length the more likely
|
||||
naming conflicts will occur. These parameters are independent of those
|
||||
used to configure module names.
|
||||
|
||||
.. warning:: Modifying the installation hash length or path scheme after
|
||||
packages have been installed will prevent Spack from being
|
||||
able to find the old installation directories.
|
||||
|
||||
--------------------
|
||||
``module_roots``
|
||||
--------------------
|
||||
|
||||
@ -27,6 +27,7 @@
|
||||
import glob
|
||||
import tempfile
|
||||
import yaml
|
||||
import re
|
||||
|
||||
from llnl.util.filesystem import join_path, mkdirp
|
||||
|
||||
@ -149,24 +150,31 @@ def remove_install_directory(self, spec):
|
||||
|
||||
|
||||
class YamlDirectoryLayout(DirectoryLayout):
|
||||
"""Lays out installation directories like this::
|
||||
"""By default lays out installation directories like this::
|
||||
<install root>/
|
||||
<platform-os-target>/
|
||||
<compiler>-<compiler version>/
|
||||
<name>-<version>-<variants>-<hash>
|
||||
<name>-<version>-<hash>
|
||||
|
||||
The hash here is a SHA-1 hash for the full DAG plus the build
|
||||
spec. TODO: implement the build spec.
|
||||
|
||||
To avoid special characters (like ~) in the directory name,
|
||||
only enabled variants are included in the install path.
|
||||
Disabled variants are omitted.
|
||||
The installation directory scheme can be modified with the
|
||||
arguments hash_len and path_scheme.
|
||||
"""
|
||||
|
||||
def __init__(self, root, **kwargs):
|
||||
super(YamlDirectoryLayout, self).__init__(root)
|
||||
self.metadata_dir = kwargs.get('metadata_dir', '.spack')
|
||||
self.hash_len = kwargs.get('hash_len', None)
|
||||
self.hash_len = kwargs.get('hash_len')
|
||||
self.path_scheme = kwargs.get('path_scheme') or (
|
||||
"${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}") # NOQA: E501
|
||||
if self.hash_len is not None:
|
||||
if re.search('\${HASH:\d+}', self.path_scheme):
|
||||
raise InvalidDirectoryLayoutParametersError(
|
||||
"Conflicting options for installation layout hash length")
|
||||
self.path_scheme = self.path_scheme.replace(
|
||||
"${HASH}", "${HASH:%d}" % self.hash_len)
|
||||
|
||||
self.spec_file_name = 'spec.yaml'
|
||||
self.extension_file_name = 'extensions.yaml'
|
||||
@ -187,16 +195,7 @@ def relative_path_for_spec(self, spec):
|
||||
if spec.external:
|
||||
return spec.external
|
||||
|
||||
dir_name = "%s-%s-%s" % (
|
||||
spec.name,
|
||||
spec.version,
|
||||
spec.dag_hash(self.hash_len))
|
||||
|
||||
path = join_path(
|
||||
spec.architecture,
|
||||
"%s-%s" % (spec.compiler.name, spec.compiler.version),
|
||||
dir_name)
|
||||
|
||||
path = spec.format(self.path_scheme)
|
||||
return path
|
||||
|
||||
def write_spec(self, spec, path):
|
||||
@ -284,8 +283,9 @@ def all_specs(self):
|
||||
if not os.path.isdir(self.root):
|
||||
return []
|
||||
|
||||
pattern = join_path(
|
||||
self.root, '*', '*', '*', self.metadata_dir, self.spec_file_name)
|
||||
path_elems = ["*"] * len(self.path_scheme.split(os.sep))
|
||||
path_elems += [self.metadata_dir, self.spec_file_name]
|
||||
pattern = join_path(self.root, *path_elems)
|
||||
spec_files = glob.glob(pattern)
|
||||
return [self.read_spec(s) for s in spec_files]
|
||||
|
||||
@ -446,6 +446,14 @@ class SpecReadError(DirectoryLayoutError):
|
||||
"""Raised when directory layout can't read a spec."""
|
||||
|
||||
|
||||
class InvalidDirectoryLayoutParametersError(DirectoryLayoutError):
|
||||
"""Raised when a invalid directory layout parameters are supplied"""
|
||||
|
||||
def __init__(self, message, long_msg=None):
|
||||
super(InvalidDirectoryLayoutParametersError, self).__init__(
|
||||
message, long_msg)
|
||||
|
||||
|
||||
class InvalidExtensionSpecError(DirectoryLayoutError):
|
||||
"""Raised when an extension file has a bad spec in it."""
|
||||
|
||||
|
||||
@ -41,6 +41,8 @@
|
||||
'additionalProperties': False,
|
||||
'properties': {
|
||||
'install_tree': {'type': 'string'},
|
||||
'install_hash_length': {'type': 'integer', 'minimum': 1},
|
||||
'install_path_scheme': {'type': 'string'},
|
||||
'build_stage': {
|
||||
'oneOf': [
|
||||
{'type': 'string'},
|
||||
|
||||
@ -2711,7 +2711,7 @@ def write(s, c):
|
||||
write(fmt % str(self.variants), '+')
|
||||
elif named_str == 'ARCHITECTURE':
|
||||
if self.architecture and str(self.architecture):
|
||||
write(fmt % str(self.architecture) + ' ', ' arch=')
|
||||
write(fmt % str(self.architecture), ' arch=')
|
||||
elif named_str == 'SHA1':
|
||||
if self.dependencies:
|
||||
out.write(fmt % str(self.dag_hash(7)))
|
||||
@ -2727,7 +2727,7 @@ def write(s, c):
|
||||
hashlen = int(hashlen)
|
||||
else:
|
||||
hashlen = None
|
||||
out.write('/' + fmt % (self.dag_hash(hashlen)))
|
||||
out.write(fmt % (self.dag_hash(hashlen)))
|
||||
|
||||
named = False
|
||||
|
||||
|
||||
@ -72,4 +72,6 @@
|
||||
# This controls how spack lays out install prefixes and
|
||||
# stage directories.
|
||||
#
|
||||
layout = YamlDirectoryLayout(root)
|
||||
layout = YamlDirectoryLayout(root,
|
||||
hash_len=config.get('install_hash_length'),
|
||||
path_scheme=config.get('install_path_scheme'))
|
||||
|
||||
@ -29,7 +29,8 @@
|
||||
|
||||
import pytest
|
||||
import spack
|
||||
from spack.directory_layout import YamlDirectoryLayout
|
||||
from spack.directory_layout import (YamlDirectoryLayout,
|
||||
InvalidDirectoryLayoutParametersError)
|
||||
from spack.repository import RepoPath
|
||||
from spack.spec import Spec
|
||||
|
||||
@ -43,6 +44,46 @@ def layout_and_dir(tmpdir):
|
||||
yield YamlDirectoryLayout(str(tmpdir)), str(tmpdir)
|
||||
|
||||
|
||||
def test_yaml_directory_layout_parameters(
|
||||
tmpdir, config
|
||||
):
|
||||
"""This tests the various parameters that can be used to configure
|
||||
the install location """
|
||||
spec = Spec('python')
|
||||
spec.concretize()
|
||||
|
||||
# Ensure default layout matches expected spec format
|
||||
layout_default = YamlDirectoryLayout(str(tmpdir))
|
||||
path_default = layout_default.relative_path_for_spec(spec)
|
||||
assert(path_default ==
|
||||
spec.format("${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}")) # NOQA: ignore=E501
|
||||
|
||||
# Test hash_length parameter works correctly
|
||||
layout_10 = YamlDirectoryLayout(str(tmpdir), hash_len=10)
|
||||
path_10 = layout_10.relative_path_for_spec(spec)
|
||||
layout_7 = YamlDirectoryLayout(str(tmpdir), hash_len=7)
|
||||
path_7 = layout_7.relative_path_for_spec(spec)
|
||||
|
||||
assert(len(path_default) - len(path_10) == 22)
|
||||
assert(len(path_default) - len(path_7) == 25)
|
||||
|
||||
# Test path_scheme
|
||||
arch, compiler, package7 = path_7.split('/')
|
||||
scheme_package7 = "${PACKAGE}-${VERSION}-${HASH:7}"
|
||||
|
||||
layout_package7 = YamlDirectoryLayout(str(tmpdir),
|
||||
path_scheme=scheme_package7)
|
||||
path_package7 = layout_package7.relative_path_for_spec(spec)
|
||||
|
||||
assert(package7 == path_package7)
|
||||
|
||||
# Ensure conflicting parameters caught
|
||||
with pytest.raises(InvalidDirectoryLayoutParametersError):
|
||||
YamlDirectoryLayout(str(tmpdir),
|
||||
hash_len=20,
|
||||
path_scheme=scheme_package7)
|
||||
|
||||
|
||||
def test_read_and_write_spec(
|
||||
layout_and_dir, config, builtin_mock
|
||||
):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user