Merge pull request #2789 from paulhopkins/features/allow_package_installation_directory_to_be_configured

Allow installation directory layout to be configured

lib/spack/docs/config_yaml.rst

@@ -42,6 +42,43 @@ or with braces to distinguish the variable from surrounding characters:
 The location where Spack will install packages and their dependencies.
 Default is ``$spack/opt/spack``.
 
+---------------------------------------------------
+``install_hash_length`` and ``install_path_scheme``
+---------------------------------------------------
+
+The default Spack installation path can be very long and can create
+problems for scripts with hardcoded shebangs. There are two parameters
+to help with that. Firstly, the ``install_hash_length`` parameter can
+set the length of the hash in the installation path from 1 to 32. The
+default path uses the full 32 characters.
+
+Secondly, it is
+also possible to modify the entire installation scheme. By default
+Spack uses
+``${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}``
+where the tokens that are available for use in this directive are the
+same as those understood by the ``Spec.format`` method. Using this parameter it
+is possible to use a different package layout or reduce the depth of
+the installation paths. For example
+
+     .. code-block:: yaml
+
+       config:
+         install_path_scheme: '${PACKAGE}/${VERSION}/${HASH:7}'
+
+would install packages into sub-directories using only the package
+name, version and a hash length of 7 characters.
+
+When using either parameter to set the hash length it only affects the
+representation of the hash in the installation directory. You
+should be aware that the smaller the hash length the more likely
+naming conflicts will occur. These parameters are independent of those
+used to configure module names.
+
+.. warning:: Modifying the installation hash length or path scheme after
+   packages have been installed will prevent Spack from being
+   able to find the old installation directories.
+
 --------------------
 ``module_roots``
 --------------------

lib/spack/spack/directory_layout.py

@@ -27,6 +27,7 @@
 import glob
 import tempfile
 import yaml
+import re
 
 from llnl.util.filesystem import join_path, mkdirp
 
@@ -149,24 +150,31 @@ def remove_install_directory(self, spec):
 
 
 class YamlDirectoryLayout(DirectoryLayout):
-    """Lays out installation directories like this::
+    """By default lays out installation directories like this::
            <install root>/
                <platform-os-target>/
                    <compiler>-<compiler version>/
-                       <name>-<version>-<variants>-<hash>
+                       <name>-<version>-<hash>
 
        The hash here is a SHA-1 hash for the full DAG plus the build
        spec.  TODO: implement the build spec.
 
-       To avoid special characters (like ~) in the directory name,
+       The installation directory scheme can be modified with the
-       only enabled variants are included in the install path.
+       arguments hash_len and path_scheme.
-       Disabled variants are omitted.
     """
 
     def __init__(self, root, **kwargs):
         super(YamlDirectoryLayout, self).__init__(root)
         self.metadata_dir   = kwargs.get('metadata_dir', '.spack')
-        self.hash_len       = kwargs.get('hash_len', None)
+        self.hash_len       = kwargs.get('hash_len')
+        self.path_scheme    = kwargs.get('path_scheme') or (
+            "${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}")  # NOQA: E501
+        if self.hash_len is not None:
+            if re.search('\${HASH:\d+}', self.path_scheme):
+                raise InvalidDirectoryLayoutParametersError(
+                    "Conflicting options for installation layout hash length")
+            self.path_scheme = self.path_scheme.replace(
+                "${HASH}", "${HASH:%d}" % self.hash_len)
 
         self.spec_file_name      = 'spec.yaml'
         self.extension_file_name = 'extensions.yaml'
@@ -187,16 +195,7 @@ def relative_path_for_spec(self, spec):
         if spec.external:
             return spec.external
 
-        dir_name = "%s-%s-%s" % (
+        path = spec.format(self.path_scheme)
-            spec.name,
-            spec.version,
-            spec.dag_hash(self.hash_len))
-
-        path = join_path(
-            spec.architecture,
-            "%s-%s" % (spec.compiler.name, spec.compiler.version),
-            dir_name)
-
         return path
 
     def write_spec(self, spec, path):
@@ -284,8 +283,9 @@ def all_specs(self):
         if not os.path.isdir(self.root):
             return []
 
-        pattern = join_path(
+        path_elems = ["*"] * len(self.path_scheme.split(os.sep))
-            self.root, '*', '*', '*', self.metadata_dir, self.spec_file_name)
+        path_elems += [self.metadata_dir, self.spec_file_name]
+        pattern = join_path(self.root, *path_elems)
         spec_files = glob.glob(pattern)
         return [self.read_spec(s) for s in spec_files]
 
@@ -446,6 +446,14 @@ class SpecReadError(DirectoryLayoutError):
     """Raised when directory layout can't read a spec."""
 
 
+class InvalidDirectoryLayoutParametersError(DirectoryLayoutError):
+    """Raised when a invalid directory layout parameters are supplied"""
+
+    def __init__(self, message, long_msg=None):
+        super(InvalidDirectoryLayoutParametersError, self).__init__(
+            message, long_msg)
+
+
 class InvalidExtensionSpecError(DirectoryLayoutError):
     """Raised when an extension file has a bad spec in it."""
 

lib/spack/spack/schema/config.py

@@ -41,6 +41,8 @@
             'additionalProperties': False,
             'properties': {
                 'install_tree': {'type': 'string'},
+                'install_hash_length': {'type': 'integer', 'minimum': 1},
+                'install_path_scheme': {'type': 'string'},
                 'build_stage': {
                     'oneOf': [
                         {'type': 'string'},

lib/spack/spack/spec.py

@@ -2711,7 +2711,7 @@ def write(s, c):
                         write(fmt % str(self.variants), '+')
                 elif named_str == 'ARCHITECTURE':
                     if self.architecture and str(self.architecture):
-                        write(fmt % str(self.architecture) + ' ', ' arch=')
+                        write(fmt % str(self.architecture), ' arch=')
                 elif named_str == 'SHA1':
                     if self.dependencies:
                         out.write(fmt % str(self.dag_hash(7)))
@@ -2727,7 +2727,7 @@ def write(s, c):
                         hashlen = int(hashlen)
                     else:
                         hashlen = None
-                    out.write('/' + fmt % (self.dag_hash(hashlen)))
+                    out.write(fmt % (self.dag_hash(hashlen)))
 
                 named = False
 

lib/spack/spack/store.py

@@ -72,4 +72,6 @@
 # This controls how spack lays out install prefixes and
 # stage directories.
 #
-layout = YamlDirectoryLayout(root)
+layout = YamlDirectoryLayout(root,
+                             hash_len=config.get('install_hash_length'),
+                             path_scheme=config.get('install_path_scheme'))

lib/spack/spack/test/directory_layout.py

@@ -29,7 +29,8 @@
 
 import pytest
 import spack
-from spack.directory_layout import YamlDirectoryLayout
+from spack.directory_layout import (YamlDirectoryLayout,
+                                    InvalidDirectoryLayoutParametersError)
 from spack.repository import RepoPath
 from spack.spec import Spec
 
@@ -43,6 +44,46 @@ def layout_and_dir(tmpdir):
     yield YamlDirectoryLayout(str(tmpdir)), str(tmpdir)
 
 
+def test_yaml_directory_layout_parameters(
+        tmpdir, config
+):
+    """This tests the various parameters that can be used to configure
+    the install location """
+    spec = Spec('python')
+    spec.concretize()
+
+    # Ensure default layout matches expected spec format
+    layout_default = YamlDirectoryLayout(str(tmpdir))
+    path_default = layout_default.relative_path_for_spec(spec)
+    assert(path_default ==
+           spec.format("${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}"))   # NOQA: ignore=E501
+
+    # Test hash_length parameter works correctly
+    layout_10 = YamlDirectoryLayout(str(tmpdir), hash_len=10)
+    path_10 = layout_10.relative_path_for_spec(spec)
+    layout_7 = YamlDirectoryLayout(str(tmpdir), hash_len=7)
+    path_7 = layout_7.relative_path_for_spec(spec)
+
+    assert(len(path_default) - len(path_10) == 22)
+    assert(len(path_default) - len(path_7) == 25)
+
+    # Test path_scheme
+    arch, compiler, package7 = path_7.split('/')
+    scheme_package7 = "${PACKAGE}-${VERSION}-${HASH:7}"
+
+    layout_package7 = YamlDirectoryLayout(str(tmpdir),
+                                          path_scheme=scheme_package7)
+    path_package7 = layout_package7.relative_path_for_spec(spec)
+
+    assert(package7 == path_package7)
+
+    # Ensure conflicting parameters caught
+    with pytest.raises(InvalidDirectoryLayoutParametersError):
+        YamlDirectoryLayout(str(tmpdir),
+                            hash_len=20,
+                            path_scheme=scheme_package7)
+
+
 def test_read_and_write_spec(
         layout_and_dir, config, builtin_mock
 ):