zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YamlDirectoryLayout now working.

Browse Source
This commit is contained in:
Todd Gamblin 2015-05-09 16:32:57 -05:00
parent 9412fc8083
commit 2f3b0481de
6 changed files with 113 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lib/spack/spack/__init__.py
View File

@ -42,7 +42,8 @@
hooks_path = join_path(module_path, "hooks") hooks_path = join_path(module_path, "hooks")
var_path = join_path(prefix, "var", "spack") var_path = join_path(prefix, "var", "spack")
stage_path = join_path(var_path, "stage") stage_path = join_path(var_path, "stage")
install_path = join_path(prefix, "opt") opt_path = join_path(prefix, "opt")
install_path = join_path(opt_path, "spack")
share_path = join_path(prefix, "share", "spack") share_path = join_path(prefix, "share", "spack")
# #
@ -65,8 +66,8 @@
# This controls how spack lays out install prefixes and # This controls how spack lays out install prefixes and
# stage directories. # stage directories.
# #
from spack.directory_layout import SpecHashDirectoryLayout from spack.directory_layout import YamlDirectoryLayout
install_layout = SpecHashDirectoryLayout(install_path) install_layout = YamlDirectoryLayout(install_path)
# #
# This controls how things are concretized in spack. # This controls how things are concretized in spack.

153
lib/spack/spack/directory_layout.py
View File

@ -27,8 +27,9 @@
import exceptions import exceptions
import hashlib import hashlib
import shutil import shutil
import glob
import tempfile import tempfile
from contextlib import closing from external import yaml
import llnl.util.tty as tty import llnl.util.tty as tty
from llnl.util.lang import memoized from llnl.util.lang import memoized
@ -81,7 +82,7 @@ def relative_path_for_spec(self, spec):
raise NotImplementedError() raise NotImplementedError()
def make_path_for_spec(self, spec): def create_install_directory(self, spec):
"""Creates the installation directory for a spec.""" """Creates the installation directory for a spec."""
raise NotImplementedError() raise NotImplementedError()
@ -131,7 +132,7 @@ def path_for_spec(self, spec):
return os.path.join(self.root, path) return os.path.join(self.root, path)
def remove_path_for_spec(self, spec): def remove_install_directory(self, spec):
"""Removes a prefix and any empty parent directories from the root. """Removes a prefix and any empty parent directories from the root.
Raised RemoveFailedError if something goes wrong. Raised RemoveFailedError if something goes wrong.
""" """
@ -153,94 +154,70 @@ def remove_path_for_spec(self, spec):
path = os.path.dirname(path) path = os.path.dirname(path)
def traverse_dirs_at_depth(root, depth, path_tuple=(), curdepth=0): class YamlDirectoryLayout(DirectoryLayout):
"""For each directory at <depth> within <root>, return a tuple representing
the ancestors of that directory.
"""
if curdepth == depth and curdepth != 0:
yield path_tuple
elif depth > curdepth:
for filename in os.listdir(root):
child = os.path.join(root, filename)
if os.path.isdir(child):
child_tuple = path_tuple + (filename,)
for tup in traverse_dirs_at_depth(
child, depth, child_tuple, curdepth+1):
yield tup
class SpecHashDirectoryLayout(DirectoryLayout):
"""Lays out installation directories like this:: """Lays out installation directories like this::
<install_root>/ <install root>/
<architecture>/ <architecture>/
<compiler>/ <compiler>-<compiler version>/
name@version+variant-<dependency_hash> <name>-<version>-<variants>-<hash>
Where dependency_hash is a SHA-1 hash prefix for the full package spec. The hash here is a SHA-1 hash for the full DAG plus the build
This accounts for dependencies. spec. TODO: implement the build spec.
If there is ever a hash collision, you won't be able to install a new To avoid special characters (like ~) in the directory name,
package unless you use a larger prefix. However, the full spec is stored only enabled variants are included in the install path.
in a file called .spec in each directory, so you can migrate an entire Disabled variants are omitted.
install directory to a new hash size pretty easily.
TODO: make a tool to migrate install directories to different hash sizes.
""" """
def __init__(self, root, **kwargs): def __init__(self, root, **kwargs):
"""Prefix size is number of characters in the SHA-1 prefix to use super(YamlDirectoryLayout, self).__init__(root)
to make each hash unique. self.metadata_dir = kwargs.get('metadata_dir', '.spack')
""" self.hash_len = kwargs.get('hash_len', None)
spec_file_name = kwargs.get('spec_file_name', '.spec')
extension_file_name = kwargs.get('extension_file_name', '.extensions') self.spec_file_name = 'spec'
super(SpecHashDirectoryLayout, self).__init__(root) self.extension_file_name = 'extensions'
self.spec_file_name = spec_file_name
self.extension_file_name = extension_file_name
# Cache of already written/read extension maps. # Cache of already written/read extension maps.
self._extension_maps = {} self._extension_maps = {}
@property @property
def hidden_file_paths(self): def hidden_file_paths(self):
return ('.spec', '.extensions') return (self.metadata_dir)
def relative_path_for_spec(self, spec): def relative_path_for_spec(self, spec):
_check_concrete(spec) _check_concrete(spec)
dir_name = spec.format('$_$@$+$#') enabled_variants = (
return join_path(spec.architecture, spec.compiler, dir_name) '-' + v.name for v in spec.variants.values()
if v.enabled)
dir_name = "%s-%s%s-%s" % (
spec.name,
spec.version,
''.join(enabled_variants),
spec.dag_hash(self.hash_len))
path = join_path(
spec.architecture,
"%s-%s" % (spec.compiler.name, spec.compiler.version),
dir_name)
return path
def write_spec(self, spec, path): def write_spec(self, spec, path):
"""Write a spec out to a file.""" """Write a spec out to a file."""
with closing(open(path, 'w')) as spec_file: _check_concrete(spec)
spec_file.write(spec.tree(ids=False, cover='nodes')) with open(path, 'w') as f:
f.write(spec.to_yaml())
def read_spec(self, path): def read_spec(self, path):
"""Read the contents of a file and parse them as a spec""" """Read the contents of a file and parse them as a spec"""
with closing(open(path)) as spec_file: with open(path) as f:
# Specs from files are assumed normal and concrete yaml_text = f.read()
spec = Spec(spec_file.read().replace('\n', '')) spec = Spec.from_yaml(yaml_text)
if all(spack.db.exists(s.name) for s in spec.traverse()): # Specs read from actual installations are always concrete
copy = spec.copy()
# TODO: It takes a lot of time to normalize every spec on read.
# TODO: Storing graph info with spec files would fix this.
copy.normalize()
if copy.concrete:
return copy # These are specs spack still understands.
# If we get here, either the spec is no longer in spack, or
# something about its dependencies has changed. So we need to
# just assume the read spec is correct. We'll lose graph
# information if we do this, but this is just for best effort
# for commands like uninstall and find. Currently Spack
# doesn't do anything that needs the graph info after install.
# TODO: store specs with full connectivity information, so
# that we don't have to normalize or reconstruct based on
# changing dependencies in the Spack tree.
spec._normal = True spec._normal = True
spec._concrete = True spec._concrete = True
return spec return spec
@ -249,10 +226,14 @@ def read_spec(self, path):
def spec_file_path(self, spec): def spec_file_path(self, spec):
"""Gets full path to spec file""" """Gets full path to spec file"""
_check_concrete(spec) _check_concrete(spec)
return join_path(self.path_for_spec(spec), self.spec_file_name) return join_path(self.metadata_path(spec), self.spec_file_name)
def make_path_for_spec(self, spec): def metadata_path(self, spec):
return join_path(self.path_for_spec(spec), self.metadata_dir)
def create_install_directory(self, spec):
_check_concrete(spec) _check_concrete(spec)
path = self.path_for_spec(spec) path = self.path_for_spec(spec)
@ -267,16 +248,13 @@ def make_path_for_spec(self, spec):
if installed_spec == self.spec: if installed_spec == self.spec:
raise InstallDirectoryAlreadyExistsError(path) raise InstallDirectoryAlreadyExistsError(path)
spec_hash = self.hash_spec(spec) if spec.dag_hash() == installed_spec.dag_hash():
installed_hash = self.hash_spec(installed_spec)
if installed_spec == spec_hash:
raise SpecHashCollisionError(installed_hash, spec_hash) raise SpecHashCollisionError(installed_hash, spec_hash)
else: else:
raise InconsistentInstallDirectoryError( raise InconsistentInstallDirectoryError(
'Spec file in %s does not match SHA-1 hash!' 'Spec file in %s does not match hash!' % spec_file_path)
% spec_file_path)
mkdirp(path) mkdirp(self.metadata_path(spec))
self.write_spec(spec, spec_file_path) self.write_spec(spec, spec_file_path)
@ -284,22 +262,14 @@ def make_path_for_spec(self, spec):
def all_specs(self): def all_specs(self):
if not os.path.isdir(self.root): if not os.path.isdir(self.root):
return [] return []
spec_files = glob.glob("%s/*/*/*/.spack/spec" % self.root)
specs = [] return [self.read_spec(s) for s in spec_files]
for path in traverse_dirs_at_depth(self.root, 3):
arch, compiler, last_dir = path
spec_file_path = join_path(
self.root, arch, compiler, last_dir, self.spec_file_name)
if os.path.exists(spec_file_path):
spec = self.read_spec(spec_file_path)
specs.append(spec)
return specs
def extension_file_path(self, spec): def extension_file_path(self, spec):
"""Gets full path to an installed package's extension file""" """Gets full path to an installed package's extension file"""
_check_concrete(spec) _check_concrete(spec)
return join_path(self.path_for_spec(spec), self.extension_file_name) return join_path(self.metadata_path(spec), self.extension_file_name)
def _extension_map(self, spec): def _extension_map(self, spec):
@ -314,7 +284,7 @@ def _extension_map(self, spec):
else: else:
exts = {} exts = {}
with closing(open(path)) as ext_file: with open(path) as ext_file:
for line in ext_file: for line in ext_file:
try: try:
spec = Spec(line.strip()) spec = Spec(line.strip())
@ -358,7 +328,7 @@ def _write_extensions(self, spec, extensions):
prefix=basename, dir=dirname, delete=False) prefix=basename, dir=dirname, delete=False)
# Write temp file. # Write temp file.
with closing(tmp): with tmp:
for extension in sorted(extensions.values()): for extension in sorted(extensions.values()):
tmp.write("%s\n" % extension) tmp.write("%s\n" % extension)
@ -392,6 +362,7 @@ def remove_extension(self, spec, ext_spec):
self._write_extensions(spec, exts) self._write_extensions(spec, exts)
class DirectoryLayoutError(SpackError): class DirectoryLayoutError(SpackError):
"""Superclass for directory layout errors.""" """Superclass for directory layout errors."""
def __init__(self, message): def __init__(self, message):
@ -399,9 +370,9 @@ def __init__(self, message):
class SpecHashCollisionError(DirectoryLayoutError): class SpecHashCollisionError(DirectoryLayoutError):
"""Raised when there is a hash collision in an SpecHashDirectoryLayout.""" """Raised when there is a hash collision in an install layout."""
def __init__(self, installed_spec, new_spec): def __init__(self, installed_spec, new_spec):
super(SpecHashDirectoryLayout, self).__init__( super(SpecHashCollisionError, self).__init__(
'Specs %s and %s have the same SHA-1 prefix!' 'Specs %s and %s have the same SHA-1 prefix!'
% installed_spec, new_spec) % installed_spec, new_spec)
@ -422,7 +393,7 @@ def __init__(self, message):
class InstallDirectoryAlreadyExistsError(DirectoryLayoutError): class InstallDirectoryAlreadyExistsError(DirectoryLayoutError):
"""Raised when make_path_for_sec is called unnecessarily.""" """Raised when create_install_directory is called unnecessarily."""
def __init__(self, path): def __init__(self, path):
super(InstallDirectoryAlreadyExistsError, self).__init__( super(InstallDirectoryAlreadyExistsError, self).__init__(
"Install path %s already exists!") "Install path %s already exists!")
@ -455,5 +426,3 @@ def __init__(self, spec, ext_spec):
super(NoSuchExtensionError, self).__init__( super(NoSuchExtensionError, self).__init__(
"%s cannot be removed from %s because it's not activated."% ( "%s cannot be removed from %s because it's not activated."% (
ext_spec.short_spec, spec.short_spec)) ext_spec.short_spec, spec.short_spec))

6
lib/spack/spack/package.py
View File

@ -658,7 +658,7 @@ def url_version(self, version):
def remove_prefix(self): def remove_prefix(self):
"""Removes the prefix for a package along with any empty parent directories.""" """Removes the prefix for a package along with any empty parent directories."""
spack.install_layout.remove_path_for_spec(self.spec) spack.install_layout.remove_install_directory(self.spec)
def do_fetch(self): def do_fetch(self):
@ -810,7 +810,7 @@ def do_install(self, **kwargs):
# create the install directory. The install layout # create the install directory. The install layout
# handles this in case so that it can use whatever # handles this in case so that it can use whatever
# package naming scheme it likes. # package naming scheme it likes.
spack.install_layout.make_path_for_spec(self.spec) spack.install_layout.create_install_directory(self.spec)
def cleanup(): def cleanup():
if not keep_prefix: if not keep_prefix:
@ -831,11 +831,11 @@ def real_work():
spack.hooks.pre_install(self) spack.hooks.pre_install(self)
# Set up process's build environment before running install. # Set up process's build environment before running install.
self.stage.chdir_to_source()
if fake_install: if fake_install:
self.do_fake_install() self.do_fake_install()
else: else:
# Subclasses implement install() to do the real work. # Subclasses implement install() to do the real work.
self.stage.chdir_to_source()
self.install(self.spec, self.prefix) self.install(self.spec, self.prefix)
# Ensure that something was actually installed. # Ensure that something was actually installed.

24
lib/spack/spack/spec.py
View File

@ -93,6 +93,7 @@
import sys import sys
import itertools import itertools
import hashlib import hashlib
import base64
from StringIO import StringIO from StringIO import StringIO
from operator import attrgetter from operator import attrgetter
from external import yaml from external import yaml
@ -578,27 +579,12 @@ def prefix(self):
return Prefix(spack.install_layout.path_for_spec(self)) return Prefix(spack.install_layout.path_for_spec(self))
def dep_hash(self, length=None):
"""Return a hash representing all dependencies of this spec
(direct and indirect).
If you want this hash to be consistent, you should
concretize the spec first so that it is not ambiguous.
"""
sha = hashlib.sha1()
sha.update(self.dep_string())
full_hash = sha.hexdigest()
return full_hash[:length]
def dag_hash(self, length=None): def dag_hash(self, length=None):
"""Return a hash of the entire spec DAG, including connectivity.""" """Return a hash of the entire spec DAG, including connectivity."""
sha = hashlib.sha1() yaml_text = yaml.dump(
hash_text = yaml.dump(
self.to_node_dict(), default_flow_style=True, width=sys.maxint) self.to_node_dict(), default_flow_style=True, width=sys.maxint)
sha.update(hash_text) sha = hashlib.sha1(yaml_text)
return sha.hexdigest()[:length] return base64.b32encode(sha.digest()).lower()[:length]
def to_node_dict(self): def to_node_dict(self):
@ -1363,7 +1349,7 @@ def write(s, c):
write(fmt % (c + str(self.architecture)), c) write(fmt % (c + str(self.architecture)), c)
elif c == '#': elif c == '#':
if self.dependencies: if self.dependencies:
out.write(fmt % ('-' + self.dep_hash(8))) out.write(fmt % ('-' + self.dag_hash(8)))
elif c == '$': elif c == '$':
if fmt != '': if fmt != '':
raise ValueError("Can't use format width with $$.") raise ValueError("Can't use format width with $$.")

52
lib/spack/spack/test/directory_layout.py
View File

@ -36,7 +36,11 @@
import spack import spack
from spack.spec import Spec from spack.spec import Spec
from spack.packages import PackageDB from spack.packages import PackageDB
from spack.directory_layout import SpecHashDirectoryLayout from spack.directory_layout import YamlDirectoryLayout
# number of packages to test (to reduce test time)
max_packages = 10
class DirectoryLayoutTest(unittest.TestCase): class DirectoryLayoutTest(unittest.TestCase):
"""Tests that a directory layout works correctly and produces a """Tests that a directory layout works correctly and produces a
@ -44,11 +48,11 @@ class DirectoryLayoutTest(unittest.TestCase):
def setUp(self): def setUp(self):
self.tmpdir = tempfile.mkdtemp() self.tmpdir = tempfile.mkdtemp()
self.layout = SpecHashDirectoryLayout(self.tmpdir) self.layout = YamlDirectoryLayout(self.tmpdir)
def tearDown(self): def tearDown(self):
shutil.rmtree(self.tmpdir, ignore_errors=True) #shutil.rmtree(self.tmpdir, ignore_errors=True)
self.layout = None self.layout = None
@ -59,7 +63,9 @@ def test_read_and_write_spec(self):
finally that the directory can be removed by the directory finally that the directory can be removed by the directory
layout. layout.
""" """
for pkg in spack.db.all_packages(): packages = list(spack.db.all_packages())[:max_packages]
for pkg in packages:
spec = pkg.spec spec = pkg.spec
# If a spec fails to concretize, just skip it. If it is a # If a spec fails to concretize, just skip it. If it is a
@ -69,7 +75,7 @@ def test_read_and_write_spec(self):
except: except:
continue continue
self.layout.make_path_for_spec(spec) self.layout.create_install_directory(spec)
install_dir = self.layout.path_for_spec(spec) install_dir = self.layout.path_for_spec(spec)
spec_path = self.layout.spec_file_path(spec) spec_path = self.layout.spec_file_path(spec)
@ -90,7 +96,7 @@ def test_read_and_write_spec(self):
# Ensure that specs that come out "normal" are really normal. # Ensure that specs that come out "normal" are really normal.
with closing(open(spec_path)) as spec_file: with closing(open(spec_path)) as spec_file:
read_separately = Spec(spec_file.read()) read_separately = Spec.from_yaml(spec_file.read())
read_separately.normalize() read_separately.normalize()
self.assertEqual(read_separately, spec_from_file) self.assertEqual(read_separately, spec_from_file)
@ -98,11 +104,11 @@ def test_read_and_write_spec(self):
read_separately.concretize() read_separately.concretize()
self.assertEqual(read_separately, spec_from_file) self.assertEqual(read_separately, spec_from_file)
# Make sure the dep hash of the read-in spec is the same # Make sure the hash of the read-in spec is the same
self.assertEqual(spec.dep_hash(), spec_from_file.dep_hash()) self.assertEqual(spec.dag_hash(), spec_from_file.dag_hash())
# Ensure directories are properly removed # Ensure directories are properly removed
self.layout.remove_path_for_spec(spec) self.layout.remove_install_directory(spec)
self.assertFalse(os.path.isdir(install_dir)) self.assertFalse(os.path.isdir(install_dir))
self.assertFalse(os.path.exists(install_dir)) self.assertFalse(os.path.exists(install_dir))
@ -120,12 +126,14 @@ def test_handle_unknown_package(self):
""" """
mock_db = PackageDB(spack.mock_packages_path) mock_db = PackageDB(spack.mock_packages_path)
not_in_mock = set(spack.db.all_package_names()).difference( not_in_mock = set.difference(
set(spack.db.all_package_names()),
set(mock_db.all_package_names())) set(mock_db.all_package_names()))
packages = list(not_in_mock)[:max_packages]
# Create all the packages that are not in mock. # Create all the packages that are not in mock.
installed_specs = {} installed_specs = {}
for pkg_name in not_in_mock: for pkg_name in packages:
spec = spack.db.get(pkg_name).spec spec = spack.db.get(pkg_name).spec
# If a spec fails to concretize, just skip it. If it is a # If a spec fails to concretize, just skip it. If it is a
@ -135,7 +143,7 @@ def test_handle_unknown_package(self):
except: except:
continue continue
self.layout.make_path_for_spec(spec) self.layout.create_install_directory(spec)
installed_specs[spec] = self.layout.path_for_spec(spec) installed_specs[spec] = self.layout.path_for_spec(spec)
tmp = spack.db tmp = spack.db
@ -144,12 +152,28 @@ def test_handle_unknown_package(self):
# Now check that even without the package files, we know # Now check that even without the package files, we know
# enough to read a spec from the spec file. # enough to read a spec from the spec file.
for spec, path in installed_specs.items(): for spec, path in installed_specs.items():
spec_from_file = self.layout.read_spec(join_path(path, '.spec')) spec_from_file = self.layout.read_spec(join_path(path, '.spack', 'spec'))
# To satisfy these conditions, directory layouts need to # To satisfy these conditions, directory layouts need to
# read in concrete specs from their install dirs somehow. # read in concrete specs from their install dirs somehow.
self.assertEqual(path, self.layout.path_for_spec(spec_from_file)) self.assertEqual(path, self.layout.path_for_spec(spec_from_file))
self.assertEqual(spec, spec_from_file) self.assertEqual(spec, spec_from_file)
self.assertEqual(spec.dep_hash(), spec_from_file.dep_hash()) self.assertTrue(spec.eq_dag(spec_from_file))
self.assertEqual(spec.dag_hash(), spec_from_file.dag_hash())
spack.db = tmp spack.db = tmp
def test_find(self):
"""Test that finding specs within an install layout works."""
packages = list(spack.db.all_packages())[:max_packages]
installed_specs = {}
for pkg in packages:
spec = pkg.spec.concretized()
installed_specs[spec.name] = spec
self.layout.create_install_directory(spec)
found_specs = dict((s.name, s) for s in self.layout.all_specs())
for name, spec in found_specs.items():
self.assertTrue(name in found_specs)
self.assertTrue(found_specs[name].eq_dag(spec))

4
lib/spack/spack/test/install.py
View File

@ -33,7 +33,7 @@
import spack import spack
from spack.stage import Stage from spack.stage import Stage
from spack.fetch_strategy import URLFetchStrategy from spack.fetch_strategy import URLFetchStrategy
from spack.directory_layout import SpecHashDirectoryLayout from spack.directory_layout import YamlDirectoryLayout
from spack.util.executable import which from spack.util.executable import which
from spack.test.mock_packages_test import * from spack.test.mock_packages_test import *
from spack.test.mock_repo import MockArchive from spack.test.mock_repo import MockArchive
@ -55,7 +55,7 @@ def setUp(self):
# installed pkgs and mock packages. # installed pkgs and mock packages.
self.tmpdir = tempfile.mkdtemp() self.tmpdir = tempfile.mkdtemp()
self.orig_layout = spack.install_layout self.orig_layout = spack.install_layout
spack.install_layout = SpecHashDirectoryLayout(self.tmpdir) spack.install_layout = YamlDirectoryLayout(self.tmpdir)
def tearDown(self): def tearDown(self):