zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor settings for install area and make them config options

Browse Source
This commit is contained in:
Benedikt Hegner 2016-05-06 11:10:02 +02:00 committed by Todd Gamblin
parent 7ccf56a0b7
commit c557e76565
22 changed files with 254 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
etc/spack/install.yaml Normal file
View File

@ -0,0 +1,8 @@
# -------------------------------------------------------------------------
# This is the default spack install setup configuration.
#
# Changes to this file will affect all users of this spack install
# -------------------------------------------------------------------------
install:
path: $spack/opt/spack
layout : YamlDirectoryLayout

14
lib/spack/spack/__init__.py
View File

@ -63,7 +63,6 @@
prefix = spack_root
opt_path = join_path(prefix, "opt")
install_path = join_path(opt_path, "spack")
etc_path = join_path(prefix, "etc")
#
@ -76,25 +75,12 @@
except spack.error.SpackError, e:
tty.die('while initializing Spack RepoPath:', e.message)
#
# Set up the installed packages database
#
from spack.database import Database
installed_db = Database(install_path)
#
# Paths to built-in Spack repositories.
#
packages_path = join_path(repos_path, "builtin")
mock_packages_path = join_path(repos_path, "builtin.mock")
#
# This controls how spack lays out install prefixes and
# stage directories.
#
from spack.directory_layout import YamlDirectoryLayout
install_layout = YamlDirectoryLayout(install_path)
#
# This controls how packages are sorted when trying to choose
# the most preferred package. More preferred packages are sorted

3
lib/spack/spack/build_environment.py
View File

@ -58,6 +58,7 @@
import llnl.util.tty as tty
import spack
import spack.install_area
from llnl.util.filesystem import *
from spack.environment import EnvironmentModifications, validate
from spack.util.environment import *
@ -276,7 +277,7 @@ def set_build_environment_variables(pkg, env, dirty=False):
env.set(SPACK_PREFIX, pkg.prefix)
# Install root prefix
env.set(SPACK_INSTALL, spack.install_path)
env.set(SPACK_INSTALL, spack.install_area.path)
# Stuff in here sanitizes the build environemnt to eliminate
# anything the user has set that may interfere.

10
lib/spack/spack/cmd/__init__.py
View File

@ -27,13 +27,15 @@
import sys
import llnl.util.tty as tty
import spack
import spack.config
import spack.spec
from llnl.util.lang import *
from llnl.util.tty.colify import *
from llnl.util.tty.color import *
import spack
import spack.config
import spack.spec
import spack.install_area
#
# Settings for commands that modify configuration
#
@ -135,7 +137,7 @@ def elide_list(line_list, max_num=10):
def disambiguate_spec(spec):
matching_specs = spack.installed_db.query(spec)
matching_specs = spack.install_area.db.query(spec)
if not matching_specs:
tty.die("Spec '%s' matches no installed packages." % spec)

3
lib/spack/spack/cmd/common/arguments.py
View File

@ -25,6 +25,7 @@
import argparse
import spack.install_area
import spack.modules
from spack.util.pattern import Bunch
__all__ = ['add_common_arguments']
@ -53,7 +54,7 @@ class ConstraintAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
# Query specs from command line
d = self.qualifiers.get(namespace.subparser_name, {})
specs = [s for s in spack.installed_db.query(**d)]
specs = [s for s in spack.install_area.db.query(**d)]
values = ' '.join(values)
if values:
specs = [x for x in specs if x.satisfies(values, strict=True)]

3
lib/spack/spack/cmd/deactivate.py
View File

@ -27,6 +27,7 @@
import spack
import spack.cmd
import spack.install_area
from spack.graph import topological_sort
description = "Deactivate a package extension."
@ -56,7 +57,7 @@ def deactivate(parser, args):
if args.all:
if pkg.extendable:
tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec)
ext_pkgs = spack.installed_db.installed_extensions_for(spec)
ext_pkgs = spack.install_area.db.installed_extensions_for(spec)
for ext_pkg in ext_pkgs:
ext_pkg.spec.normalize()

8
lib/spack/spack/cmd/extensions.py
View File

@ -30,6 +30,7 @@
import spack
import spack.cmd
import spack.cmd.find
import spack.install_area
description = "List extensions for package."
@ -86,8 +87,9 @@ def extensions(parser, args):
#
# List specs of installed extensions.
#
installed = [
s.spec for s in spack.installed_db.installed_extensions_for(spec)]
installed = [s.spec
for s in spack.install_area.db.installed_extensions_for(spec)]
print
if not installed:
tty.msg("None installed.")
@ -98,7 +100,7 @@ def extensions(parser, args):
#
# List specs of activated extensions.
#
activated = spack.install_layout.extension_map(spec)
activated = spack.install_area.layout.extension_map(spec)
print
if not activated:
tty.msg("None activated.")

5
lib/spack/spack/cmd/find.py
View File

@ -28,6 +28,7 @@
import llnl.util.tty as tty
import spack
import spack.spec
import spack.install_area
from llnl.util.lang import *
from llnl.util.tty.colify import *
from llnl.util.tty.color import *
@ -145,9 +146,9 @@ def find(parser, args):
# Get all the specs the user asked for
if not query_specs:
specs = set(spack.installed_db.query(**q_args))
specs = set(spack.install_area.db.query(**q_args))
else:
results = [set(spack.installed_db.query(qs, **q_args))
results = [set(spack.install_area.db.query(qs, **q_args))
for qs in query_specs]
specs = set.union(*results)

4
lib/spack/spack/cmd/reindex.py
View File

@ -23,9 +23,9 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import spack
import spack.install_area
description = "Rebuild Spack's package database."
def reindex(parser, args):
spack.installed_db.reindex(spack.install_layout)
spack.install_area.db.reindex(spack.install_area.layout)

3
lib/spack/spack/cmd/uninstall.py
View File

@ -29,6 +29,7 @@
import llnl.util.tty as tty
import spack
import spack.cmd
import spack.install_area
import spack.repository
description = "Remove an installed package"
@ -89,7 +90,7 @@ def concretize_specs(specs, allow_multiple_matches=False, force=False):
specs_from_cli = []
has_errors = False
for spec in specs:
matching = spack.installed_db.query(spec)
matching = spack.install_area.db.query(spec)
# For each spec provided, make sure it refers to only one package.
# Fail and ask user to be unambiguous if it doesn't
if not allow_multiple_matches and len(matching) > 1:

6
lib/spack/spack/config.py
View File

@ -171,7 +171,7 @@
# Hacked yaml for configuration files preserves line numbers.
import spack.util.spack_yaml as syaml
from spack.build_environment import get_path_from_module
"""Dict from section names -> schema for that section."""
section_schemas = {
@ -180,6 +180,7 @@
'repos': spack.schema.repos.schema,
'packages': spack.schema.packages.schema,
'modules': spack.schema.modules.schema,
'install': spack.schema.install.schema,
}
"""OrderedDict of config scopes keyed by name.
@ -510,6 +511,9 @@ def print_section(section):
def spec_externals(spec):
"""Return a list of external specs (with external directory path filled in),
one for each known external installation."""
# break circular import.
from spack.build_environment import get_path_from_module
allpkgs = get_config('packages')
name = spec.name

20
lib/spack/spack/database.py
View File

@ -48,14 +48,15 @@
from llnl.util.filesystem import *
from llnl.util.lock import *
import spack.spec
import spack.install_area
import spack.repository
from spack.directory_layout import DirectoryLayoutError
from spack.version import Version
from spack.spec import *
import spack.spec
from spack.error import SpackError
from spack.repository import UnknownPackageError
import spack.util.spack_yaml as syaml
# DB goes in this directory underneath the root
_db_dirname = '.spack-db'
@ -66,7 +67,7 @@
_db_lock_timeout = 60
# Types of dependencies tracked by the database
_tracked_deps = nobuild
_tracked_deps = 'nobuild'
def _autospec(function):
@ -218,7 +219,7 @@ def _read_spec_from_yaml(self, hash_key, installs):
spec_dict[name]['hash'] = hash_key
# Build spec from dict first.
spec = Spec.from_node_dict(spec_dict)
spec = spack.spec.Spec.from_node_dict(spec_dict)
return spec
def _assign_dependencies(self, hash_key, installs, data):
@ -229,7 +230,8 @@ def _assign_dependencies(self, hash_key, installs, data):
if 'dependencies' in spec_dict[spec.name]:
yaml_deps = spec_dict[spec.name]['dependencies']
for dname, dhash, dtypes in Spec.read_yaml_dep_specs(yaml_deps):
for dname, dhash, dtypes in spack.spec.Spec.read_yaml_dep_specs(
yaml_deps):
if dhash not in data:
tty.warn("Missing dependency not in database: ",
"%s needs %s-%s" % (
@ -278,7 +280,7 @@ def check(cond, msg):
if version > _db_version:
raise InvalidDatabaseVersionError(_db_version, version)
elif version < _db_version:
self.reindex(spack.install_layout)
self.reindex(spack.install_area.layout)
installs = dict((k, v.to_dict()) for k, v in self._data.items())
def invalid_record(hash_key, error):
@ -444,7 +446,7 @@ def _read(self):
else:
# The file doesn't exist, try to traverse the directory.
# reindex() takes its own write lock, so no lock here.
self.reindex(spack.install_layout)
self.reindex(spack.install_area.layout)
def _add(self, spec, directory_layout=None, explicit=False):
"""Add an install record for this spec to the database.
@ -587,7 +589,7 @@ def installed_extensions_for(self, extendee_spec):
try:
if s.package.extends(extendee_spec):
yield s.package
except UnknownPackageError:
except spack.repository.UnknownPackageError:
continue
# skips unknown packages
# TODO: conditional way to do this instead of catching exceptions

4
lib/spack/spack/directory_layout.py
View File

@ -32,7 +32,7 @@
from llnl.util.filesystem import join_path, mkdirp
import spack
from spack.spec import Spec
import spack.spec
from spack.error import SpackError
@ -210,7 +210,7 @@ def read_spec(self, path):
"""Read the contents of a file and parse them as a spec"""
try:
with open(path) as f:
spec = Spec.from_yaml(f)
spec = spack.spec.Spec.from_yaml(f)
except Exception as e:
if spack.debug:
raise

36
lib/spack/spack/install_area.py Normal file
View File

@ -0,0 +1,36 @@
__author__ = "Benedikt Hegner (CERN)"
import re
import llnl.util.tty as tty
from llnl.util.filesystem import *
#
# Read in the config
#
import spack.config
config = spack.config.get_config("install")
#
# Set up the install path
#
path = re.sub(r'^\$spack', spack.prefix, config['path'])
#
# Set up the installed packages database
#
from spack.database import Database
db = Database(path)
#
# This controls how spack lays out install prefixes and
# stage directories.
#
import spack.directory_layout
try:
layout_name = config["layout"]
layout_class = getattr(spack.directory_layout,layout_name)
layout = layout_class(path)
except:
tty.die("Invalid install directory layout %s chosen." %layout_name)

59
lib/spack/spack/package.py
View File

@ -47,6 +47,7 @@
import llnl.util.lock
import llnl.util.tty as tty
import spack
import spack.install_area
import spack.compilers
import spack.directives
import spack.error
@ -831,7 +832,7 @@ def activated(self):
if not self.is_extension:
raise ValueError(
"is_extension called on package that is not an extension.")
exts = spack.install_layout.extension_map(self.extendee_spec)
exts = spack.install_area.layout.extension_map(self.extendee_spec)
return (self.name in exts) and (exts[self.name] == self.spec)
def provides(self, vpkg_name):
@ -852,7 +853,7 @@ def installed_dependents(self):
TODO: move this method to database.py?
"""
dependents = []
for spec in spack.installed_db.query():
for spec in spack.install_area.db.query():
if self.name == spec.name:
continue
# XXX(deptype): Should build dependencies not count here?
@ -866,7 +867,7 @@ def installed_dependents(self):
def prefix_lock(self):
"""Prefix lock is a byte range lock on the nth byte of a file.
The lock file is ``spack.installed_db.prefix_lock`` -- the DB
The lock file is ``spack.install_area.db.prefix_lock`` -- the DB
tells us what to call it and it lives alongside the install DB.
n is the sys.maxsize-bit prefix of the DAG hash. This makes
@ -878,7 +879,7 @@ def prefix_lock(self):
prefix = self.spec.prefix
if prefix not in Package.prefix_locks:
Package.prefix_locks[prefix] = llnl.util.lock.Lock(
spack.installed_db.prefix_lock_path,
spack.install_area.db.prefix_lock_path,
self.spec.dag_hash_bit_prefix(bit_length(sys.maxsize)), 1)
self._prefix_lock = Package.prefix_locks[prefix]
@ -915,7 +916,7 @@ def remove_prefix(self):
Removes the prefix for a package along with any empty parent
directories
"""
spack.install_layout.remove_install_directory(self.spec)
spack.install_area.layout.remove_install_directory(self.spec)
def do_fetch(self, mirror_only=False):
"""
@ -1152,15 +1153,15 @@ def do_install(self,
return
# Ensure package is not already installed
layout = spack.install_layout
layout = spack.install_area.layout
with self._prefix_read_lock():
if layout.check_installed(self.spec):
tty.msg(
"%s is already installed in %s" % (self.name, self.prefix))
rec = spack.installed_db.get_record(self.spec)
rec = spack.install_area.db.get_record(self.spec)
if (not rec.explicit) and explicit:
with spack.installed_db.write_transaction():
rec = spack.installed_db.get_record(self.spec)
with spack.install_area.db.write_transaction():
rec = spack.install_area.db.get_record(self.spec)
rec.explicit = True
return
@ -1265,15 +1266,15 @@ def build_process():
try:
# Create the install prefix and fork the build process.
spack.install_layout.create_install_directory(self.spec)
spack.install_area.layout.create_install_directory(self.spec)
# Fork a child to do the actual installation
spack.build_environment.fork(self, build_process, dirty=dirty)
# If we installed then we should keep the prefix
keep_prefix = True if self.last_phase is None else keep_prefix
# note: PARENT of the build process adds the new package to
# the database, so that we don't need to re-read from file.
spack.installed_db.add(
self.spec, spack.install_layout, explicit=explicit
spack.install_area.db.add(
self.spec, spack.install_area.layout, explicit=explicit
)
except directory_layout.InstallDirectoryAlreadyExistsError:
# Abort install if install directory exists.
@ -1306,11 +1307,11 @@ def _do_install_pop_kwargs(self, kwargs):
def log(self):
# Copy provenance into the install directory on success
log_install_path = spack.install_layout.build_log_path(
log_install_path = spack.install_area.layout.build_log_path(
self.spec)
env_install_path = spack.install_layout.build_env_path(
env_install_path = spack.install_area.layout.build_env_path(
self.spec)
packages_dir = spack.install_layout.build_packages_path(
packages_dir = spack.install_area.layout.build_packages_path(
self.spec)
# Remove first if we're overwriting another build
@ -1344,7 +1345,7 @@ def check_paths(path_list, filetype, predicate):
check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir)
installed = set(os.listdir(self.prefix))
installed.difference_update(spack.install_layout.hidden_file_paths)
installed.difference_update(spack.install_area.layout.hidden_file_paths)
if not installed:
raise InstallError(
"Install failed for %s. Nothing was installed!" % self.name)
@ -1352,7 +1353,7 @@ def check_paths(path_list, filetype, predicate):
@property
def build_log_path(self):
if self.installed:
return spack.install_layout.build_log_path(self.spec)
return spack.install_area.layout.build_log_path(self.spec)
else:
return join_path(self.stage.source_path, 'spack-build.out')
@ -1482,9 +1483,9 @@ def do_uninstall(self, force=False):
if not self.installed:
# prefix may not exist, but DB may be inconsistent. Try to fix by
# removing, but omit hooks.
specs = spack.installed_db.query(self.spec, installed=True)
specs = spack.install_area.db.query(self.spec, installed=True)
if specs:
spack.installed_db.remove(specs[0])
spack.install_area.db.remove(specs[0])
tty.msg("Removed stale DB entry for %s" % self.spec.short_spec)
return
else:
@ -1501,7 +1502,7 @@ def do_uninstall(self, force=False):
# Uninstalling in Spack only requires removing the prefix.
self.remove_prefix()
#
spack.installed_db.remove(self.spec)
spack.install_area.db.remove(self.spec)
tty.msg("Successfully uninstalled %s" % self.spec.short_spec)
# Once everything else is done, run post install hooks
@ -1535,7 +1536,7 @@ def do_activate(self, force=False):
"""
self._sanity_check_extension()
spack.install_layout.check_extension_conflict(self.extendee_spec,
spack.install_area.layout.check_extension_conflict(self.extendee_spec,
self.spec)
# Activate any package dependencies that are also extensions.
@ -1547,7 +1548,7 @@ def do_activate(self, force=False):
self.extendee_spec.package.activate(self, **self.extendee_args)
spack.install_layout.add_extension(self.extendee_spec, self.spec)
spack.install_area.layout.add_extension(self.extendee_spec, self.spec)
tty.msg("Activated extension %s for %s" %
(self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@")))
@ -1562,7 +1563,7 @@ def activate(self, extension, **kwargs):
"""
def ignore(filename):
return (filename in spack.install_layout.hidden_file_paths or
return (filename in spack.install_area.layout.hidden_file_paths or
kwargs.get('ignore', lambda f: False)(filename))
tree = LinkTree(extension.prefix)
@ -1580,9 +1581,9 @@ def do_deactivate(self, **kwargs):
# Allow a force deactivate to happen. This can unlink
# spurious files if something was corrupted.
if not force:
spack.install_layout.check_activated(self.extendee_spec, self.spec)
spack.install_area.layout.check_activated(self.extendee_spec, self.spec)
activated = spack.install_layout.extension_map(self.extendee_spec)
activated = spack.install_area.layout.extension_map(self.extendee_spec)
for name, aspec in activated.items():
if aspec == self.spec:
continue
@ -1598,7 +1599,7 @@ def do_deactivate(self, **kwargs):
# redundant activation check -- makes SURE the spec is not
# still activated even if something was wrong above.
if self.activated:
spack.install_layout.remove_extension(self.extendee_spec,
spack.install_area.layout.remove_extension(self.extendee_spec,
self.spec)
tty.msg("Deactivated extension %s for %s" %
@ -1615,7 +1616,7 @@ def deactivate(self, extension, **kwargs):
"""
def ignore(filename):
return (filename in spack.install_layout.hidden_file_paths or
return (filename in spack.install_area.layout.hidden_file_paths or
kwargs.get('ignore', lambda f: False)(filename))
tree = LinkTree(extension.prefix)
@ -1716,7 +1717,7 @@ def flatten_dependencies(spec, flat_dir):
for dep in spec.traverse(root=False):
name = dep.name
dep_path = spack.install_layout.path_for_spec(dep)
dep_path = spack.install_area.layout.path_for_spec(dep)
dep_files = LinkTree(dep_path)
os.mkdir(flat_dir + '/' + name)
@ -1745,7 +1746,7 @@ def dump_packages(spec, path):
if node is not spec:
# Locate the dependency package in the install tree and find
# its provenance information.
source = spack.install_layout.build_packages_path(node)
source = spack.install_area.layout.build_packages_path(node)
source_repo_root = join_path(source, node.namespace)
# There's no provenance installed for the source package. Skip it.

2
lib/spack/spack/repository.py
View File

@ -42,7 +42,6 @@
import spack
import spack.error
import spack.config
import spack.spec
from spack.provider_index import ProviderIndex
from spack.util.naming import *
@ -128,6 +127,7 @@ def __init__(self, *repo_dirs, **kwargs):
# If repo_dirs is empty, just use the configuration
if not repo_dirs:
import spack.config
repo_dirs = spack.config.get_config('repos')
if not repo_dirs:
raise NoRepoConfiguredError(

43
lib/spack/spack/schema/install.py Normal file
View File

@ -0,0 +1,43 @@
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""Schema for packages.yaml configuration files."""
schema = {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack module file configuration file schema',
'type': 'object',
'additionalProperties': False,
'patternProperties': {
'install': {
'type': 'object',
'default' : {},
'additionalProperties': False,
'properties': {
'path' : { 'type': 'string' },
'layout' : { 'type': 'string' }
}
},
},
}

7
lib/spack/spack/spec.py
View File

@ -111,6 +111,7 @@
import spack
import spack.architecture
import spack.install_area
import spack.compilers as compilers
import spack.error
import spack.parse
@ -964,7 +965,7 @@ def cshort_spec(self):
@property
def prefix(self):
return Prefix(spack.install_layout.path_for_spec(self))
return Prefix(spack.install_area.layout.path_for_spec(self))
def dag_hash(self, length=None):
"""Return a hash of the entire spec DAG, including connectivity."""
@ -2328,7 +2329,7 @@ def write(s, c):
elif named_str == 'SPACK_ROOT':
out.write(fmt % spack.prefix)
elif named_str == 'SPACK_INSTALL':
out.write(fmt % spack.install_path)
out.write(fmt % spack.install_area.path)
elif named_str == 'PREFIX':
out.write(fmt % self.prefix)
elif named_str.startswith('HASH'):
@ -2532,7 +2533,7 @@ def parse_compiler(self, text):
def spec_by_hash(self):
self.expect(ID)
specs = spack.installed_db.query()
specs = spack.install_area.db.query()
matches = [spec for spec in specs if
spec.dag_hash()[:len(self.token.value)] == self.token.value]

4
lib/spack/spack/test/cmd/uninstall.py
View File

@ -23,7 +23,7 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import spack.test.mock_database
import spack.install_area
from spack.cmd.uninstall import uninstall
@ -51,7 +51,7 @@ def test_uninstall(self):
args = MockArgs(['callpath'], all=True, dependents=True)
uninstall(parser, args)
all_specs = spack.install_layout.all_specs()
all_specs = spack.install_area.layout.all_specs()
self.assertEqual(len(all_specs), 7)
# query specs with multiple configurations
mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')]

129
lib/spack/spack/test/database.py
View File

@ -30,6 +30,7 @@
import os.path
import spack
import spack.install_area
from llnl.util.filesystem import join_path
from llnl.util.tty.colify import colify
from spack.test.mock_database import MockDatabase
@ -40,16 +41,16 @@ def _print_ref_counts():
recs = []
def add_rec(spec):
cspecs = spack.installed_db.query(spec, installed=any)
cspecs = spack.install_area.db.query(spec, installed=any)
if not cspecs:
recs.append("[ %-7s ] %-20s-" % ('', spec))
else:
key = cspecs[0].dag_hash()
rec = spack.installed_db.get_record(cspecs[0])
rec = spack.install_area.db.get_record(cspecs[0])
recs.append("[ %-7s ] %-20s%d" % (key[:7], spec, rec.ref_count))
with spack.installed_db.read_transaction():
with spack.install_area.db.read_transaction():
add_rec('mpileaks ^mpich')
add_rec('callpath ^mpich')
add_rec('mpich')
@ -82,7 +83,7 @@ def test_005_db_exists(self):
def test_010_all_install_sanity(self):
"""Ensure that the install layout reflects what we think it does."""
all_specs = spack.install_layout.all_specs()
all_specs = spack.install_area.layout.all_specs()
self.assertEqual(len(all_specs), 13)
# query specs with multiple configurations
@ -113,12 +114,12 @@ def test_010_all_install_sanity(self):
def test_015_write_and_read(self):
# write and read DB
with spack.installed_db.write_transaction():
specs = spack.installed_db.query()
recs = [spack.installed_db.get_record(s) for s in specs]
with spack.install_area.db.write_transaction():
specs = spack.install_area.db.query()
recs = [spack.install_area.db.get_record(s) for s in specs]
for spec, rec in zip(specs, recs):
new_rec = spack.installed_db.get_record(spec)
new_rec = spack.install_area.db.get_record(spec)
self.assertEqual(new_rec.ref_count, rec.ref_count)
self.assertEqual(new_rec.spec, rec.spec)
self.assertEqual(new_rec.path, rec.path)
@ -126,7 +127,7 @@ def test_015_write_and_read(self):
def _check_merkleiness(self):
"""Ensure the spack database is a valid merkle graph."""
all_specs = spack.installed_db.query(installed=any)
all_specs = spack.install_area.db.query(installed=any)
seen = {}
for spec in all_specs:
@ -139,8 +140,8 @@ def _check_merkleiness(self):
def _check_db_sanity(self):
"""Utiilty function to check db against install layout."""
expected = sorted(spack.install_layout.all_specs())
actual = sorted(self.installed_db.query())
expected = sorted(spack.install_area.layout.all_specs())
actual = sorted(self.install_db.query())
self.assertEqual(len(expected), len(actual))
for e, a in zip(expected, actual):
@ -154,13 +155,13 @@ def test_020_db_sanity(self):
def test_025_reindex(self):
"""Make sure reindex works and ref counts are valid."""
spack.installed_db.reindex(spack.install_layout)
spack.install_area.db.reindex(spack.install_area.layout)
self._check_db_sanity()
def test_030_db_sanity_from_another_process(self):
def read_and_modify():
self._check_db_sanity() # check that other process can read DB
with self.installed_db.write_transaction():
with self.install_db.write_transaction():
self._mock_remove('mpileaks ^zmpi')
p = multiprocessing.Process(target=read_and_modify, args=())
@ -168,40 +169,40 @@ def read_and_modify():
p.join()
# ensure child process change is visible in parent process
with self.installed_db.read_transaction():
self.assertEqual(len(self.installed_db.query('mpileaks ^zmpi')), 0)
with self.install_db.read_transaction():
self.assertEqual(len(self.install_db.query('mpileaks ^zmpi')), 0)
def test_040_ref_counts(self):
"""Ensure that we got ref counts right when we read the DB."""
self.installed_db._check_ref_counts()
self.install_db._check_ref_counts()
def test_050_basic_query(self):
"""Ensure querying database is consistent with what is installed."""
# query everything
self.assertEqual(len(spack.installed_db.query()), 13)
self.assertEqual(len(spack.install_area.db.query()), 13)
# query specs with multiple configurations
mpileaks_specs = self.installed_db.query('mpileaks')
callpath_specs = self.installed_db.query('callpath')
mpi_specs = self.installed_db.query('mpi')
mpileaks_specs = self.install_db.query('mpileaks')
callpath_specs = self.install_db.query('callpath')
mpi_specs = self.install_db.query('mpi')
self.assertEqual(len(mpileaks_specs), 3)
self.assertEqual(len(callpath_specs), 3)
self.assertEqual(len(mpi_specs), 3)
# query specs with single configurations
dyninst_specs = self.installed_db.query('dyninst')
libdwarf_specs = self.installed_db.query('libdwarf')
libelf_specs = self.installed_db.query('libelf')
dyninst_specs = self.install_db.query('dyninst')
libdwarf_specs = self.install_db.query('libdwarf')
libelf_specs = self.install_db.query('libelf')
self.assertEqual(len(dyninst_specs), 1)
self.assertEqual(len(libdwarf_specs), 1)
self.assertEqual(len(libelf_specs), 1)
# Query by dependency
self.assertEqual(len(self.installed_db.query('mpileaks ^mpich')), 1)
self.assertEqual(len(self.installed_db.query('mpileaks ^mpich2')), 1)
self.assertEqual(len(self.installed_db.query('mpileaks ^zmpi')), 1)
self.assertEqual(len(self.install_db.query('mpileaks ^mpich')), 1)
self.assertEqual(len(self.install_db.query('mpileaks ^mpich2')), 1)
self.assertEqual(len(self.install_db.query('mpileaks ^zmpi')), 1)
def _check_remove_and_add_package(self, spec):
"""Remove a spec from the DB, then add it and make sure everything's
@ -209,13 +210,13 @@ def _check_remove_and_add_package(self, spec):
removed, that it's back when added again, and that ref
counts are consistent.
"""
original = self.installed_db.query()
self.installed_db._check_ref_counts()
original = self.install_db.query()
self.install_db._check_ref_counts()
# Remove spec
concrete_spec = self.installed_db.remove(spec)
self.installed_db._check_ref_counts()
remaining = self.installed_db.query()
concrete_spec = self.install_db.remove(spec)
self.install_db._check_ref_counts()
remaining = self.install_db.query()
# ensure spec we removed is gone
self.assertEqual(len(original) - 1, len(remaining))
@ -223,14 +224,14 @@ def _check_remove_and_add_package(self, spec):
self.assertTrue(concrete_spec not in remaining)
# add it back and make sure everything is ok.
self.installed_db.add(concrete_spec, spack.install_layout)
installed = self.installed_db.query()
self.install_db.add(concrete_spec, spack.install_area.layout)
installed = self.install_db.query()
self.assertTrue(concrete_spec in installed)
self.assertEqual(installed, original)
# sanity check against direcory layout and check ref counts.
self._check_db_sanity()
self.installed_db._check_ref_counts()
self.install_db._check_ref_counts()
def test_060_remove_and_add_root_package(self):
self._check_remove_and_add_package('mpileaks ^mpich')
@ -239,95 +240,95 @@ def test_070_remove_and_add_dependency_package(self):
self._check_remove_and_add_package('dyninst')
def test_080_root_ref_counts(self):
rec = self.installed_db.get_record('mpileaks ^mpich')
rec = self.install_db.get_record('mpileaks ^mpich')
# Remove a top-level spec from the DB
self.installed_db.remove('mpileaks ^mpich')
self.install_db.remove('mpileaks ^mpich')
# record no longer in DB
self.assertEqual(
self.installed_db.query('mpileaks ^mpich', installed=any), [])
self.install_db.query('mpileaks ^mpich', installed=any), [])
# record's deps have updated ref_counts
self.assertEqual(
self.installed_db.get_record('callpath ^mpich').ref_count, 0)
self.assertEqual(self.installed_db.get_record('mpich').ref_count, 1)
self.install_db.get_record('callpath ^mpich').ref_count, 0)
self.assertEqual(self.install_db.get_record('mpich').ref_count, 1)
# Put the spec back
self.installed_db.add(rec.spec, spack.install_layout)
self.install_db.add(rec.spec, spack.install_area.layout)
# record is present again
self.assertEqual(
len(self.installed_db.query('mpileaks ^mpich', installed=any)), 1)
len(self.install_db.query('mpileaks ^mpich', installed=any)), 1)
# dependencies have ref counts updated
self.assertEqual(
self.installed_db.get_record('callpath ^mpich').ref_count, 1)
self.assertEqual(self.installed_db.get_record('mpich').ref_count, 2)
self.install_db.get_record('callpath ^mpich').ref_count, 1)
self.assertEqual(self.install_db.get_record('mpich').ref_count, 2)
def test_090_non_root_ref_counts(self):
self.installed_db.get_record('mpileaks ^mpich')
self.installed_db.get_record('callpath ^mpich')
self.install_db.get_record('mpileaks ^mpich')
self.install_db.get_record('callpath ^mpich')
# "force remove" a non-root spec from the DB
self.installed_db.remove('callpath ^mpich')
self.install_db.remove('callpath ^mpich')
# record still in DB but marked uninstalled
self.assertEqual(
self.installed_db.query('callpath ^mpich', installed=True), [])
self.install_db.query('callpath ^mpich', installed=True), [])
self.assertEqual(
len(self.installed_db.query('callpath ^mpich', installed=any)), 1)
len(self.install_db.query('callpath ^mpich', installed=any)), 1)
# record and its deps have same ref_counts
self.assertEqual(self.installed_db.get_record(
self.assertEqual(self.install_db.get_record(
'callpath ^mpich', installed=any).ref_count, 1)
self.assertEqual(self.installed_db.get_record('mpich').ref_count, 2)
self.assertEqual(self.install_db.get_record('mpich').ref_count, 2)
# remove only dependent of uninstalled callpath record
self.installed_db.remove('mpileaks ^mpich')
self.install_db.remove('mpileaks ^mpich')
# record and parent are completely gone.
self.assertEqual(
self.installed_db.query('mpileaks ^mpich', installed=any), [])
self.install_db.query('mpileaks ^mpich', installed=any), [])
self.assertEqual(
self.installed_db.query('callpath ^mpich', installed=any), [])
self.install_db.query('callpath ^mpich', installed=any), [])
# mpich ref count updated properly.
mpich_rec = self.installed_db.get_record('mpich')
mpich_rec = self.install_db.get_record('mpich')
self.assertEqual(mpich_rec.ref_count, 0)
def test_100_no_write_with_exception_on_remove(self):
def fail_while_writing():
with self.installed_db.write_transaction():
with self.install_db.write_transaction():
self._mock_remove('mpileaks ^zmpi')
raise Exception()
with self.installed_db.read_transaction():
with self.install_db.read_transaction():
self.assertEqual(
len(self.installed_db.query('mpileaks ^zmpi', installed=any)),
len(self.install_db.query('mpileaks ^zmpi', installed=any)),
1)
self.assertRaises(Exception, fail_while_writing)
# reload DB and make sure zmpi is still there.
with self.installed_db.read_transaction():
with self.install_db.read_transaction():
self.assertEqual(
len(self.installed_db.query('mpileaks ^zmpi', installed=any)),
len(self.install_db.query('mpileaks ^zmpi', installed=any)),
1)
def test_110_no_write_with_exception_on_install(self):
def fail_while_writing():
with self.installed_db.write_transaction():
with self.install_db.write_transaction():
self._mock_install('cmake')
raise Exception()
with self.installed_db.read_transaction():
with self.install_db.read_transaction():
self.assertEqual(
self.installed_db.query('cmake', installed=any), [])
self.install_db.query('cmake', installed=any), [])
self.assertRaises(Exception, fail_while_writing)
# reload DB and make sure cmake was not written.
with self.installed_db.read_transaction():
with self.install_db.read_transaction():
self.assertEqual(
self.installed_db.query('cmake', installed=any), [])
self.install_db.query('cmake', installed=any), [])

15
lib/spack/spack/test/install.py
View File

@ -26,6 +26,7 @@
import tempfile
import spack
import spack.install_area
from llnl.util.filesystem import *
from spack.directory_layout import YamlDirectoryLayout
from spack.database import Database
@ -49,11 +50,11 @@ def setUp(self):
# Use a fake install directory to avoid conflicts bt/w
# installed pkgs and mock packages.
self.tmpdir = tempfile.mkdtemp()
self.orig_layout = spack.install_layout
self.orig_db = spack.installed_db
self.orig_layout = spack.install_area.layout
self.orig_db = spack.install_area.db
spack.install_layout = YamlDirectoryLayout(self.tmpdir)
spack.installed_db = Database(self.tmpdir)
spack.install_area.layout = YamlDirectoryLayout(self.tmpdir)
spack.install_area.db = Database(self.tmpdir)
def tearDown(self):
super(InstallTest, self).tearDown()
@ -63,8 +64,8 @@ def tearDown(self):
spack.do_checksum = True
# restore spack's layout.
spack.install_layout = self.orig_layout
spack.installed_db = self.orig_db
spack.install_area.layout = self.orig_layout
spack.install_area.db = self.orig_db
shutil.rmtree(self.tmpdir, ignore_errors=True)
def fake_fetchify(self, pkg):
@ -91,7 +92,7 @@ def test_install_and_uninstall(self):
pkg.remove_prefix()
raise
def test_install_environment(self):
def test_install_area(self):
spec = Spec('cmake-client').concretized()
for s in spec.traverse():

29
lib/spack/spack/test/mock_database.py
View File

@ -26,6 +26,7 @@
import tempfile
import spack
import spack.install_area
from spack.spec import Spec
from spack.database import Database
from spack.directory_layout import YamlDirectoryLayout
@ -41,7 +42,7 @@ def _mock_install(self, spec):
pkg.do_install(fake=True)
def _mock_remove(self, spec):
specs = spack.installed_db.query(spec)
specs = spack.install_area.db.query(spec)
assert len(specs) == 1
spec = specs[0]
spec.package.do_uninstall(spec)
@ -54,17 +55,17 @@ def setUp(self):
# Make a fake install directory
self.install_path = tempfile.mkdtemp()
self.spack_install_path = spack.install_path
spack.install_path = self.install_path
self.spack_install_path = spack.install_area.path
spack.install_area.path = self.install_path
self.install_layout = YamlDirectoryLayout(self.install_path)
self.spack_install_layout = spack.install_layout
spack.install_layout = self.install_layout
self.spack_install_layout = spack.install_area.layout
spack.install_area.layout = self.install_layout
# Make fake database and fake install directory.
self.installed_db = Database(self.install_path)
self.spack_installed_db = spack.installed_db
spack.installed_db = self.installed_db
self.install_db = Database(self.install_path)
self.spack_install_db = spack.install_area.db
spack.install_area.db = self.install_db
# make a mock database with some packages installed note that
# the ref count for dyninst here will be 3, as it's recycled
@ -90,18 +91,18 @@ def setUp(self):
#
# Transaction used to avoid repeated writes.
with spack.installed_db.write_transaction():
with spack.install_area.db.write_transaction():
self._mock_install('mpileaks ^mpich')
self._mock_install('mpileaks ^mpich2')
self._mock_install('mpileaks ^zmpi')
def tearDown(self):
with spack.installed_db.write_transaction():
for spec in spack.installed_db.query():
with spack.install_area.db.write_transaction():
for spec in spack.install_area.db.query():
spec.package.do_uninstall(spec)
super(MockDatabase, self).tearDown()
shutil.rmtree(self.install_path)
spack.install_path = self.spack_install_path
spack.install_layout = self.spack_install_layout
spack.installed_db = self.spack_installed_db
spack.install_area.path = self.spack_install_path
spack.install_area.layout = self.spack_install_layout
spack.install_area.db = self.spack_install_db