Use a single lock file for stages and a single file for prefixes.

- Locks now use fcntl range locks on a single file.

How it works for prefixes:

- Each lock is a byte range lock on the nth byte of a file.

- The lock file is ``spack.installed_db.prefix_lock`` -- the DB tells us
  what to call it and it lives alongside the install DB.  n is the
  sys.maxsize-bit prefix of the DAG hash.

For stages, we take the sha1 of the stage name and use that to select a
byte to lock.

With 100 concurrent builds, the likelihood of a false lock collision is
~5.36e-16, so this scheme should retain more than sufficient paralellism
(with no chance of false negatives), and get us reader-writer lock
semantics with a single file, so no need to clean up lots of lock files.
This commit is contained in:
Todd Gamblin 2016-10-06 01:31:31 -07:00
parent 080a78664e
commit 222f551c37
5 changed files with 57 additions and 18 deletions

View File

@ -156,6 +156,9 @@ def __init__(self, root, db_dir=None):
self._index_path = join_path(self._db_dir, 'index.yaml') self._index_path = join_path(self._db_dir, 'index.yaml')
self._lock_path = join_path(self._db_dir, 'lock') self._lock_path = join_path(self._db_dir, 'lock')
# This is for other classes to use to lock prefix directories.
self.prefix_lock_path = join_path(self._db_dir, 'prefix_lock')
# Create needed directories and files # Create needed directories and files
if not os.path.exists(self._db_dir): if not os.path.exists(self._db_dir):
mkdirp(self._db_dir) mkdirp(self._db_dir)

View File

@ -309,6 +309,7 @@ class SomePackage(Package):
Package creators override functions like install() (all of them do this), Package creators override functions like install() (all of them do this),
clean() (some of them do this), and others to provide custom behavior. clean() (some of them do this), and others to provide custom behavior.
""" """
# #
# These are default values for instance variables. # These are default values for instance variables.
# #
@ -340,6 +341,9 @@ class SomePackage(Package):
""" """
sanity_check_is_dir = [] sanity_check_is_dir = []
"""Per-process lock objects for each install prefix."""
prefix_locks = {}
class __metaclass__(type): class __metaclass__(type):
"""Ensure attributes required by Spack directives are present.""" """Ensure attributes required by Spack directives are present."""
def __init__(cls, name, bases, dict): def __init__(cls, name, bases, dict):
@ -700,11 +704,24 @@ def installed_dependents(self):
@property @property
def prefix_lock(self): def prefix_lock(self):
"""Prefix lock is a byte range lock on the nth byte of a file.
The lock file is ``spack.installed_db.prefix_lock`` -- the DB
tells us what to call it and it lives alongside the install DB.
n is the sys.maxsize-bit prefix of the DAG hash. This makes
likelihood of collision is very low AND it gives us
readers-writer lock semantics with just a single lockfile, so no
cleanup required.
"""
if self._prefix_lock is None: if self._prefix_lock is None:
dirname = join_path(os.path.dirname(self.spec.prefix), '.locks') prefix = self.spec.prefix
basename = os.path.basename(self.spec.prefix) if prefix not in Package.prefix_locks:
self._prefix_lock = llnl.util.lock.Lock( Package.prefix_locks[prefix] = llnl.util.lock.Lock(
join_path(dirname, basename)) spack.installed_db.prefix_lock_path,
self.spec.dag_hash_bit_prefix(sys.maxsize.bit_length()), 1)
self._prefix_lock = Package.prefix_locks[prefix]
return self._prefix_lock return self._prefix_lock

View File

@ -120,6 +120,7 @@
from spack.util.string import * from spack.util.string import *
import spack.util.spack_yaml as syaml import spack.util.spack_yaml as syaml
from spack.util.spack_yaml import syaml_dict from spack.util.spack_yaml import syaml_dict
from spack.util.crypto import prefix_bits
from spack.version import * from spack.version import *
from spack.provider_index import ProviderIndex from spack.provider_index import ProviderIndex
@ -2733,17 +2734,7 @@ def base32_prefix_bits(hash_string, bits):
% (bits, hash_string)) % (bits, hash_string))
hash_bytes = base64.b32decode(hash_string, casefold=True) hash_bytes = base64.b32decode(hash_string, casefold=True)
return prefix_bits(hash_bytes, bits)
result = 0
n = 0
for i, b in enumerate(hash_bytes):
n += 8
result = (result << 8) | ord(b)
if n >= bits:
break
result >>= (n - bits)
return result
class SpecError(spack.error.SpackError): class SpecError(spack.error.SpackError):

View File

@ -23,7 +23,9 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
############################################################################## ##############################################################################
import os import os
import sys
import errno import errno
import hashlib
import shutil import shutil
import tempfile import tempfile
from urlparse import urljoin from urlparse import urljoin
@ -39,6 +41,7 @@
import spack.fetch_strategy as fs import spack.fetch_strategy as fs
import spack.error import spack.error
from spack.version import * from spack.version import *
from spack.util.crypto import prefix_bits
STAGE_PREFIX = 'spack-stage-' STAGE_PREFIX = 'spack-stage-'
@ -89,6 +92,9 @@ class Stage(object):
similar, and are intended to persist for only one run of spack. similar, and are intended to persist for only one run of spack.
""" """
"""Shared dict of all stage locks."""
stage_locks = {}
def __init__( def __init__(
self, url_or_fetch_strategy, self, url_or_fetch_strategy,
name=None, mirror_path=None, keep=False, path=None, lock=True): name=None, mirror_path=None, keep=False, path=None, lock=True):
@ -149,11 +155,19 @@ def __init__(
# Flag to decide whether to delete the stage folder on exit or not # Flag to decide whether to delete the stage folder on exit or not
self.keep = keep self.keep = keep
# File lock for the stage directory # File lock for the stage directory. We use one file for all
# stage locks. See Spec.prefix_lock for details on this approach.
self._lock = None self._lock = None
if lock: if lock:
self._lock = llnl.util.lock.Lock( if self.name not in Stage.stage_locks:
join_path(spack.stage_path, self.name + '.lock')) sha1 = hashlib.sha1(self.name).digest()
lock_id = prefix_bits(sha1, sys.maxsize.bit_length())
stage_lock_path = join_path(spack.stage_path, '.lock')
Stage.stage_locks[self.name] = llnl.util.lock.Lock(
stage_lock_path, lock_id, 1)
self._lock = Stage.stage_locks[self.name]
def __enter__(self): def __enter__(self):
""" """

View File

@ -100,3 +100,17 @@ def check(self, filename):
self.sum = checksum( self.sum = checksum(
self.hash_fun, filename, block_size=self.block_size) self.hash_fun, filename, block_size=self.block_size)
return self.sum == self.hexdigest return self.sum == self.hexdigest
def prefix_bits(byte_array, bits):
"""Return the first <bits> bits of a byte array as an integer."""
result = 0
n = 0
for i, b in enumerate(byte_array):
n += 8
result = (result << 8) | ord(b)
if n >= bits:
break
result >>= (n - bits)
return result