zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #977 from trws/explicit-docs

Browse Source 
initial docs for find and flake8 cleanup
This commit is contained in:
Todd Gamblin 2016-05-18 15:20:23 -07:00
commit c53b1be8b4
6 changed files with 425 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/spack/docs/basic_usage.rst
View File

@ -246,6 +246,12 @@ Packages are divided into groups according to their architecture and
compiler. Within each group, Spack tries to keep the view simple, and compiler. Within each group, Spack tries to keep the view simple, and
only shows the version of installed packages. only shows the version of installed packages.
``spack find`` can filter the package list based on the package name, spec, or
a number of properties of their installation status. For example, missing
dependencies of a spec can be shown with ``-m``, packages which were
explicitly installed with ``spack install <package>`` can be singled out with
``-e`` and those which have been pulled in only as dependencies with ``-E``.
In some cases, there may be different configurations of the *same* In some cases, there may be different configurations of the *same*
version of a package installed. For example, there are two version of a package installed. For example, there are two
installations of of ``libdwarf@20130729`` above. We can look at them installations of of ``libdwarf@20130729`` above. We can look at them

40
lib/spack/spack/cmd/find.py
View File

@ -37,71 +37,59 @@
def setup_parser(subparser): def setup_parser(subparser):
format_group = subparser.add_mutually_exclusive_group() format_group = subparser.add_mutually_exclusive_group()
format_group.add_argument('-s', format_group.add_argument('-s', '--short',
'--short',
action='store_const', action='store_const',
dest='mode', dest='mode',
const='short', const='short',
help='Show only specs (default)') help='Show only specs (default)')
format_group.add_argument('-p', format_group.add_argument('-p', '--paths',
'--paths',
action='store_const', action='store_const',
dest='mode', dest='mode',
const='paths', const='paths',
help='Show paths to package install directories') help='Show paths to package install directories')
format_group.add_argument( format_group.add_argument(
'-d', '-d', '--deps',
'--deps',
action='store_const', action='store_const',
dest='mode', dest='mode',
const='deps', const='deps',
help='Show full dependency DAG of installed packages') help='Show full dependency DAG of installed packages')
subparser.add_argument('-l', subparser.add_argument('-l', '--long',
'--long',
action='store_true', action='store_true',
dest='long', dest='long',
help='Show dependency hashes as well as versions.') help='Show dependency hashes as well as versions.')
subparser.add_argument('-L', subparser.add_argument('-L', '--very-long',
'--very-long',
action='store_true', action='store_true',
dest='very_long', dest='very_long',
help='Show dependency hashes as well as versions.') help='Show dependency hashes as well as versions.')
subparser.add_argument('-f', subparser.add_argument('-f', '--show-flags',
'--show-flags',
action='store_true', action='store_true',
dest='show_flags', dest='show_flags',
help='Show spec compiler flags.') help='Show spec compiler flags.')
subparser.add_argument( subparser.add_argument(
'-e', '-e', '--explicit',
'--explicit',
action='store_true', action='store_true',
help='Show only specs that were installed explicitly') help='Show only specs that were installed explicitly')
subparser.add_argument( subparser.add_argument(
'-E', '-E', '--implicit',
'--implicit',
action='store_true', action='store_true',
help='Show only specs that were installed as dependencies') help='Show only specs that were installed as dependencies')
subparser.add_argument( subparser.add_argument(
'-u', '-u', '--unknown',
'--unknown',
action='store_true', action='store_true',
dest='unknown', dest='unknown',
help='Show only specs Spack does not have a package for.') help='Show only specs Spack does not have a package for.')
subparser.add_argument( subparser.add_argument(
'-m', '-m', '--missing',
'--missing',
action='store_true', action='store_true',
dest='missing', dest='missing',
help='Show missing dependencies as well as installed specs.') help='Show missing dependencies as well as installed specs.')
subparser.add_argument('-M', subparser.add_argument('-M', '--only-missing',
'--only-missing',
action='store_true', action='store_true',
dest='only_missing', dest='only_missing',
help='Show only missing dependencies.') help='Show only missing dependencies.')
subparser.add_argument('-N', subparser.add_argument('-N', '--namespace',
'--namespace',
action='store_true', action='store_true',
help='Show fully qualified package names.') help='Show fully qualified package names.')
@ -187,7 +175,9 @@ def fmt(s):
print(hsh + spec.format(format_string, color=True) + '\n') print(hsh + spec.format(format_string, color=True) + '\n')
else: else:
raise ValueError("Invalid mode for display_specs: %s. Must be one of (paths, deps, short)." % mode) # NOQA: ignore=E501 raise ValueError(
"Invalid mode for display_specs: %s. Must be one of (paths,"
"deps, short)." % mode) # NOQA: ignore=E501
def find(parser, args): def find(parser, args):

91
lib/spack/spack/database.py
View File

@ -40,7 +40,6 @@
""" """
import os import os
import time
import socket import socket
import yaml import yaml
@ -56,6 +55,7 @@
from spack.error import SpackError from spack.error import SpackError
from spack.repository import UnknownPackageError from spack.repository import UnknownPackageError
# DB goes in this directory underneath the root # DB goes in this directory underneath the root
_db_dirname = '.spack-db' _db_dirname = '.spack-db'
@ -69,10 +69,12 @@
def _autospec(function): def _autospec(function):
"""Decorator that automatically converts the argument of a single-arg """Decorator that automatically converts the argument of a single-arg
function to a Spec.""" function to a Spec."""
def converter(self, spec_like, *args, **kwargs): def converter(self, spec_like, *args, **kwargs):
if not isinstance(spec_like, spack.spec.Spec): if not isinstance(spec_like, spack.spec.Spec):
spec_like = spack.spec.Spec(spec_like) spec_like = spack.spec.Spec(spec_like)
return function(self, spec_like, *args, **kwargs) return function(self, spec_like, *args, **kwargs)
return converter return converter
@ -92,6 +94,7 @@ class InstallRecord(object):
dependents left. dependents left.
""" """
def __init__(self, spec, path, installed, ref_count=0, explicit=False): def __init__(self, spec, path, installed, ref_count=0, explicit=False):
self.spec = spec self.spec = spec
self.path = str(path) self.path = str(path)
@ -100,16 +103,19 @@ def __init__(self, spec, path, installed, ref_count=0, explicit=False):
self.explicit = explicit self.explicit = explicit
def to_dict(self): def to_dict(self):
return { 'spec' : self.spec.to_node_dict(), return {
'path' : self.path, 'spec': self.spec.to_node_dict(),
'installed' : self.installed, 'path': self.path,
'ref_count' : self.ref_count, 'installed': self.installed,
'explicit' : self.explicit } 'ref_count': self.ref_count,
'explicit': self.explicit
}
@classmethod @classmethod
def from_dict(cls, spec, dictionary): def from_dict(cls, spec, dictionary):
d = dictionary d = dictionary
return InstallRecord(spec, d['path'], d['installed'], d['ref_count'], d.get('explicit', False)) return InstallRecord(spec, d['path'], d['installed'], d['ref_count'],
d.get('explicit', False))
class Database(object): class Database(object):
@ -157,17 +163,14 @@ def __init__(self, root, db_dir=None):
self.lock = Lock(self._lock_path) self.lock = Lock(self._lock_path)
self._data = {} self._data = {}
def write_transaction(self, timeout=_db_lock_timeout): def write_transaction(self, timeout=_db_lock_timeout):
"""Get a write lock context manager for use in a `with` block.""" """Get a write lock context manager for use in a `with` block."""
return WriteTransaction(self, self._read, self._write, timeout) return WriteTransaction(self, self._read, self._write, timeout)
def read_transaction(self, timeout=_db_lock_timeout): def read_transaction(self, timeout=_db_lock_timeout):
"""Get a read lock context manager for use in a `with` block.""" """Get a read lock context manager for use in a `with` block."""
return ReadTransaction(self, self._read, None, timeout) return ReadTransaction(self, self._read, None, timeout)
def _write_to_yaml(self, stream): def _write_to_yaml(self, stream):
"""Write out the databsae to a YAML file. """Write out the databsae to a YAML file.
@ -183,9 +186,9 @@ def _write_to_yaml(self, stream):
# different paths, it can't differentiate. # different paths, it can't differentiate.
# TODO: fix this before we support multiple install locations. # TODO: fix this before we support multiple install locations.
database = { database = {
'database' : { 'database': {
'installs' : installs, 'installs': installs,
'version' : str(_db_version) 'version': str(_db_version)
} }
} }
@ -194,15 +197,11 @@ def _write_to_yaml(self, stream):
except YAMLError as e: except YAMLError as e:
raise SpackYAMLError("error writing YAML database:", str(e)) raise SpackYAMLError("error writing YAML database:", str(e))
def _read_spec_from_yaml(self, hash_key, installs, parent_key=None): def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
"""Recursively construct a spec from a hash in a YAML database. """Recursively construct a spec from a hash in a YAML database.
Does not do any locking. Does not do any locking.
""" """
if hash_key not in installs:
parent = read_spec(installs[parent_key]['path'])
spec_dict = installs[hash_key]['spec'] spec_dict = installs[hash_key]['spec']
# Install records don't include hash with spec, so we add it in here # Install records don't include hash with spec, so we add it in here
@ -224,7 +223,6 @@ def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
spec._mark_concrete() spec._mark_concrete()
return spec return spec
def _read_from_yaml(self, stream): def _read_from_yaml(self, stream):
""" """
Fill database from YAML, do not maintain old data Fill database from YAML, do not maintain old data
@ -246,7 +244,8 @@ def _read_from_yaml(self, stream):
return return
def check(cond, msg): def check(cond, msg):
if not cond: raise CorruptDatabaseError(self._index_path, msg) if not cond:
raise CorruptDatabaseError(self._index_path, msg)
check('database' in yfile, "No 'database' attribute in YAML.") check('database' in yfile, "No 'database' attribute in YAML.")
@ -255,7 +254,6 @@ def check(cond, msg):
check('installs' in db, "No 'installs' in YAML DB.") check('installs' in db, "No 'installs' in YAML DB.")
check('version' in db, "No 'version' in YAML DB.") check('version' in db, "No 'version' in YAML DB.")
installs = db['installs'] installs = db['installs']
# TODO: better version checking semantics. # TODO: better version checking semantics.
@ -277,25 +275,25 @@ def check(cond, msg):
# hashes are the same. # hashes are the same.
spec_hash = spec.dag_hash() spec_hash = spec.dag_hash()
if not spec_hash == hash_key: if not spec_hash == hash_key:
tty.warn("Hash mismatch in database: %s -> spec with hash %s" tty.warn(
% (hash_key, spec_hash)) "Hash mismatch in database: %s -> spec with hash %s" %
(hash_key, spec_hash))
continue # TODO: is skipping the right thing to do? continue # TODO: is skipping the right thing to do?
# Insert the brand new spec in the database. Each # Insert the brand new spec in the database. Each
# spec has its own copies of its dependency specs. # spec has its own copies of its dependency specs.
# TODO: would a more immmutable spec implementation simplify this? # TODO: would a more immmutable spec implementation simplify
# this?
data[hash_key] = InstallRecord.from_dict(spec, rec) data[hash_key] = InstallRecord.from_dict(spec, rec)
except Exception as e: except Exception as e:
tty.warn("Invalid database reecord:", tty.warn("Invalid database reecord:",
"file: %s" % self._index_path, "file: %s" % self._index_path,
"hash: %s" % hash_key, "hash: %s" % hash_key, "cause: %s" % str(e))
"cause: %s" % str(e))
raise raise
self._data = data self._data = data
def reindex(self, directory_layout): def reindex(self, directory_layout):
"""Build database index from scratch based from a directory layout. """Build database index from scratch based from a directory layout.
@ -320,7 +318,6 @@ def reindex(self, directory_layout):
self._data = old_data self._data = old_data
raise raise
def _check_ref_counts(self): def _check_ref_counts(self):
"""Ensure consistency of reference counts in the DB. """Ensure consistency of reference counts in the DB.
@ -342,9 +339,8 @@ def _check_ref_counts(self):
found = rec.ref_count found = rec.ref_count
if not expected == found: if not expected == found:
raise AssertionError( raise AssertionError(
"Invalid ref_count: %s: %d (expected %d), in DB %s" "Invalid ref_count: %s: %d (expected %d), in DB %s" %
% (key, found, expected, self._index_path)) (key, found, expected, self._index_path))
def _write(self): def _write(self):
"""Write the in-memory database index to its file path. """Write the in-memory database index to its file path.
@ -366,7 +362,6 @@ def _write(self):
os.remove(temp_file) os.remove(temp_file)
raise raise
def _read(self): def _read(self):
"""Re-read Database from the data in the set location. """Re-read Database from the data in the set location.
@ -381,7 +376,6 @@ def _read(self):
# reindex() takes its own write lock, so no lock here. # reindex() takes its own write lock, so no lock here.
self.reindex(spack.install_layout) self.reindex(spack.install_layout)
def _add(self, spec, path, directory_layout=None, explicit=False): def _add(self, spec, path, directory_layout=None, explicit=False):
"""Add an install record for spec at path to the database. """Add an install record for spec at path to the database.
@ -404,11 +398,11 @@ def _add(self, spec, path, directory_layout=None, explicit=False):
rec.path = path rec.path = path
else: else:
self._data[key] = InstallRecord(spec, path, True, explicit=explicit) self._data[key] = InstallRecord(spec, path, True,
explicit=explicit)
for dep in spec.dependencies.values(): for dep in spec.dependencies.values():
self._increment_ref_count(dep, directory_layout) self._increment_ref_count(dep, directory_layout)
def _increment_ref_count(self, spec, directory_layout=None): def _increment_ref_count(self, spec, directory_layout=None):
"""Recursively examine dependencies and update their DB entries.""" """Recursively examine dependencies and update their DB entries."""
key = spec.dag_hash() key = spec.dag_hash()
@ -438,28 +432,25 @@ def add(self, spec, path, explicit=False):
with self.write_transaction(): with self.write_transaction():
self._add(spec, path, explicit=explicit) self._add(spec, path, explicit=explicit)
def _get_matching_spec_key(self, spec, **kwargs): def _get_matching_spec_key(self, spec, **kwargs):
"""Get the exact spec OR get a single spec that matches.""" """Get the exact spec OR get a single spec that matches."""
key = spec.dag_hash() key = spec.dag_hash()
if not key in self._data: if key not in self._data:
match = self.query_one(spec, **kwargs) match = self.query_one(spec, **kwargs)
if match: if match:
return match.dag_hash() return match.dag_hash()
raise KeyError("No such spec in database! %s" % spec) raise KeyError("No such spec in database! %s" % spec)
return key return key
@_autospec @_autospec
def get_record(self, spec, **kwargs): def get_record(self, spec, **kwargs):
key = self._get_matching_spec_key(spec, **kwargs) key = self._get_matching_spec_key(spec, **kwargs)
return self._data[key] return self._data[key]
def _decrement_ref_count(self, spec): def _decrement_ref_count(self, spec):
key = spec.dag_hash() key = spec.dag_hash()
if not key in self._data: if key not in self._data:
# TODO: print something here? DB is corrupt, but # TODO: print something here? DB is corrupt, but
# not much we can do. # not much we can do.
return return
@ -472,7 +463,6 @@ def _decrement_ref_count(self, spec):
for dep in spec.dependencies.values(): for dep in spec.dependencies.values():
self._decrement_ref_count(dep) self._decrement_ref_count(dep)
def _remove(self, spec): def _remove(self, spec):
"""Non-locking version of remove(); does real work. """Non-locking version of remove(); does real work.
""" """
@ -491,7 +481,6 @@ def _remove(self, spec):
# query spec was passed in. # query spec was passed in.
return rec.spec return rec.spec
@_autospec @_autospec
def remove(self, spec): def remove(self, spec):
"""Removes a spec from the database. To be called on uninstall. """Removes a spec from the database. To be called on uninstall.
@ -508,7 +497,6 @@ def remove(self, spec):
with self.write_transaction(): with self.write_transaction():
return self._remove(spec) return self._remove(spec)
@_autospec @_autospec
def installed_extensions_for(self, extendee_spec): def installed_extensions_for(self, extendee_spec):
""" """
@ -519,12 +507,11 @@ def installed_extensions_for(self, extendee_spec):
try: try:
if s.package.extends(extendee_spec): if s.package.extends(extendee_spec):
yield s.package yield s.package
except UnknownPackageError as e: except UnknownPackageError:
continue continue
# skips unknown packages # skips unknown packages
# TODO: conditional way to do this instead of catching exceptions # TODO: conditional way to do this instead of catching exceptions
def query(self, query_spec=any, known=any, installed=True, explicit=any): def query(self, query_spec=any, known=any, installed=True, explicit=any):
"""Run a query on the database. """Run a query on the database.
@ -567,14 +554,14 @@ def query(self, query_spec=any, known=any, installed=True, explicit=any):
continue continue
if explicit is not any and rec.explicit != explicit: if explicit is not any and rec.explicit != explicit:
continue continue
if known is not any and spack.repo.exists(rec.spec.name) != known: if known is not any and spack.repo.exists(
rec.spec.name) != known:
continue continue
if query_spec is any or rec.spec.satisfies(query_spec): if query_spec is any or rec.spec.satisfies(query_spec):
results.append(rec.spec) results.append(rec.spec)
return sorted(results) return sorted(results)
def query_one(self, query_spec, known=any, installed=True): def query_one(self, query_spec, known=any, installed=True):
"""Query for exactly one spec that matches the query spec. """Query for exactly one spec that matches the query spec.
@ -586,7 +573,6 @@ def query_one(self, query_spec, known=any, installed=True):
assert len(concrete_specs) <= 1 assert len(concrete_specs) <= 1
return concrete_specs[0] if concrete_specs else None return concrete_specs[0] if concrete_specs else None
def missing(self, spec): def missing(self, spec):
with self.read_transaction(): with self.read_transaction():
key = spec.dag_hash() key = spec.dag_hash()
@ -601,7 +587,10 @@ class _Transaction(object):
Timeout for lock is customizable. Timeout for lock is customizable.
""" """
def __init__(self, db, acquire_fn=None, release_fn=None,
def __init__(self, db,
acquire_fn=None,
release_fn=None,
timeout=_db_lock_timeout): timeout=_db_lock_timeout):
self._db = db self._db = db
self._timeout = timeout self._timeout = timeout
@ -636,11 +625,11 @@ def _exit(self):
class CorruptDatabaseError(SpackError): class CorruptDatabaseError(SpackError):
def __init__(self, path, msg=''): def __init__(self, path, msg=''):
super(CorruptDatabaseError, self).__init__( super(CorruptDatabaseError, self).__init__(
"Spack database is corrupt: %s. %s" %(path, msg)) "Spack database is corrupt: %s. %s" % (path, msg))
class InvalidDatabaseVersionError(SpackError): class InvalidDatabaseVersionError(SpackError):
def __init__(self, expected, found): def __init__(self, expected, found):
super(InvalidDatabaseVersionError, self).__init__( super(InvalidDatabaseVersionError, self).__init__(
"Expected database version %s but found version %s" "Expected database version %s but found version %s" %
% (expected, found)) (expected, found))

479
lib/spack/spack/spec.py
View File

File diff suppressed because it is too large Load Diff

16
var/spack/repos/builtin/packages/py-autopep8/package.py Normal file
View File

@ -0,0 +1,16 @@
from spack import *
class PyAutopep8(Package):
"""Automatic pep8 formatter"""
homepage = "https://github.com/hhatto/autopep8"
url = "https://github.com/hhatto/autopep8/archive/ver1.2.2.tar.gz"
version('1.2.2', 'def3d023fc9dfd1b7113602e965ad8e1')
extends('python')
depends_on('py-setuptools')
depends_on('py-pep8')
def install(self, spec, prefix):
python('setup.py', 'install', '--prefix=%s' % prefix)

15
var/spack/repos/builtin/packages/py-pep8/package.py Normal file
View File

@ -0,0 +1,15 @@
from spack import *
class PyPep8(Package):
"""python pep8 format checker"""
homepage = "https://github.com/PyCQA/pycodestyle"
url = "https://github.com/PyCQA/pycodestyle/archive/1.7.0.tar.gz"
version('1.7.0', '31070a3a6391928893cbf5fa523eb8d9')
extends('python')
depends_on('py-setuptools')
def install(self, spec, prefix):
python('setup.py', 'install', '--prefix=%s' % prefix)