zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #977 from trws/explicit-docs

Browse Source 
initial docs for find and flake8 cleanup
This commit is contained in:
Todd Gamblin 2016-05-18 15:20:23 -07:00
commit c53b1be8b4
6 changed files with 425 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/spack/docs/basic_usage.rst
View File

@ -246,6 +246,12 @@ Packages are divided into groups according to their architecture and
compiler. Within each group, Spack tries to keep the view simple, and
only shows the version of installed packages.
``spack find`` can filter the package list based on the package name, spec, or
a number of properties of their installation status. For example, missing
dependencies of a spec can be shown with ``-m``, packages which were
explicitly installed with ``spack install <package>`` can be singled out with
``-e`` and those which have been pulled in only as dependencies with ``-E``.
In some cases, there may be different configurations of the *same*
version of a package installed. For example, there are two
installations of of ``libdwarf@20130729`` above. We can look at them

40
lib/spack/spack/cmd/find.py
View File

@ -37,71 +37,59 @@
def setup_parser(subparser):
format_group = subparser.add_mutually_exclusive_group()
format_group.add_argument('-s',
'--short',
format_group.add_argument('-s', '--short',
action='store_const',
dest='mode',
const='short',
help='Show only specs (default)')
format_group.add_argument('-p',
'--paths',
format_group.add_argument('-p', '--paths',
action='store_const',
dest='mode',
const='paths',
help='Show paths to package install directories')
format_group.add_argument(
'-d',
'--deps',
'-d', '--deps',
action='store_const',
dest='mode',
const='deps',
help='Show full dependency DAG of installed packages')
subparser.add_argument('-l',
'--long',
subparser.add_argument('-l', '--long',
action='store_true',
dest='long',
help='Show dependency hashes as well as versions.')
subparser.add_argument('-L',
'--very-long',
subparser.add_argument('-L', '--very-long',
action='store_true',
dest='very_long',
help='Show dependency hashes as well as versions.')
subparser.add_argument('-f',
'--show-flags',
subparser.add_argument('-f', '--show-flags',
action='store_true',
dest='show_flags',
help='Show spec compiler flags.')
subparser.add_argument(
'-e',
'--explicit',
'-e', '--explicit',
action='store_true',
help='Show only specs that were installed explicitly')
subparser.add_argument(
'-E',
'--implicit',
'-E', '--implicit',
action='store_true',
help='Show only specs that were installed as dependencies')
subparser.add_argument(
'-u',
'--unknown',
'-u', '--unknown',
action='store_true',
dest='unknown',
help='Show only specs Spack does not have a package for.')
subparser.add_argument(
'-m',
'--missing',
'-m', '--missing',
action='store_true',
dest='missing',
help='Show missing dependencies as well as installed specs.')
subparser.add_argument('-M',
'--only-missing',
subparser.add_argument('-M', '--only-missing',
action='store_true',
dest='only_missing',
help='Show only missing dependencies.')
subparser.add_argument('-N',
'--namespace',
subparser.add_argument('-N', '--namespace',
action='store_true',
help='Show fully qualified package names.')
@ -187,7 +175,9 @@ def fmt(s):
print(hsh + spec.format(format_string, color=True) + '\n')
else:
raise ValueError("Invalid mode for display_specs: %s. Must be one of (paths, deps, short)." % mode) # NOQA: ignore=E501
raise ValueError(
"Invalid mode for display_specs: %s. Must be one of (paths,"
"deps, short)." % mode) # NOQA: ignore=E501
def find(parser, args):

99
lib/spack/spack/database.py
View File

@ -40,7 +40,6 @@
"""
import os
import time
import socket
import yaml
@ -56,6 +55,7 @@
from spack.error import SpackError
from spack.repository import UnknownPackageError
# DB goes in this directory underneath the root
_db_dirname = '.spack-db'
@ -69,10 +69,12 @@
def _autospec(function):
"""Decorator that automatically converts the argument of a single-arg
function to a Spec."""
def converter(self, spec_like, *args, **kwargs):
if not isinstance(spec_like, spack.spec.Spec):
spec_like = spack.spec.Spec(spec_like)
return function(self, spec_like, *args, **kwargs)
return converter
@ -92,6 +94,7 @@ class InstallRecord(object):
dependents left.
"""
def __init__(self, spec, path, installed, ref_count=0, explicit=False):
self.spec = spec
self.path = str(path)
@ -100,16 +103,19 @@ def __init__(self, spec, path, installed, ref_count=0, explicit=False):
self.explicit = explicit
def to_dict(self):
return { 'spec' : self.spec.to_node_dict(),
'path' : self.path,
'installed' : self.installed,
'ref_count' : self.ref_count,
'explicit' : self.explicit }
return {
'spec': self.spec.to_node_dict(),
'path': self.path,
'installed': self.installed,
'ref_count': self.ref_count,
'explicit': self.explicit
}
@classmethod
def from_dict(cls, spec, dictionary):
d = dictionary
return InstallRecord(spec, d['path'], d['installed'], d['ref_count'], d.get('explicit', False))
return InstallRecord(spec, d['path'], d['installed'], d['ref_count'],
d.get('explicit', False))
class Database(object):
@ -144,7 +150,7 @@ def __init__(self, root, db_dir=None):
# Set up layout of database files within the db dir
self._index_path = join_path(self._db_dir, 'index.yaml')
self._lock_path = join_path(self._db_dir, 'lock')
self._lock_path = join_path(self._db_dir, 'lock')
# Create needed directories and files
if not os.path.exists(self._db_dir):
@ -157,17 +163,14 @@ def __init__(self, root, db_dir=None):
self.lock = Lock(self._lock_path)
self._data = {}
def write_transaction(self, timeout=_db_lock_timeout):
"""Get a write lock context manager for use in a `with` block."""
return WriteTransaction(self, self._read, self._write, timeout)
def read_transaction(self, timeout=_db_lock_timeout):
"""Get a read lock context manager for use in a `with` block."""
return ReadTransaction(self, self._read, None, timeout)
def _write_to_yaml(self, stream):
"""Write out the databsae to a YAML file.
@ -183,9 +186,9 @@ def _write_to_yaml(self, stream):
# different paths, it can't differentiate.
# TODO: fix this before we support multiple install locations.
database = {
'database' : {
'installs' : installs,
'version' : str(_db_version)
'database': {
'installs': installs,
'version': str(_db_version)
}
}
@ -194,15 +197,11 @@ def _write_to_yaml(self, stream):
except YAMLError as e:
raise SpackYAMLError("error writing YAML database:", str(e))
def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
"""Recursively construct a spec from a hash in a YAML database.
Does not do any locking.
"""
if hash_key not in installs:
parent = read_spec(installs[parent_key]['path'])
spec_dict = installs[hash_key]['spec']
# Install records don't include hash with spec, so we add it in here
@ -224,7 +223,6 @@ def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
spec._mark_concrete()
return spec
def _read_from_yaml(self, stream):
"""
Fill database from YAML, do not maintain old data
@ -246,15 +244,15 @@ def _read_from_yaml(self, stream):
return
def check(cond, msg):
if not cond: raise CorruptDatabaseError(self._index_path, msg)
if not cond:
raise CorruptDatabaseError(self._index_path, msg)
check('database' in yfile, "No 'database' attribute in YAML.")
# High-level file checks
db = yfile['database']
check('installs' in db, "No 'installs' in YAML DB.")
check('version' in db, "No 'version' in YAML DB.")
check('version' in db, "No 'version' in YAML DB.")
installs = db['installs']
@ -277,25 +275,25 @@ def check(cond, msg):
# hashes are the same.
spec_hash = spec.dag_hash()
if not spec_hash == hash_key:
tty.warn("Hash mismatch in database: %s -> spec with hash %s"
% (hash_key, spec_hash))
continue # TODO: is skipping the right thing to do?
tty.warn(
"Hash mismatch in database: %s -> spec with hash %s" %
(hash_key, spec_hash))
continue # TODO: is skipping the right thing to do?
# Insert the brand new spec in the database. Each
# spec has its own copies of its dependency specs.
# TODO: would a more immmutable spec implementation simplify this?
# TODO: would a more immmutable spec implementation simplify
# this?
data[hash_key] = InstallRecord.from_dict(spec, rec)
except Exception as e:
tty.warn("Invalid database reecord:",
"file: %s" % self._index_path,
"hash: %s" % hash_key,
"cause: %s" % str(e))
"hash: %s" % hash_key, "cause: %s" % str(e))
raise
self._data = data
def reindex(self, directory_layout):
"""Build database index from scratch based from a directory layout.
@ -320,7 +318,6 @@ def reindex(self, directory_layout):
self._data = old_data
raise
def _check_ref_counts(self):
"""Ensure consistency of reference counts in the DB.
@ -342,9 +339,8 @@ def _check_ref_counts(self):
found = rec.ref_count
if not expected == found:
raise AssertionError(
"Invalid ref_count: %s: %d (expected %d), in DB %s"
% (key, found, expected, self._index_path))
"Invalid ref_count: %s: %d (expected %d), in DB %s" %
(key, found, expected, self._index_path))
def _write(self):
"""Write the in-memory database index to its file path.
@ -366,7 +362,6 @@ def _write(self):
os.remove(temp_file)
raise
def _read(self):
"""Re-read Database from the data in the set location.
@ -381,7 +376,6 @@ def _read(self):
# reindex() takes its own write lock, so no lock here.
self.reindex(spack.install_layout)
def _add(self, spec, path, directory_layout=None, explicit=False):
"""Add an install record for spec at path to the database.
@ -404,11 +398,11 @@ def _add(self, spec, path, directory_layout=None, explicit=False):
rec.path = path
else:
self._data[key] = InstallRecord(spec, path, True, explicit=explicit)
self._data[key] = InstallRecord(spec, path, True,
explicit=explicit)
for dep in spec.dependencies.values():
self._increment_ref_count(dep, directory_layout)
def _increment_ref_count(self, spec, directory_layout=None):
"""Recursively examine dependencies and update their DB entries."""
key = spec.dag_hash()
@ -438,28 +432,25 @@ def add(self, spec, path, explicit=False):
with self.write_transaction():
self._add(spec, path, explicit=explicit)
def _get_matching_spec_key(self, spec, **kwargs):
"""Get the exact spec OR get a single spec that matches."""
key = spec.dag_hash()
if not key in self._data:
if key not in self._data:
match = self.query_one(spec, **kwargs)
if match:
return match.dag_hash()
raise KeyError("No such spec in database! %s" % spec)
return key
@_autospec
def get_record(self, spec, **kwargs):
key = self._get_matching_spec_key(spec, **kwargs)
return self._data[key]
def _decrement_ref_count(self, spec):
key = spec.dag_hash()
if not key in self._data:
if key not in self._data:
# TODO: print something here? DB is corrupt, but
# not much we can do.
return
@ -472,7 +463,6 @@ def _decrement_ref_count(self, spec):
for dep in spec.dependencies.values():
self._decrement_ref_count(dep)
def _remove(self, spec):
"""Non-locking version of remove(); does real work.
"""
@ -491,7 +481,6 @@ def _remove(self, spec):
# query spec was passed in.
return rec.spec
@_autospec
def remove(self, spec):
"""Removes a spec from the database. To be called on uninstall.
@ -508,7 +497,6 @@ def remove(self, spec):
with self.write_transaction():
return self._remove(spec)
@_autospec
def installed_extensions_for(self, extendee_spec):
"""
@ -519,12 +507,11 @@ def installed_extensions_for(self, extendee_spec):
try:
if s.package.extends(extendee_spec):
yield s.package
except UnknownPackageError as e:
except UnknownPackageError:
continue
# skips unknown packages
# TODO: conditional way to do this instead of catching exceptions
def query(self, query_spec=any, known=any, installed=True, explicit=any):
"""Run a query on the database.
@ -567,14 +554,14 @@ def query(self, query_spec=any, known=any, installed=True, explicit=any):
continue
if explicit is not any and rec.explicit != explicit:
continue
if known is not any and spack.repo.exists(rec.spec.name) != known:
if known is not any and spack.repo.exists(
rec.spec.name) != known:
continue
if query_spec is any or rec.spec.satisfies(query_spec):
results.append(rec.spec)
return sorted(results)
def query_one(self, query_spec, known=any, installed=True):
"""Query for exactly one spec that matches the query spec.
@ -586,10 +573,9 @@ def query_one(self, query_spec, known=any, installed=True):
assert len(concrete_specs) <= 1
return concrete_specs[0] if concrete_specs else None
def missing(self, spec):
with self.read_transaction():
key = spec.dag_hash()
key = spec.dag_hash()
return key in self._data and not self._data[key].installed
@ -601,7 +587,10 @@ class _Transaction(object):
Timeout for lock is customizable.
"""
def __init__(self, db, acquire_fn=None, release_fn=None,
def __init__(self, db,
acquire_fn=None,
release_fn=None,
timeout=_db_lock_timeout):
self._db = db
self._timeout = timeout
@ -636,11 +625,11 @@ def _exit(self):
class CorruptDatabaseError(SpackError):
def __init__(self, path, msg=''):
super(CorruptDatabaseError, self).__init__(
"Spack database is corrupt: %s. %s" %(path, msg))
"Spack database is corrupt: %s. %s" % (path, msg))
class InvalidDatabaseVersionError(SpackError):
def __init__(self, expected, found):
super(InvalidDatabaseVersionError, self).__init__(
"Expected database version %s but found version %s"
% (expected, found))
"Expected database version %s but found version %s" %
(expected, found))

629
lib/spack/spack/spec.py
View File

File diff suppressed because it is too large Load Diff

16
var/spack/repos/builtin/packages/py-autopep8/package.py Normal file
View File

@ -0,0 +1,16 @@
from spack import *
class PyAutopep8(Package):
"""Automatic pep8 formatter"""
homepage = "https://github.com/hhatto/autopep8"
url = "https://github.com/hhatto/autopep8/archive/ver1.2.2.tar.gz"
version('1.2.2', 'def3d023fc9dfd1b7113602e965ad8e1')
extends('python')
depends_on('py-setuptools')
depends_on('py-pep8')
def install(self, spec, prefix):
python('setup.py', 'install', '--prefix=%s' % prefix)

15
var/spack/repos/builtin/packages/py-pep8/package.py Normal file
View File

@ -0,0 +1,15 @@
from spack import *
class PyPep8(Package):
"""python pep8 format checker"""
homepage = "https://github.com/PyCQA/pycodestyle"
url = "https://github.com/PyCQA/pycodestyle/archive/1.7.0.tar.gz"
version('1.7.0', '31070a3a6391928893cbf5fa523eb8d9')
extends('python')
depends_on('py-setuptools')
def install(self, spec, prefix):
python('setup.py', 'install', '--prefix=%s' % prefix)