Eliminated all calls that relied on finding all packages in the opt directory

Replaced them all with references to the database

Implemented caching in the database. The database now only re-reads data
if the database file exists and was changed since this file last wrote to it.

Added the installed_db field to the spack instance

Left the call to all_specs from testdirectory_layout.py for now.

lib/spack/spack/__init__.py

@@ -53,6 +53,12 @@
 packages_path = join_path(var_path, "packages")
 db = PackageDB(packages_path)
 
+#
+# Set up the installed packages database
+#
+from spack.database import Database
+installed_db = Database(install_path)
+
 #
 # Paths to mock files for testing.
 #

lib/spack/spack/cmd/__init__.py

@@ -124,7 +124,7 @@ def elide_list(line_list, max_num=10):
 
 
 def disambiguate_spec(spec):
-    matching_specs = spack.db.get_installed(spec)
+    matching_specs = spack.installed_db.get_installed(spec)
     if not matching_specs:
         tty.die("Spec '%s' matches no installed packages." % spec)
 

lib/spack/spack/cmd/deactivate.py

@@ -54,7 +54,7 @@ def deactivate(parser, args):
     if args.all:
         if pkg.extendable:
             tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec)
-            ext_pkgs = spack.db.installed_extensions_for(spec)
+            ext_pkgs = spack.installed_db.installed_extensions_for(spec)
 
             for ext_pkg in ext_pkgs:
                 ext_pkg.spec.normalize()

lib/spack/spack/cmd/extensions.py

@@ -80,7 +80,7 @@ def extensions(parser, args):
     colify(ext.name for ext in extensions)
 
     # List specs of installed extensions.
-    installed  = [s.spec for s in spack.db.installed_extensions_for(spec)]
+    installed  = [s.spec for s in spack.installed_db.installed_extensions_for(spec)]
     print
     if not installed:
         tty.msg("None installed.")

lib/spack/spack/cmd/find.py

@@ -138,9 +138,9 @@ def find(parser, args):
 
     # Get all the specs the user asked for
     if not query_specs:
-        specs = set(spack.db.installed_package_specs())
+        specs = set(spack.installed_db.installed_package_specs())
     else:
-        results = [set(spack.db.get_installed(qs)) for qs in query_specs]
+        results = [set(spack.installed_db.get_installed(qs)) for qs in query_specs]
         specs = set.union(*results)
 
     if not args.mode:

lib/spack/spack/cmd/module.py

@@ -65,7 +65,7 @@ def module_find(mtype, spec_array):
         tty.die("You can only pass one spec.")
     spec = specs[0]
 
-    specs = [s for s in spack.db.installed_package_specs() if s.satisfies(spec)]
+    specs = [s for s in spack.installed_db.installed_package_specs() if s.satisfies(spec)]
     if len(specs) == 0:
         tty.die("No installed packages match spec %s" % spec)
 
@@ -86,7 +86,7 @@ def module_find(mtype, spec_array):
 def module_refresh():
     """Regenerate all module files for installed packages known to
        spack (some packages may no longer exist)."""
-    specs = [s for s in spack.db.installed_known_package_specs()]
+    specs = [s for s in spack.installed_db.installed_known_package_specs()]
 
     for name, cls in module_types.items():
         tty.msg("Regenerating %s module files." % name)

lib/spack/spack/cmd/uninstall.py

@@ -59,7 +59,7 @@ def uninstall(parser, args):
     # Fail and ask user to be unambiguous if it doesn't
     pkgs = []
     for spec in specs:
-        matching_specs = spack.db.get_installed(spec)
+        matching_specs = spack.installed_db.get_installed(spec)
         if not args.all and len(matching_specs) > 1:
             tty.error("%s matches multiple packages:" % spec)
             print

lib/spack/spack/database.py

@@ -28,6 +28,9 @@
 import glob
 import imp
 
+import time
+import copy
+
 from external import yaml
 from external.yaml.error import MarkedYAMLError
 
@@ -43,8 +46,18 @@
 from spack.util.naming import mod_to_class, validate_module_name
 
 
+def _autospec(function):
+    """Decorator that automatically converts the argument of a single-arg
+       function to a Spec."""
+    def converter(self, spec_like, **kwargs):
+        if not isinstance(spec_like, spack.spec.Spec):
+            spec_like = spack.spec.Spec(spec_like)
+        return function(self, spec_like, **kwargs)
+    return converter
+
+
 class Database(object):
-    def __init__(self,file_name="specDB.yaml"):
+    def __init__(self,root,file_name="specDB.yaml"):
         """
         Create an empty Database
         Location defaults to root/specDB.yaml
@@ -53,13 +66,16 @@ def __init__(self,file_name="specDB.yaml"):
         path: the path to the install of that package
         dep_hash: a hash of the dependence DAG for that package
         """
+        self.root = root
         self.file_name = file_name
+        self.file_path = join_path(self.root,self.file_name)
         self.data = []
+        self.last_write_time = 0
 
 
     def from_yaml(self,stream):
         """
-        Fill database from YAML
+        Fill database from YAML, do not maintain old data
         Translate the spec portions from node-dict form to spec from
         """
         try:
@@ -70,6 +86,7 @@ def from_yaml(self,stream):
         if file==None:
             return
 
+        self.data = []
         for sp in file['database']:
             spec = Spec.from_node_dict(sp['spec'])
             path = sp['path']
@@ -78,19 +95,14 @@ def from_yaml(self,stream):
             self.data.append(db_entry)
 
 
-    @staticmethod
+    def read_database(self):
-    def read_database(root):
+        """Reread Database from the data in the set location"""
-        """Create a Database from the data in the standard location"""
+        if os.path.isfile(self.file_path):
-        database = Database()
+            with open(self.file_path,'r') as f:
-        full_path = join_path(root,database.file_name)
+                self.from_yaml(f)
-        if os.path.isfile(full_path):
-            with open(full_path,'r') as f:
-                database.from_yaml(f)
         else:
-            with open(full_path,'w+') as f:
+            #The file doesn't exist, construct empty data.
-                database.from_yaml(f)
+            self.data = []
-
-        return database
 
 
     def write_database_to_yaml(self,stream):
@@ -110,48 +122,104 @@ def write_database_to_yaml(self,stream):
                          stream=stream, default_flow_style=False)
 
 
-    def write(self,root):
+    def write(self):
         """Write the database to the standard location"""
-        full_path = join_path(root,self.file_name)
+        #creates file if necessary
-        #test for file existence
+        with open(self.file_path,'w') as f:
-        with open(full_path,'w') as f:
+            self.last_write_time = int(time.time())
             self.write_database_to_yaml(f)
 
 
-    @staticmethod
+    def is_dirty(self):
-    def add(root, spec, path):
+        """
-        """Read the database from the standard location
+        Returns true iff the database file exists
+        and was most recently written to by another spack instance.
+        """
+        return (os.path.isfile(self.file_path) and (os.path.getmtime(self.file_path) > self.last_write_time))
+
+
+#    @_autospec
+    def add(self, spec, path):
+        """Re-read the database from the set location if data is dirty
         Add the specified entry as a dict
         Write the database back to memory
-
-        TODO: Caching databases
         """
-        database = Database.read_database(root)
+        if self.is_dirty():
+            self.read_database()
 
         sph = {}
         sph['spec']=spec
         sph['path']=path
         sph['hash']=spec.dag_hash()
 
-        database.data.append(sph)
+        self.data.append(sph)
 
-        database.write(root)
+        self.write()
 
 
-    @staticmethod
+    @_autospec
-    def remove(root, spec):
+    def remove(self, spec):
         """
-        Reads the database from the standard location
+        Re-reads the database from the set location if data is dirty
         Searches for and removes the specified spec
         Writes the database back to memory
-
-        TODO: Caching databases
         """
-        database = Database.read_database(root)
+        if self.is_dirty():
+            self.read_database()
 
-        for sp in database.data:
+        for sp in self.data:
-            #This requires specs w/o dependencies, is that sustainable?
+
-            if sp['spec'] == spec:
+            if sp['hash'] == spec.dag_hash() and sp['spec'] == Spec.from_node_dict(spec.to_node_dict()):
-                database.data.remove(sp)
+                self.data.remove(sp)
+
+        self.write()
+
+
+    @_autospec
+    def get_installed(self, spec):
+        """
+        Get all the installed specs that satisfy the provided spec constraint
+        """
+        return [s for s in self.installed_package_specs() if s.satisfies(spec)]
+
+
+    @_autospec
+    def installed_extensions_for(self, extendee_spec):
+        """
+        Return the specs of all packages that extend
+        the given spec
+        """
+        for s in self.installed_package_specs():
+            try:
+                if s.package.extends(extendee_spec):
+                    yield s.package
+            except UnknownPackageError, e:
+                continue
+            #skips unknown packages
+            #TODO: conditional way to do this instead of catching exceptions
+
+
+    def installed_package_specs(self):
+        """
+        Read installed package names from the database
+        and return their specs
+        """
+        if self.is_dirty():
+            self.read_database()
+
+        installed = []
+        for sph in self.data:
+            sph['spec'].normalize()
+            sph['spec'].concretize()
+            installed.append(sph['spec'])
+        return installed
+
+
+    def installed_known_package_specs(self):
+        """
+        Read installed package names from the database.
+        Return only the specs for which the package is known
+        to this version of spack
+        """
+        return [s for s in self.installed_package_specs() if spack.db.exists(s.name)]
 
-        database.write(root)

lib/spack/spack/directory_layout.py

@@ -153,7 +153,6 @@ def remove_install_directory(self, spec):
                 os.rmdir(path)
             path = os.path.dirname(path)
 
-        Database.remove(self.root,spec)
 
 
 class YamlDirectoryLayout(DirectoryLayout):
@@ -266,11 +265,6 @@ def create_install_directory(self, spec):
         self.write_spec(spec, spec_file_path)
 
 
-    def add_to_database(self, spec):
-        """Simply adds a spec to the database"""
-        Database.add(self.root, spec, self.path_for_spec(spec))
-
-
     @memoized
     def all_specs(self):
         if not os.path.isdir(self.root):

lib/spack/spack/package.py

@@ -565,7 +565,7 @@ def installed_dependents(self):
         """Return a list of the specs of all installed packages that depend
            on this one."""
         dependents = []
-        for spec in spack.db.installed_package_specs():
+        for spec in spack.installed_db.installed_package_specs():
             if self.name == spec.name:
                 continue
             for dep in spec.traverse():
@@ -601,7 +601,7 @@ def url_version(self, version):
     def remove_prefix(self):
         """Removes the prefix for a package along with any empty parent directories."""
         spack.install_layout.remove_install_directory(self.spec)
-
+        spack.installed_db.remove(self.spec)
 
     def do_fetch(self):
         """Creates a stage directory and downloads the taball for this package.
@@ -818,7 +818,7 @@ def real_work():
                     install(log_path, log_install_path)
 
                 #Update the database once we know install successful
-                spack.install_layout.add_to_database(self.spec)
+                spack.installed_db.add(self.spec, spack.install_layout.path_for_spec(self.spec))
 
                 # On successful install, remove the stage.
                 if not keep_stage:

lib/spack/spack/packages.py

@@ -95,12 +95,6 @@ def purge(self):
         self.instances.clear()
 
 
-    @_autospec
-    def get_installed(self, spec):
-        """Get all the installed specs that satisfy the provided spec constraint."""
-        return [s for s in self.installed_package_specs() if s.satisfies(spec)]
-
-
     @_autospec
     def providers_for(self, vpkg_spec):
         if self.provider_index is None:
@@ -117,19 +111,6 @@ def extensions_for(self, extendee_spec):
         return [p for p in self.all_packages() if p.extends(extendee_spec)]
 
 
-    @_autospec
-    def installed_extensions_for(self, extendee_spec):
-        for s in self.installed_package_specs():
-            try:
-                if s.package.extends(extendee_spec):
-                    yield s.package
-            except UnknownPackageError, e:
-                # Skip packages we know nothing about
-                continue
-                # TODO: add some conditional way to do this instead of
-                # catching exceptions.
-
-
     def dirname_for_package_name(self, pkg_name):
         """Get the directory name for a particular package.  This is the
            directory that contains its package.py file."""
@@ -150,29 +131,6 @@ def filename_for_package_name(self, pkg_name):
         return join_path(pkg_dir, _package_file_name)
 
 
-    def installed_package_specs(self):
-        """Read installed package names straight from the install directory
-           layout.
-        """
-        # Get specs from the directory layout but ensure that they're
-        # all normalized properly.
-        installed = []
-        for spec in spack.install_layout.all_specs():
-            spec.normalize()
-            installed.append(spec)
-        return installed
-
-
-    def installed_known_package_specs(self):
-        """Read installed package names straight from the install
-           directory layout, but return only specs for which the
-           package is known to this version of spack.
-        """
-        for spec in spack.install_layout.all_specs():
-            if self.exists(spec.name):
-                yield spec
-
-
     @memoized
     def all_package_names(self):
         """Generator function for all packages.  This looks for