Integrate namespace attribute into spec, spec DAG, spec YAML.

lib/spack/spack/config.py

@@ -115,7 +115,7 @@ def __init__(self, name, filename, merge, strip):
         self.result_dict = {}
         _config_sections[name] = self
 
-_ConfigCategory('config',    'config.yaml',    True, False)
+_ConfigCategory('repos',     'repos.yaml',     True,  True)
 _ConfigCategory('compilers', 'compilers.yaml', True,  True)
 _ConfigCategory('mirrors',   'mirrors.yaml',   True,  True)
 _ConfigCategory('view',      'views.yaml',     True,  True)
@@ -212,7 +212,7 @@ def substitute_spack_prefix(path):
     return path.replace('$spack', spack.prefix)
 
 
-def get_config(category='config'):
+def get_config(category):
     """Get the confguration tree for a category.
 
     Strips off the top-level category entry from the dict
@@ -233,6 +233,10 @@ def get_config(category='config'):
                 continue
             result = result[category.name]
 
+            # ignore empty sections for easy commenting of single-line configs.
+            if result is None:
+                continue
+
         category.files_read_from.insert(0, path)
         if category.merge:
             category.result_dict = _merge_yaml(category.result_dict, result)
@@ -266,12 +270,18 @@ def get_compilers_config(arch=None):
 
 
 def get_repos_config():
-    config = get_config()
+    repo_list = get_config('repos')
-    if 'repos' not in config:
+    if repo_list is None:
         return []
 
-    repo_list = config['repos']
+    if not isinstance(repo_list, list):
-    return [substitute_spack_prefix(repo) for repo in repo_list]
+        tty.die("Bad repository configuration. 'repos' element does not contain a list.")
+
+    def expand_repo_path(path):
+        path = substitute_spack_prefix(path)
+        path = os.path.expanduser(path)
+        return path
+    return [expand_repo_path(repo) for repo in repo_list]
 
 
 def get_mirror_config():

lib/spack/spack/database.py

@@ -211,6 +211,8 @@ def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
             child = self._read_spec_from_yaml(dep_hash, installs, hash_key)
             spec._add_dependency(child)
 
+        spec._normal = True
+        spec._concrete = True
         return spec
 
 

lib/spack/spack/directory_layout.py

@@ -211,9 +211,12 @@ def read_spec(self, path):
         with open(path) as f:
             spec = Spec.from_yaml(f)
 
-        # Specs read from actual installations are always concrete
+        # Specs read from actual installs are always concrete, so mark
-        spec._normal = True
+        # all parts of the spec.
-        spec._concrete = True
+        for s in spec.traverse():
+            s._normal = True
+            s._concrete = True
+
         return spec
 
 

lib/spack/spack/repository.py

@@ -238,6 +238,15 @@ def get(self, spec, new=False):
 
            Raises UnknownPackageError if not found.
         """
+        # if the spec has a fully qualified namespace, we grab it
+        # directly and ignore overlay precedence.
+        if spec.namespace:
+            fullspace = '%s.%s' % (self.super_namespace, spec.namespace)
+            if not fullspace in self.by_namespace:
+                raise UnknownPackageError(
+                    "No configured repository contains package %s." % spec.fullname)
+            return self.by_namespace[fullspace].get(spec)
+        else:
             return self.repo_for_pkg(spec.name).get(spec)
 
 
@@ -454,19 +463,23 @@ def get(self, spec, new=False):
         if spec.virtual:
             raise UnknownPackageError(spec.name)
 
-        if new and spec in self._instances:
+        if spec.namespace and spec.namespace != self.namespace:
-            del self._instances[spec]
+            raise UnknownPackageError("Repository %s does not contain package %s."
+                                      % (self.namespace, spec.fullname))
 
-        if not spec in self._instances:
+        if new or spec not in self._instances:
             PackageClass = self._get_pkg_class(spec.name)
             try:
-                copy = spec.copy()
+                package = PackageClass(spec.copy())
-                self._instances[copy] = PackageClass(copy)
+                self._instances[spec] = package
+                return package
+
             except Exception, e:
                 if spack.debug:
                     sys.excepthook(*sys.exc_info())
-                raise FailedConstructorError(spec.name, *sys.exc_info())
+                raise FailedConstructorError(spec.fullname, *sys.exc_info())
 
+        else:
             return self._instances[spec]
 
 

lib/spack/spack/spec.py

@@ -465,6 +465,13 @@ def _add_dependency(self, spec):
         self.dependencies[spec.name] = spec
         spec.dependents[self.name] = self
 
+    #
+    # Public interface
+    #
+    @property
+    def fullname(self):
+        return '%s.%s' % (self.namespace, self.name) if self.namespace else self.name
+
 
     @property
     def root(self):
@@ -518,6 +525,7 @@ def concrete(self):
             return True
 
         self._concrete = bool(not self.virtual
+                              and self.namespace is not None
                               and self.versions.concrete
                               and self.variants.concrete
                               and self.architecture
@@ -658,6 +666,12 @@ def to_node_dict(self):
             'dependencies' : dict((d, self.dependencies[d].dag_hash())
                                   for d in sorted(self.dependencies))
         }
+
+        # Older concrete specs do not have a namespace.  Omit for
+        # consistent hashing.
+        if not self.concrete or self.namespace:
+            d['namespace'] = self.namespace
+
         if self.compiler:
             d.update(self.compiler.to_dict())
         else:
@@ -682,6 +696,7 @@ def from_node_dict(node):
         node = node[name]
 
         spec = Spec(name)
+        spec.namespace = node.get('namespace', None)
         spec.versions = VersionList.from_dict(node)
         spec.architecture = node['arch']
 
@@ -834,7 +849,20 @@ def concretize(self):
             changed = any(changes)
             force=True
 
-        self._concrete = True
+        for s in self.traverse():
+            # After concretizing, assign namespaces to anything left.
+            # Note that this doesn't count as a "change".  The repository
+            # configuration is constant throughout a spack run, and
+            # normalize and concretize evaluate Packages using Repo.get(),
+            # which respects precedence.  So, a namespace assignment isn't
+            # changing how a package name would have been interpreted and
+            # we can do it as late as possible to allow as much
+            # compatibility across repositories as possible.
+            if s.namespace is None:
+                s.namespace = spack.repo.repo_for_pkg(s.name).namespace
+
+            # Mark everything in the spec as concrete, as well.
+            s._concrete = True
 
 
     def concretized(self):
@@ -909,7 +937,7 @@ def _evaluate_dependency_conditions(self, name):
         the dependency.  If no conditions are True (and we don't
         depend on it), return None.
         """
-        pkg = spack.repo.get(self.name)
+        pkg = spack.repo.get(self.fullname)
         conditions = pkg.dependencies[name]
 
         # evaluate when specs to figure out constraints on the dependency.
@@ -1037,7 +1065,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index):
         any_change = False
         changed = True
 
-        pkg = spack.repo.get(self.name)
+        pkg = spack.repo.get(self.fullname)
         while changed:
             changed = False
             for dep_name in pkg.dependencies:
@@ -1058,18 +1086,17 @@ def normalize(self, force=False):
            the root, and ONLY the ones that were explicitly provided are there.
            Normalization turns a partial flat spec into a DAG, where:
 
-           1. ALL dependencies of the root package are in the DAG.
+           1. Known dependencies of the root package are in the DAG.
-           2. Each node's dependencies dict only contains its direct deps.
+           2. Each node's dependencies dict only contains its known direct deps.
            3. There is only ONE unique spec for each package in the DAG.
 
               * This includes virtual packages.  If there a non-virtual
                 package that provides a virtual package that is in the spec,
                 then we replace the virtual package with the non-virtual one.
 
-           4. The spec DAG matches package DAG, including default variant values.
-
            TODO: normalize should probably implement some form of cycle detection,
            to ensure that the spec is actually a DAG.
+
         """
         if self._normal and not force:
             return False
@@ -1115,7 +1142,7 @@ def validate_names(self):
         for spec in self.traverse():
             # Don't get a package for a virtual name.
             if not spec.virtual:
-                spack.repo.get(spec.name)
+                spack.repo.get(spec.fullname)
 
             # validate compiler in addition to the package name.
             if spec.compiler:
@@ -1138,6 +1165,10 @@ def constrain(self, other, deps=True):
         if not self.name == other.name:
             raise UnsatisfiableSpecNameError(self.name, other.name)
 
+        if other.namespace is not None:
+            if self.namespace is not None and other.namespace != self.namespace:
+                raise UnsatisfiableSpecNameError(self.fullname, other.fullname)
+
         if not self.versions.overlaps(other.versions):
             raise UnsatisfiableVersionSpecError(self.versions, other.versions)
 
@@ -1181,7 +1212,7 @@ def _constrain_dependencies(self, other):
 
         # TODO: might want more detail than this, e.g. specific deps
         # in violation. if this becomes a priority get rid of this
-        # check and be more specici about what's wrong.
+        # check and be more specific about what's wrong.
         if not other.satisfies_dependencies(self):
             raise UnsatisfiableDependencySpecError(other, self)
 
@@ -1247,7 +1278,7 @@ def satisfies(self, other, deps=True, strict=False):
 
         # A concrete provider can satisfy a virtual dependency.
         if not self.virtual and other.virtual:
-            pkg = spack.repo.get(self.name)
+            pkg = spack.repo.get(self.fullname)
             if pkg.provides(other.name):
                 for provided, when_spec in pkg.provided.items():
                     if self.satisfies(when_spec, deps=False, strict=strict):
@@ -1259,6 +1290,11 @@ def satisfies(self, other, deps=True, strict=False):
         if self.name != other.name:
             return False
 
+        # namespaces either match, or other doesn't require one.
+        if other.namespace is not None:
+            if self.namespace is not None and self.namespace != other.namespace:
+                return False
+
         if self.versions and other.versions:
             if not self.versions.satisfies(other.versions, strict=strict):
                 return False
@@ -1476,8 +1512,8 @@ def ne_dag(self, other):
 
     def _cmp_node(self):
         """Comparison key for just *this node* and not its deps."""
-        return (self.name, self.versions, self.variants,
+        return (self.name, self.namespace, self.versions,
-                self.architecture, self.compiler)
+                self.variants, self.architecture, self.compiler)
 
 
     def eq_node(self, other):
@@ -1507,7 +1543,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs):
            in the format string.  The format strings you can provide are::
 
                $_   Package name
-               $.   Long package name
+               $.   Full package name (with namespace)
                $@   Version
                $%   Compiler
                $%@  Compiler & compiler version
@@ -1556,8 +1592,7 @@ def write(s, c):
                 if c == '_':
                     out.write(fmt % self.name)
                 elif c == '.':
-                    longname = '%s.%s.%s' % (self.namespace, self.name) if self.namespace else self.name
+                    out.write(fmt % self.fullname)
-                    out.write(fmt % longname)
                 elif c == '@':
                     if self.versions and self.versions != _any_version:
                         write(fmt % (c + str(self.versions)), c)

lib/spack/spack/test/spec_semantics.py

@@ -35,6 +35,9 @@ class SpecSematicsTest(MockPackagesTest):
     # ================================================================================
     def check_satisfies(self, spec, anon_spec, concrete=False):
         left = Spec(spec, concrete=concrete)
+        try:
+            right = Spec(anon_spec)  # if it's not anonymous, allow it.
+        except:
             right = parse_anonymous_spec(anon_spec, left.name)
 
         # Satisfies is one-directional.
@@ -48,6 +51,9 @@ def check_satisfies(self, spec, anon_spec, concrete=False):
 
     def check_unsatisfiable(self, spec, anon_spec, concrete=False):
         left = Spec(spec, concrete=concrete)
+        try:
+            right = Spec(anon_spec)  # if it's not anonymous, allow it.
+        except:
             right = parse_anonymous_spec(anon_spec, left.name)
 
         self.assertFalse(left.satisfies(right))
@@ -88,6 +94,28 @@ def test_satisfies(self):
         self.check_satisfies('libdwarf^libelf@0.8.13', '^libelf@0:1')
 
 
+    def test_satisfies_namespace(self):
+        self.check_satisfies('builtin.mpich', 'mpich')
+        self.check_satisfies('builtin.mock.mpich', 'mpich')
+
+        # TODO: only works for deps now, but shouldn't we allow this for root spec?
+        # self.check_satisfies('builtin.mock.mpich', 'mpi')
+
+        self.check_satisfies('builtin.mock.mpich', 'builtin.mock.mpich')
+
+        self.check_unsatisfiable('builtin.mock.mpich', 'builtin.mpich')
+
+
+    def test_satisfies_namespaced_dep(self):
+        """Ensure spec from same or unspecified namespace satisfies namespace constraint."""
+        self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpich')
+
+        self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpi')
+        self.check_satisfies('mpileaks ^builtin.mock.mpich', '^builtin.mock.mpich')
+
+        self.check_unsatisfiable('mpileaks ^builtin.mock.mpich', '^builtin.mpich')
+
+
     def test_satisfies_compiler(self):
         self.check_satisfies('foo%gcc', '%gcc')
         self.check_satisfies('foo%intel', '%intel')
@@ -327,4 +355,3 @@ def test_constrain_dependency_not_changed(self):
         self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug')
         self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug')
         self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0')
-