Moving utilities to a common LLNL package.

bin/spack

@@ -41,8 +41,8 @@ sys.path.insert(0, SPACK_LIB_PATH)
 
 # clean up the scope and start using spack package instead.
 del SPACK_FILE, SPACK_PREFIX, SPACK_LIB_PATH
+import llnl.util.tty as tty
 import spack
-import spack.tty as tty
 from spack.error import SpackError
 
 # Command parsing

lib/spack/env/cc

@@ -18,7 +18,7 @@ if not spack_lib:
 # Grab a minimal set of spack packages
 sys.path.append(spack_lib)
 from spack.compilation import *
-import spack.tty as tty
+import llnl.util.tty as tty
 
 spack_prefix     = get_env_var("SPACK_PREFIX")
 spack_build_root = get_env_var("SPACK_BUILD_ROOT")

lib/spack/llnl/util/filesystem.py

@@ -29,7 +29,7 @@
 import getpass
 from contextlib import contextmanager, closing
 
-import spack.tty as tty
+import llnl.util.tty as tty
 from spack.util.compression import ALLOWED_ARCHIVE_TYPES
 
 
@@ -70,14 +70,10 @@ def mkdirp(*paths):
             raise OSError(errno.EEXIST, "File alredy exists", path)
 
 
-def new_path(prefix, *args):
+def join_path(prefix, *args):
     path = str(prefix)
     for elt in args:
         path = os.path.join(path, str(elt))
-
-    if re.search(r'\s', path):
-        tty.die("Invalid path: '%s'.  Use a path without whitespace." % path)
-
     return path
 
 
@@ -89,16 +85,6 @@ def ancestor(dir, n=1):
     return parent
 
 
-def stem(path):
-    """Get the part of a path that does not include its compressed
-       type extension."""
-    for type in ALLOWED_ARCHIVE_TYPES:
-        suffix = r'\.%s$' % type
-        if re.search(suffix, path):
-            return re.sub(suffix, "", path)
-    return path
-
-
 def can_access(file_name):
     """True if we have read/write access to the file."""
     return os.access(file_name, os.R_OK|os.W_OK)

lib/spack/llnl/util/lang.py

@@ -27,12 +27,10 @@
 import sys
 import functools
 import inspect
-from spack.util.filesystem import new_path
 
 # Ignore emacs backups when listing modules
 ignore_modules = [r'^\.#', '~$']
 
-
 def caller_locals():
     """This will return the locals of the *parent* of the caller.
        This allows a fucntion to insert variables into its caller's
@@ -114,9 +112,9 @@ def list_modules(directory, **kwargs):
         if name == '__init__.py':
             continue
 
-        path = new_path(directory, name)
+        path = os.path.join(directory, name)
         if list_directories and os.path.isdir(path):
-            init_py = new_path(path, '__init__.py')
+            init_py = os.path.join(path, '__init__.py')
             if os.path.isfile(init_py):
                 yield name
 

lib/spack/llnl/util/tty/__init__.py

@@ -23,10 +23,11 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import sys
-import spack
+from llnl.util.tty.color import *
-from spack.color import *
 
-indent = "  "
+debug   = False
+verbose = False
+indent  = "  "
 
 def msg(message, *args):
     cprint("@*b{==>} %s" % cescape(message))
@@ -42,13 +43,13 @@ def info(message, *args, **kwargs):
 
 
 def verbose(message, *args):
-    if spack.verbose:
+    if verbose:
-        info(str(message), *args, format='c')
+        info(message, *args, format='c')
 
 
 def debug(*args):
-    if spack.debug:
+    if debug:
-        info("Debug: " + str(message), *args, format='*g')
+        info("Debug: " + message, *args, format='*g')
 
 
 def error(message, *args):
@@ -64,19 +65,6 @@ def die(message, *args):
     sys.exit(1)
 
 
-def pkg(message):
-    """Outputs a message with a package icon."""
-    import platform
-    from version import Version
-
-    mac_ver = platform.mac_ver()[0]
-    if mac_ver and Version(mac_ver) >= Version('10.7'):
-        print u"\U0001F4E6" + indent,
-    else:
-        cwrite('@*g{[+]} ')
-    print message
-
-
 def get_number(prompt, **kwargs):
     default = kwargs.get('default', None)
     abort = kwargs.get('abort', None)

lib/spack/llnl/util/tty/color.py

@@ -74,9 +74,8 @@
 """
 import re
 import sys
-import spack.error
 
-class ColorParseError(spack.error.SpackError):
+class ColorParseError(Exception):
     """Raised when a color format fails to parse."""
     def __init__(self, message):
         super(ColorParseError, self).__init__(message)

lib/spack/spack/architecture.py

@@ -25,10 +25,11 @@
 import os
 import platform as py_platform
 
+from llnl.util.lang import memoized
+
 import spack
 import spack.error as serr
 from spack.version import Version
-from spack.util.lang import memoized
 
 
 class InvalidSysTypeError(serr.SpackError):

lib/spack/spack/cmd/__init__.py

@@ -26,10 +26,11 @@
 import re
 import sys
 
+import llnl.util.tty as tty
+from llnl.util.lang import attr_setdefault
+
 import spack
 import spack.spec
-import spack.tty as tty
-from spack.util.lang import attr_setdefault
 
 # cmd has a submodule called "list" so preserve the python list module
 python_list = list

lib/spack/spack/cmd/bootstrap.py

@@ -24,9 +24,11 @@
 ##############################################################################
 import os
 from subprocess import check_call, check_output
+
+import llnl.util.tty as tty
+
 import spack
-from spack import new_path
+from spack import join_path
-import spack.tty as tty
 
 description = "Create a new installation of spack in another prefix"
 
@@ -35,7 +37,7 @@ def setup_parser(subparser):
 
 
 def get_origin_url():
-    git_dir = new_path(spack.prefix, '.git')
+    git_dir = join_path(spack.prefix, '.git')
     origin_url = check_output(
         ['git', '--git-dir=%s' % git_dir, 'config', '--get', 'remote.origin.url'])
     return origin_url.strip()
@@ -47,7 +49,7 @@ def bootstrap(parser, args):
 
     tty.msg("Fetching spack from origin: %s" % origin_url)
 
-    if os.path.exists(new_path(prefix, '.git')):
+    if os.path.exists(join_path(prefix, '.git')):
         tty.die("There already seems to be a git repository in %s" % prefix)
 
     files_in_the_way = os.listdir(prefix)

lib/spack/spack/cmd/checksum.py

@@ -29,13 +29,14 @@
 from pprint import pprint
 from subprocess import CalledProcessError
 
+import llnl.util.tty as tty
+from llnl.util.tty.colify import colify
+
 import spack
 import spack.cmd
-import spack.tty as tty
 import spack.packages as packages
 import spack.util.crypto
 from spack.stage import Stage, FailedDownloadError
-from spack.colify import colify
 from spack.version import *
 
 description ="Checksum available versions of a package to update a package file."

lib/spack/spack/cmd/clean.py

@@ -24,9 +24,10 @@
 ##############################################################################
 import argparse
 
+import llnl.util.tty as tty
+
 import spack.cmd
 import spack.packages as packages
-import spack.tty as tty
 import spack.stage as stage
 
 description = "Remove staged files for packages"

lib/spack/spack/cmd/compilers.py

@@ -22,9 +22,10 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+import llnl.util.tty as tty
+from llnl.util.tty.colify import colify
+
 import spack.compilers
-import spack.tty as tty
-from spack.colify import colify
 
 description = "List available compilers"
 

lib/spack/spack/cmd/create.py

@@ -28,11 +28,12 @@
 import re
 from contextlib import closing
 
+import llnl.util.tty as tty
+
 import spack
 import spack.cmd
 import spack.package
 import spack.packages as packages
-import spack.tty as tty
 import spack.url
 import spack.util.crypto as crypto
 import spack.cmd.checksum

lib/spack/spack/cmd/edit.py

@@ -26,9 +26,10 @@
 import string
 from contextlib import closing
 
+import llnl.util.tty as tty
+
 import spack
 import spack.packages as packages
-import spack.tty as tty
 
 description = "Open package files in $EDITOR"
 

lib/spack/spack/cmd/find.py

@@ -26,12 +26,13 @@
 import argparse
 from StringIO import StringIO
 
+from llnl.util.tty.colify import colify
+from llnl.util.tty.color import *
+
 import spack
 import spack.spec
 import spack.packages as packages
-import spack.colify
+
-from spack.color import *
-from spack.colify import colify
 
 description ="Find installed spack packages"
 

lib/spack/spack/cmd/info.py

@@ -24,10 +24,9 @@
 ##############################################################################
 import re
 import textwrap
-
+from llnl.util.tty.colify import colify
 import spack
 import spack.packages as packages
-from spack.colify import colify
 
 description = "Get detailed information on a particular package"
 

lib/spack/spack/cmd/list.py

@@ -23,7 +23,7 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import spack.packages as packages
-from spack.colify import colify
+from llnl.util.tty.colify import colify
 
 description ="List available spack packages"
 

lib/spack/spack/cmd/mirror.py

@@ -26,12 +26,14 @@
 import shutil
 import argparse
 
+import llnl.util.tty as tty
+from llnl.util.filesystem import mkdirp, join_path
+
 import spack.packages as packages
 import spack.cmd
-import spack.tty as tty
 
 from spack.stage import Stage
-from spack.util.filesystem import mkdirp, new_path
+
 
 description = "Create a directory full of package tarballs that can be used as a spack mirror."
 
@@ -66,7 +68,7 @@ def mirror(parser, args):
             continue
 
         # create a subdir for the current package.
-        pkg_path = new_path(args.directory, pkg_name)
+        pkg_path = join_path(args.directory, pkg_name)
         mkdirp(pkg_path)
 
         # Download all the tarballs using Stages, then move them into place
@@ -76,7 +78,7 @@ def mirror(parser, args):
             try:
                 stage.fetch()
                 basename = os.path.basename(stage.archive_file)
-                final_dst = new_path(pkg_path, basename)
+                final_dst = join_path(pkg_path, basename)
 
                 os.chdir(working_dir)
                 shutil.move(stage.archive_file, final_dst)

lib/spack/spack/cmd/providers.py

@@ -25,9 +25,10 @@
 import os
 import argparse
 
+from llnl.util.tty.colify import colify
+
 import spack.cmd
 import spack.packages
-from spack.colify import colify
 
 description ="List packages that provide a particular virtual package"
 

lib/spack/spack/cmd/spec.py

@@ -25,7 +25,8 @@
 import argparse
 import spack.cmd
 
-import spack.tty as tty
+import llnl.util.tty as tty
+
 import spack.url as url
 import spack
 

lib/spack/spack/cmd/test.py

@@ -22,12 +22,14 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+from pprint import pprint
+
+from llnl.util.tty.colify import colify
+from llnl.util.lang import list_modules
+
 import spack
 import spack.packages as packages
 import spack.test
-from spack.util.lang import list_modules
-from spack.colify import colify
-from pprint import pprint
 
 description ="Run unit tests"
 

lib/spack/spack/cmd/uninstall.py

@@ -24,8 +24,9 @@
 ##############################################################################
 import argparse
 
+import llnl.util.tty as tty
+
 import spack.cmd
-import spack.tty as tty
 import spack.packages as packages
 
 description="Remove an installed package"

lib/spack/spack/cmd/versions.py

@@ -23,8 +23,8 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import os
+from llnl.util.tty.colify import colify
 import spack.packages as packages
-from spack.colify import colify
 
 description ="List available versions of a package"
 

lib/spack/spack/compilers/__init__.py

@@ -25,10 +25,9 @@
 #
 # This needs to be expanded for full compiler support.
 #
-
+from llnl.util.lang import memoized, list_modules
 import spack
 import spack.compilers.gcc
-from spack.util.lang import memoized, list_modules
 
 @memoized
 def supported_compilers():

lib/spack/spack/directory_layout.py

@@ -24,12 +24,13 @@
 ##############################################################################
 import re
 import os
-import os.path
 import exceptions
 import hashlib
+import shutil
+from contextlib import closing
 
+from llnl.util.filesystem import join_path, mkdirp
 from spack.spec import Spec
-from spack.util.filesystem import *
 from spack.error import SpackError
 
 
@@ -138,7 +139,7 @@ def __init__(self, root, **kwargs):
     def relative_path_for_spec(self, spec):
         _check_concrete(spec)
 
-        path = new_path(
+        path = join_path(
             spec.architecture,
             spec.compiler,
             "%s@%s%s" % (spec.name, spec.version, spec.variants))
@@ -168,7 +169,7 @@ def make_path_for_spec(self, spec):
         _check_concrete(spec)
 
         path = self.path_for_spec(spec)
-        spec_file_path = new_path(path, self.spec_file)
+        spec_file_path = join_path(path, self.spec_file)
 
         if os.path.isdir(path):
             if not os.path.isfile(spec_file_path):
@@ -199,7 +200,7 @@ def all_specs(self):
 
         for path in traverse_dirs_at_depth(self.root, 3):
             arch, compiler, last_dir = path
-            spec_file_path = new_path(
+            spec_file_path = join_path(
                 self.root, arch, compiler, last_dir, self.spec_file)
             if os.path.exists(spec_file_path):
                 spec = self.read_spec(spec_file_path)

lib/spack/spack/globals.py

@@ -24,8 +24,9 @@
 ##############################################################################
 import os
 
+from llnl.util.filesystem import *
+
 from spack.version import Version
-from spack.util.filesystem import *
 from spack.util.executable import *
 from spack.directory_layout import SpecHashDirectoryLayout
 from spack.concretize import DefaultConcretizer
@@ -34,20 +35,20 @@
 prefix = ancestor(__file__, 4)
 
 # The spack script itself
-spack_file = new_path(prefix, "bin", "spack")
+spack_file = join_path(prefix, "bin", "spack")
 
 # spack directory hierarchy
-lib_path       = new_path(prefix, "lib", "spack")
+lib_path       = join_path(prefix, "lib", "spack")
-env_path       = new_path(lib_path, "env")
+env_path       = join_path(lib_path, "env")
-module_path    = new_path(lib_path, "spack")
+module_path    = join_path(lib_path, "spack")
-packages_path  = new_path(module_path, "packages")
+packages_path  = join_path(module_path, "packages")
-compilers_path = new_path(module_path, "compilers")
+compilers_path = join_path(module_path, "compilers")
-test_path      = new_path(module_path, "test")
+test_path      = join_path(module_path, "test")
 
-var_path       = new_path(prefix, "var", "spack")
+var_path       = join_path(prefix, "var", "spack")
-stage_path     = new_path(var_path, "stage")
+stage_path     = join_path(var_path, "stage")
 
-install_path   = new_path(prefix, "opt")
+install_path   = join_path(prefix, "opt")
 
 #
 # This controls how spack lays out install prefixes and
@@ -116,7 +117,7 @@
 # For no mirrors:
 #   mirrors = []
 #
-mirrors = []
+mirrors = ['file:///Users/gamblin2/mirror']
 
 # Important environment variables
 SPACK_NO_PARALLEL_MAKE = 'SPACK_NO_PARALLEL_MAKE'

lib/spack/spack/multimethod.py

@@ -47,9 +47,10 @@
 import functools
 import collections
 
+from llnl.util.lang import *
+
 import spack.architecture
 import spack.error
-from spack.util.lang import *
 from spack.spec import parse_anonymous_spec, Spec
 
 

lib/spack/spack/package.py

@@ -39,23 +39,28 @@
 import subprocess
 import platform as py_platform
 import shutil
+import multiprocessing
+from urlparse import urlparse
+
+import llnl.util.tty as tty
+from llnl.util.tty.color import cwrite
+from llnl.util.filesystem import touch
+from llnl.util.lang import *
 
 from spack import *
 import spack.spec
 import spack.error
-import packages
+import spack.packages as packages
-import tty
+import spack.url as url
-import validate
-import multiprocessing
-import url
-
 import spack.util.crypto as crypto
 from spack.version import *
 from spack.stage import Stage
-from spack.util.lang import *
 from spack.util.web import get_pages
 from spack.util.environment import *
-from spack.util.filesystem import touch
+from spack.util.compression import allowed_archive
+
+"""Allowed URL schemes for spack packages."""
+_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file"]
 
 
 class Package(object):
@@ -337,7 +342,7 @@ def __init__(self, spec):
             self.name = self.name[self.name.rindex('.') + 1:]
 
         # Make sure URL is an allowed type
-        validate.url(self.url)
+        validate_package_url(self.url)
 
         # patch up the URL with a new version if the spec version is concrete
         if self.spec.versions.concrete:
@@ -620,8 +625,8 @@ def do_patch(self):
         # Construct paths to special files in the archive dir used to
         # keep track of whether patches were successfully applied.
         archive_dir = self.stage.expanded_archive_path
-        good_file = new_path(archive_dir, '.spack_patched')
+        good_file = join_path(archive_dir, '.spack_patched')
-        bad_file  = new_path(archive_dir, '.spack_patch_failed')
+        bad_file  = join_path(archive_dir, '.spack_patch_failed')
 
         # If we encounter an archive that failed to patch, restage it
         # so that we can apply all the patches again.
@@ -664,7 +669,7 @@ def do_install(self):
 
         if os.path.exists(self.prefix):
             tty.msg("%s is already installed." % self.name)
-            tty.pkg(self.prefix)
+            print_pkg(self.prefix)
             return
 
         if not self.ignore_dependencies:
@@ -678,27 +683,28 @@ def do_install(self):
         self.add_commands_to_module()
 
         tty.msg("Building %s." % self.name)
-        try:
-            # create the install directory (allow the layout to handle this in
-            # case it needs to add extra files)
-            spack.install_layout.make_path_for_spec(self.spec)
 
+        # create the install directory (allow the layout to handle this in
+        # case it needs to add extra files)
+        spack.install_layout.make_path_for_spec(self.spec)
+
+        try:
             self.install(self.spec, self.prefix)
             if not os.path.isdir(self.prefix):
                 tty.die("Install failed for %s.  No install dir created." % self.name)
 
+            tty.msg("Successfully installed %s" % self.name)
+            print_pkg(self.prefix)
+
         except Exception, e:
-            if not self.dirty:
+            self.remove_prefix()
-                self.remove_prefix()
             raise
 
-        tty.msg("Successfully installed %s" % self.name)
+        finally:
-        tty.pkg(self.prefix)
+            # Once the install is done, destroy the stage where we built it,
-
+            # unless the user wants it kept around.
-        # Once the install is done, destroy the stage where we built it,
+            if not self.dirty:
-        # unless the user wants it kept around.
+                self.stage.destroy()
-        if not self.dirty:
-            self.stage.destroy()
 
 
     def setup_install_environment(self):
@@ -713,7 +719,7 @@ def setup_install_environment(self):
         # in directories called "case*" within the env directory.
         env_paths = [env_path]
         for file in os.listdir(env_path):
-            path = new_path(env_path, file)
+            path = join_path(env_path, file)
             if file.startswith("case") and os.path.isdir(path):
                 env_paths.append(path)
         path_put_first("PATH", env_paths)
@@ -883,6 +889,26 @@ def __call__(self, *args, **kwargs):
         super(MakeExecutable, self).__call__(*args, **kwargs)
 
 
+def validate_package_url(url_string):
+    """Determine whether spack can handle a particular URL or not."""
+    url = urlparse(url_string)
+    if url.scheme not in _ALLOWED_URL_SCHEMES:
+        tty.die("Invalid protocol in URL: '%s'" % url_string)
+
+    if not allowed_archive(url_string):
+        tty.die("Invalid file type in URL: '%s'" % url_string)
+
+
+def print_pkg(message):
+    """Outputs a message with a package icon."""
+    mac_ver = py_platform.mac_ver()[0]
+    if mac_ver and Version(mac_ver) >= Version('10.7'):
+        print u"\U0001F4E6" + tty.indent,
+    else:
+        cwrite('@*g{[+]} ')
+    print message
+
+
 class InvalidPackageDependencyError(spack.error.SpackError):
     """Raised when package specification is inconsistent with requirements of
        its dependencies."""

lib/spack/spack/packages/__init__.py

@@ -29,12 +29,13 @@
 import inspect
 import glob
 
+import llnl.util.tty as tty
+from llnl.util.filesystem import join_path
+from llnl.util.lang import list_modules
+
 import spack
 import spack.error
 import spack.spec
-import spack.tty as tty
-from spack.util.filesystem import new_path
-from spack.util.lang import list_modules
 
 # Valid package names can contain '-' but can't start with it.
 valid_package_re = r'^\w[\w-]*$'
@@ -212,7 +213,7 @@ def validate_package_name(pkg_name):
 def dirname_for_package_name(pkg_name):
     """Get the directory name for a particular package would use, even if it's a
        foo.py package and not a directory with a foo/__init__.py file."""
-    return new_path(spack.packages_path, pkg_name)
+    return join_path(spack.packages_path, pkg_name)
 
 
 def filename_for_package_name(pkg_name):
@@ -236,7 +237,7 @@ def filename_for_package_name(pkg_name):
     pkg_dir = dirname_for_package_name(pkg_name)
 
     if os.path.isdir(pkg_dir):
-        init_file = new_path(pkg_dir, '__init__.py')
+        init_file = join_path(pkg_dir, '__init__.py')
         return init_file
     else:
         pkg_file  = "%s.py" % pkg_dir

lib/spack/spack/patch.py

@@ -24,14 +24,15 @@
 ##############################################################################
 import os
 
+import llnl.util.tty as tty
+from llnl.util.filesystem import join_path
+
 import spack
 import spack.stage
 import spack.error
 import spack.packages as packages
-import spack.tty as tty
 
 from spack.util.executable import which
-from spack.util.filesystem import new_path
 
 # Patch tool for patching archives.
 _patch = which("patch", required=True)
@@ -55,7 +56,7 @@ def __init__(self, pkg_name, path_or_url, level):
             self.url = path_or_url
         else:
             pkg_dir = packages.dirname_for_package_name(pkg_name)
-            self.path = new_path(pkg_dir, path_or_url)
+            self.path = join_path(pkg_dir, path_or_url)
             if not os.path.isfile(self.path):
                 raise NoSuchPatchFileError(pkg_name, self.path)
 

lib/spack/spack/relations.py

@@ -72,6 +72,8 @@ class Mpileaks(Package):
 import inspect
 import importlib
 
+from llnl.util.lang import *
+
 import spack
 import spack.spec
 import spack.error
@@ -79,7 +81,6 @@ class Mpileaks(Package):
 from spack.patch import Patch
 from spack.spec import Spec, parse_anonymous_spec
 from spack.packages import packages_module
-from spack.util.lang import *
 
 
 """Adds a dependencies local variable in the locals of

lib/spack/spack/spec.py

@@ -95,16 +95,17 @@
 import hashlib
 from StringIO import StringIO
 
+import llnl.util.tty as tty
+from llnl.util.lang import *
+from llnl.util.tty.color import *
+
 import spack.parse
 import spack.error
 import spack.compilers
 import spack.compilers.gcc
 import spack.packages as packages
-import spack.tty as tty
 
 from spack.version import *
-from spack.color import *
-from spack.util.lang import *
 from spack.util.string import *
 from spack.util.prefix import Prefix
 

lib/spack/spack/stage.py

@@ -27,11 +27,11 @@
 import shutil
 import tempfile
 
+import llnl.util.tty as tty
+from llnl.util.filesystem import *
+
 import spack
 import spack.error as serr
-import spack.tty as tty
-
-from spack.util.filesystem import *
 from spack.util.compression import decompressor_for
 
 STAGE_PREFIX = 'spack-stage-'
@@ -88,7 +88,7 @@ def __init__(self, url, **kwargs):
     def _cleanup_dead_links(self):
         """Remove any dead links in the stage directory."""
         for file in os.listdir(spack.stage_path):
-            path = new_path(spack.stage_path, file)
+            path = join_path(spack.stage_path, file)
             if os.path.islink(path):
                 real_path = os.path.realpath(path)
                 if not os.path.exists(path):
@@ -150,7 +150,7 @@ def _setup(self):
 
         # If this is a named stage, then construct a named path.
         if self.name is not None:
-            self.path = new_path(spack.stage_path, self.name)
+            self.path = join_path(spack.stage_path, self.name)
 
         # If this is a temporary stage, them make the temp directory
         tmp_dir = None
@@ -159,7 +159,7 @@ def _setup(self):
                 # Unnamed tmp root.  Link the path in
                 tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root)
                 self.name = os.path.basename(tmp_dir)
-                self.path = new_path(spack.stage_path, self.name)
+                self.path = join_path(spack.stage_path, self.name)
                 if self._need_to_create_path():
                     os.symlink(tmp_dir, self.path)
 
@@ -200,7 +200,7 @@ def expanded_archive_path(self):
             return None
 
         for file in os.listdir(self.path):
-            archive_path = spack.new_path(self.path, file)
+            archive_path = join_path(self.path, file)
             if os.path.isdir(archive_path):
                 return archive_path
         return None
@@ -333,7 +333,7 @@ def purge():
     """Remove all build directories in the top-level stage path."""
     if os.path.isdir(spack.stage_path):
         for stage_dir in os.listdir(spack.stage_path):
-            stage_path = spack.new_path(spack.stage_path, stage_dir)
+            stage_path = join_path(spack.stage_path, stage_dir)
             remove_linked_tree(stage_path)
 
 

lib/spack/spack/test/__init__.py

@@ -25,9 +25,11 @@
 import sys
 import unittest
 
+import llnl.util.tty as tty
+from llnl.util.tty.colify import colify
+
 import spack
-from spack.colify import colify
+
-import spack.tty as tty
 
 """Names of tests to be included in Spack's test suite"""
 test_names = ['versions',

lib/spack/spack/test/mock_packages_test.py

@@ -24,12 +24,14 @@
 ##############################################################################
 import unittest
 
+from llnl.util.lang import list_modules
+from llnl.util.filesystem import join_path
+
 import spack
 import spack.packages as packages
 from spack.spec import Spec
-from spack.util.lang import new_path, list_modules
 
-mock_packages_path = new_path(spack.module_path, 'test', 'mock_packages')
+mock_packages_path = join_path(spack.module_path, 'test', 'mock_packages')
 original_deps = None
 
 

lib/spack/spack/test/packages.py

@@ -40,7 +40,7 @@ def test_regular_package_name(self):
 
     def test_regular_package_filename(self):
         filename = packages.filename_for_package_name('mpich')
-        self.assertEqual(filename, new_path(mock_packages_path, 'mpich.py'))
+        self.assertEqual(filename, join_path(mock_packages_path, 'mpich.py'))
 
 
     def test_regular_package_name(self):
@@ -61,9 +61,9 @@ def test_directory_package_name(self):
 
     def test_directory_package_filename(self):
         filename = packages.filename_for_package_name('directory-pkg')
-        self.assertEqual(filename, new_path(mock_packages_path, 'directory-pkg/__init__.py'))
+        self.assertEqual(filename, join_path(mock_packages_path, 'directory-pkg/__init__.py'))
 
 
     def test_nonexisting_package_filename(self):
         filename = packages.filename_for_package_name('some-nonexisting-package')
-        self.assertEqual(filename, new_path(mock_packages_path, 'some-nonexisting-package.py'))
+        self.assertEqual(filename, join_path(mock_packages_path, 'some-nonexisting-package.py'))

lib/spack/spack/test/spec_dag.py

@@ -32,7 +32,8 @@
 import spack.package
 import spack.packages as packages
 
-from spack.util.lang import new_path, list_modules
+from llnl.util.lang import list_modules
+
 from spack.spec import Spec
 from spack.test.mock_packages_test import *
 

lib/spack/spack/test/stage.py

@@ -31,28 +31,32 @@
 import getpass
 from contextlib import *
 
+from llnl.util.filesystem import *
+
 import spack
 from spack.stage import Stage
-from spack.util.filesystem import *
 from spack.util.executable import which
 
-test_files_dir = new_path(spack.stage_path, '.test')
+test_files_dir = join_path(spack.stage_path, '.test')
-test_tmp_path  = new_path(test_files_dir, 'tmp')
+test_tmp_path  = join_path(test_files_dir, 'tmp')
 
 archive_dir      = 'test-files'
 archive_name     = archive_dir + '.tar.gz'
-archive_dir_path = new_path(test_files_dir, archive_dir)
+archive_dir_path = join_path(test_files_dir, archive_dir)
-archive_url      = 'file://' + new_path(test_files_dir, archive_name)
+archive_url      = 'file://' + join_path(test_files_dir, archive_name)
 readme_name      = 'README.txt'
-test_readme      = new_path(archive_dir_path, readme_name)
+test_readme      = join_path(archive_dir_path, readme_name)
 readme_text      = "hello world!\n"
 
 stage_name = 'spack-test-stage'
 
 
 class with_tmp(object):
-    """Decorator that executes a function with or without spack set
+    """Decorator that executes a function with or without spack set to use
-       to use a temp dir."""
+       a temp dir.  Spack allows builds to happen directly in the
+       stage directory or in a tmp dir and symlinked into the stage
+       directory, so this lets us use the same test in both cases.
+    """
     def __init__(self, use_tmp):
         self.use_tmp = use_tmp
 
@@ -107,7 +111,7 @@ def get_stage_path(self, stage, stage_name):
         """
         if stage_name:
             # If it is a named stage, we know where the stage should be
-            stage_path = new_path(spack.stage_path, stage_name)
+            stage_path = join_path(spack.stage_path, stage_name)
         else:
             # If it's unnamed, ensure that we ran mkdtemp in the right spot.
             stage_path = stage.path
@@ -143,7 +147,7 @@ def check_setup(self, stage, stage_name):
     def check_fetch(self, stage, stage_name):
         stage_path = self.get_stage_path(stage, stage_name)
         self.assertIn(archive_name, os.listdir(stage_path))
-        self.assertEqual(new_path(stage_path, archive_name),
+        self.assertEqual(join_path(stage_path, archive_name),
                          stage.archive_file)
 
 
@@ -153,10 +157,10 @@ def check_expand_archive(self, stage, stage_name):
         self.assertIn(archive_dir, os.listdir(stage_path))
 
         self.assertEqual(
-            new_path(stage_path, archive_dir),
+            join_path(stage_path, archive_dir),
             stage.expanded_archive_path)
 
-        readme = new_path(stage_path, archive_dir, readme_name)
+        readme = join_path(stage_path, archive_dir, readme_name)
         self.assertTrue(os.path.isfile(readme))
 
         with closing(open(readme)) as file:
@@ -171,7 +175,7 @@ def check_chdir(self, stage, stage_name):
     def check_chdir_to_archive(self, stage, stage_name):
         stage_path = self.get_stage_path(stage, stage_name)
         self.assertEqual(
-            new_path(os.path.realpath(stage_path), archive_dir),
+            join_path(os.path.realpath(stage_path), archive_dir),
             os.getcwd())
 
 

lib/spack/spack/url.py

@@ -48,7 +48,7 @@
 import re
 
 import spack.error
-import spack.util.filesystem as fs
+import spack.util.compression as comp
 from spack.version import Version
 
 #
@@ -85,9 +85,9 @@ def parse_version_string_with_indices(path):
     if os.path.isdir(path):
         stem = os.path.basename(path)
     elif re.search(r'((?:sourceforge.net|sf.net)/.*)/download$', path):
-        stem = fs.stem(os.path.dirname(path))
+        stem = comp.stem(os.path.dirname(path))
     else:
-        stem = fs.stem(path)
+        stem = comp.stem(path)
 
     version_types = [
         # GitHub tarballs, e.g. v1.2.3

lib/spack/spack/util/compression.py

@@ -22,6 +22,7 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+import re
 from itertools import product
 from spack.util.executable import which
 
@@ -45,3 +46,13 @@ def decompressor_for(path):
     tar = which('tar', required=True)
     tar.add_default_arg('-xf')
     return tar
+
+
+def stem(path):
+    """Get the part of a path that does not include its compressed
+       type extension."""
+    for type in ALLOWED_ARCHIVE_TYPES:
+        suffix = r'\.%s$' % type
+        if re.search(suffix, path):
+            return re.sub(suffix, "", path)
+    return path

lib/spack/spack/util/executable.py

@@ -27,7 +27,7 @@
 import re
 import subprocess
 
-import spack.tty as tty
+import llnl.util.tty as tty
 from spack.error import SpackError
 
 

lib/spack/spack/util/prefix.py

@@ -25,7 +25,7 @@
 """
 This file contains utilities to help with installing packages.
 """
-from spack.util.filesystem import new_path
+from llnl.util.filesystem import join_path
 
 class Prefix(str):
     """This class represents an installation prefix, but provides useful
@@ -59,23 +59,23 @@ class Prefix(str):
 
     def __new__(cls, path):
         s = super(Prefix, cls).__new__(cls, path)
-        s.bin     = new_path(s, 'bin')
+        s.bin     = join_path(s, 'bin')
-        s.sbin    = new_path(s, 'sbin')
+        s.sbin    = join_path(s, 'sbin')
-        s.etc     = new_path(s, 'etc')
+        s.etc     = join_path(s, 'etc')
-        s.include = new_path(s, 'include')
+        s.include = join_path(s, 'include')
-        s.lib     = new_path(s, 'lib')
+        s.lib     = join_path(s, 'lib')
-        s.lib64   = new_path(s, 'lib64')
+        s.lib64   = join_path(s, 'lib64')
-        s.libexec = new_path(s, 'libexec')
+        s.libexec = join_path(s, 'libexec')
-        s.share   = new_path(s, 'share')
+        s.share   = join_path(s, 'share')
-        s.doc     = new_path(s.share, 'doc')
+        s.doc     = join_path(s.share, 'doc')
-        s.info    = new_path(s.share, 'info')
+        s.info    = join_path(s.share, 'info')
-        s.man     = new_path(s.share, 'man')
+        s.man     = join_path(s.share, 'man')
-        s.man1    = new_path(s.man, 'man1')
+        s.man1    = join_path(s.man, 'man1')
-        s.man2    = new_path(s.man, 'man2')
+        s.man2    = join_path(s.man, 'man2')
-        s.man3    = new_path(s.man, 'man3')
+        s.man3    = join_path(s.man, 'man3')
-        s.man4    = new_path(s.man, 'man4')
+        s.man4    = join_path(s.man, 'man4')
-        s.man5    = new_path(s.man, 'man5')
+        s.man5    = join_path(s.man, 'man5')
-        s.man6    = new_path(s.man, 'man6')
+        s.man6    = join_path(s.man, 'man6')
-        s.man7    = new_path(s.man, 'man7')
+        s.man7    = join_path(s.man, 'man7')
-        s.man8    = new_path(s.man, 'man8')
+        s.man8    = join_path(s.man, 'man8')
         return s

lib/spack/spack/util/web.py

@@ -29,9 +29,10 @@
 from multiprocessing import Pool
 from HTMLParser import HTMLParser
 
+import llnl.util.tty as tty
+
 import spack
 import spack.error
-import spack.tty as tty
 from spack.util.compression import ALLOWED_ARCHIVE_TYPES
 
 # Timeout in seconds for web requests

lib/spack/spack/validate.py

@@ -1,38 +0,0 @@
-##############################################################################
-# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
-# Produced at the Lawrence Livermore National Laboratory.
-#
-# This file is part of Spack.
-# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
-# LLNL-CODE-647188
-#
-# For details, see https://scalability-llnl.github.io/spack
-# Please also see the LICENSE file for our notice and the LGPL.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License (as published by
-# the Free Software Foundation) version 2.1 dated February 1999.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
-# conditions of the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program; if not, write to the Free Software Foundation,
-# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-##############################################################################
-import tty
-from urlparse import urlparse
-
-from spack.util.compression import allowed_archive
-
-ALLOWED_SCHEMES    = ["http", "https", "ftp", "file"]
-
-def url(url_string):
-    url = urlparse(url_string)
-    if url.scheme not in ALLOWED_SCHEMES:
-        tty.die("Invalid protocol in URL: '%s'" % url_string)
-
-    if not allowed_archive(url_string):
-        tty.die("Invalid file type in URL: '%s'" % url_string)

lib/spack/spack/version.py

@@ -49,8 +49,8 @@
 from bisect import bisect_left
 from functools import total_ordering, wraps
 
-import spack.util.none_high as none_high
+import llnl.util.compare.none_high as none_high
-import spack.util.none_low as none_low
+import llnl.util.compare.none_low as none_low
 import spack.error
 
 # Valid version characters