init: simplify import ordering in __init__.py

lib/spack/spack/__init__.py

@@ -23,13 +23,11 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import multiprocessing
 import os
 import sys
-import tempfile
+import multiprocessing
-import getpass
+
-from llnl.util.filesystem import *
+from llnl.util.filesystem import ancestor
-import llnl.util.tty as tty
 
 #-----------------------------------------------------------------------------
 # Variables describing how Spack is laid out in its prefix.
@@ -38,102 +36,73 @@
 spack_root = ancestor(__file__, 4)
 
 # The spack script itself
-spack_file = join_path(spack_root, "bin", "spack")
+spack_file = os.path.join(spack_root, "bin", "spack")
 
 # spack directory hierarchy
-lib_path       = join_path(spack_root, "lib", "spack")
+lib_path              = os.path.join(spack_root, "lib", "spack")
-external_path  = join_path(lib_path, "external")
+external_path         = os.path.join(lib_path, "external")
-build_env_path = join_path(lib_path, "env")
+build_env_path        = os.path.join(lib_path, "env")
-module_path    = join_path(lib_path, "spack")
+module_path           = os.path.join(lib_path, "spack")
-platform_path  = join_path(module_path, 'platforms')
+platform_path         = os.path.join(module_path, 'platforms')
-compilers_path = join_path(module_path, "compilers")
+compilers_path        = os.path.join(module_path, "compilers")
-build_systems_path = join_path(module_path, 'build_systems')
+build_systems_path    = os.path.join(module_path, 'build_systems')
-operating_system_path = join_path(module_path, 'operating_systems')
+operating_system_path = os.path.join(module_path, 'operating_systems')
-test_path      = join_path(module_path, "test")
+test_path             = os.path.join(module_path, "test")
-hooks_path     = join_path(module_path, "hooks")
+hooks_path            = os.path.join(module_path, "hooks")
-var_path       = join_path(spack_root, "var", "spack")
+var_path              = os.path.join(spack_root, "var", "spack")
-stage_path     = join_path(var_path, "stage")
+stage_path            = os.path.join(var_path, "stage")
-repos_path     = join_path(var_path, "repos")
+repos_path            = os.path.join(var_path, "repos")
-share_path     = join_path(spack_root, "share", "spack")
+share_path            = os.path.join(spack_root, "share", "spack")
 
 # Paths to built-in Spack repositories.
-packages_path      = join_path(repos_path, "builtin")
+packages_path      = os.path.join(repos_path, "builtin")
-mock_packages_path = join_path(repos_path, "builtin.mock")
+mock_packages_path = os.path.join(repos_path, "builtin.mock")
 
 # User configuration location
 user_config_path = os.path.expanduser('~/.spack')
 
 prefix = spack_root
-opt_path        = join_path(prefix, "opt")
+opt_path        = os.path.join(prefix, "opt")
-etc_path        = join_path(prefix, "etc")
+etc_path        = os.path.join(prefix, "etc")
 system_etc_path = '/etc'
 
 # GPG paths.
-gpg_keys_path      = join_path(var_path, "gpg")
+gpg_keys_path      = os.path.join(var_path, "gpg")
-mock_gpg_data_path = join_path(var_path, "gpg.mock", "data")
+mock_gpg_data_path = os.path.join(var_path, "gpg.mock", "data")
-mock_gpg_keys_path = join_path(var_path, "gpg.mock", "keys")
+mock_gpg_keys_path = os.path.join(var_path, "gpg.mock", "keys")
-gpg_path           = join_path(opt_path, "spack", "gpg")
+gpg_path           = os.path.join(opt_path, "spack", "gpg")
 
 
 #-----------------------------------------------------------------------------
-# Initial imports (only for use in this file -- see __all__ below.)
+# Below code imports spack packages.
+#-----------------------------------------------------------------------------
+# The imports depend on paths above, or on each other, so ordering is tricky.
+# TODO: refactor everything below to be more init order agnostic.
+
+
+#-----------------------------------------------------------------------------
+# Import spack.config first, as other modules may rely on its options.
+# TODO: Below code should not import modules other than spack.config
 #-----------------------------------------------------------------------------
-# These imports depend on the paths above, or on each other
-# Group them here so it's easy to understand the order.
-# TODO: refactor this stuff to be more init order agnostic.
-import spack.repository
-import spack.error
 import spack.config
-import spack.fetch_strategy
-from spack.file_cache import FileCache
-from spack.abi import ABI
-from spack.concretize import DefaultConcretizer
-from spack.version import Version
 from spack.util.path import canonicalize_path
-from spack.package_prefs import PackageTesting
-
-#-----------------------------------------------------------------------------
-# Initialize various data structures & objects at the core of Spack.
-#-----------------------------------------------------------------------------
-# Version information
-spack_version = Version("0.11.2")
 
 
-# Set up the default packages database.
+# handle basic configuration first
-try:
-    repo = spack.repository.RepoPath()
-    sys.meta_path.append(repo)
-except spack.error.SpackError as e:
-    tty.die('while initializing Spack RepoPath:', e.message)
-
-
-# Tests ABI compatibility between packages
-abi = ABI()
-
-
-# This controls how things are concretized in spack.
-# Replace it with a subclass if you want different
-# policies.
-concretizer = DefaultConcretizer()
-
-#-----------------------------------------------------------------------------
-# config.yaml options
-#-----------------------------------------------------------------------------
 _config = spack.config.get_config('config')
 
 
 # Path where downloaded source code is cached
 cache_path = canonicalize_path(
-    _config.get('source_cache', join_path(var_path, "cache")))
+    _config.get('source_cache', os.path.join(var_path, "cache")))
-fetch_cache = spack.fetch_strategy.FsCache(cache_path)
 
 
 # cache for miscellaneous stuff.
 misc_cache_path = canonicalize_path(
-    _config.get('misc_cache', join_path(user_config_path, 'cache')))
+    _config.get('misc_cache', os.path.join(user_config_path, 'cache')))
-misc_cache = FileCache(misc_cache_path)
 
 
+# TODO: get this out of __init__.py
 binary_cache_retrieved_specs = set()
 
 
@@ -141,13 +110,14 @@
 template_dirs = spack.config.get_config('config')['template_dirs']
 template_dirs = [canonicalize_path(x) for x in template_dirs]
 
-# If this is enabled, tools that use SSL should not verify
+
-# certifiates. e.g., curl should use the -k option.
+#: If this is enabled, tools that use SSL should not verify
+#: certifiates. e.g., curl should use the -k option.
 insecure = not _config.get('verify_ssl', True)
 
 
-# Whether spack should allow installation of unsafe versions of software.
+#: Whether spack should allow installation of unsafe versions of software.
-# "Unsafe" versions are ones it doesn't have a checksum for.
+#: "Unsafe" versions are ones it doesn't have a checksum for.
 do_checksum = _config.get('checksum', True)
 
 
@@ -156,17 +126,57 @@
 dirty = _config.get('dirty', False)
 
 
-# The number of jobs to use when building in parallel.
+#: The number of jobs to use when building in parallel.
-# By default, use all cores on the machine.
+#: By default, use all cores on the machine.
 build_jobs = _config.get('build_jobs', multiprocessing.cpu_count())
 
 
-# Needed for test dependencies
+#-----------------------------------------------------------------------------
+# Initialize various data structures & objects at the core of Spack.
+#
+# TODO: move all of these imports out of __init__ to avoid importing the whole
+# TODO: world on Spack startup. There are some design changes that need to be
+# TODO: made to enable this (decoupling Spec, repo, DB, and store state).
+#
+# TODO: Spack probably needs some kind of object to manage this state so that
+# TODO: this stuff doesn't have to be at module scope.
+# -----------------------------------------------------------------------------
+# Version information
+from spack.version import Version
+spack_version = Version("0.11.2")
+
+
+# set up the caches after getting all config options
+import spack.fetch_strategy
+from spack.file_cache import FileCache
+misc_cache = FileCache(misc_cache_path)
+fetch_cache = spack.fetch_strategy.FsCache(cache_path)
+
+
+# Set up the default packages database.
+import spack.error
+try:
+    import spack.repository
+    repo = spack.repository.RepoPath()
+    sys.meta_path.append(repo)
+except spack.error.SpackError as e:
+    import llnl.util.tty as tty
+    tty.die('while initializing Spack RepoPath:', e.message)
+
+
+#: Concretizer class implements policy decisions for concretization
+from spack.concretize import Concretizer
+concretizer = Concretizer()
+
+
+#: Needed for test dependencies
+from spack.package_prefs import PackageTesting
 package_testing = PackageTesting()
 
 
 #-----------------------------------------------------------------------------
-# When packages call 'from spack import *', this extra stuff is brought in.
+# When packages call 'from spack import *', we import a set of things that
+# should be useful for builds.
 #
 # Spack internal code should call 'import spack' and accesses other
 # variables (spack.repo, paths, etc.) directly.

lib/spack/spack/abi.py

@@ -22,15 +22,16 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-
 import os
+
+from llnl.util.lang import memoized
+
 import spack
 import spack.spec
 from spack.build_environment import dso_suffix
 from spack.spec import CompilerSpec
 from spack.util.executable import Executable, ProcessError
 from spack.compilers.clang import Clang
-from llnl.util.lang import memoized
 
 
 class ABI(object):

lib/spack/spack/concretize.py

@@ -39,6 +39,7 @@
 from six import iteritems
 
 import spack
+import spack.abi
 import spack.spec
 import spack.compilers
 import spack.architecture
@@ -47,7 +48,11 @@
 from spack.package_prefs import PackagePrefs, spec_externals, is_spec_buildable
 
 
-class DefaultConcretizer(object):
+#: impements rudimentary logic for ABI compatibility
+abi = spack.abi.ABI()
+
+
+class Concretizer(object):
     """You can subclass this class to override some of the default
        concretization strategies, or you can override all of them.
     """
@@ -132,8 +137,8 @@ def choose_virtual_or_external(self, spec):
         return sorted(candidates,
                       reverse=True,
                       key=lambda spec: (
-                          spack.abi.compatible(spec, abi_exemplar, loose=True),
+                          abi.compatible(spec, abi_exemplar, loose=True),
-                          spack.abi.compatible(spec, abi_exemplar)))
+                          abi.compatible(spec, abi_exemplar)))
 
     def concretize_version(self, spec):
         """If the spec is already concrete, return.  Otherwise take

lib/spack/spack/config.py

@@ -60,15 +60,16 @@
 import jsonschema
 from yaml.error import MarkedYAMLError
 from jsonschema import Draft4Validator, validators
-from spack.util.ordereddict import OrderedDict
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import mkdirp
 
 import spack
 import spack.architecture
-from spack.error import SpackError
 import spack.schema
+from spack.error import SpackError
+from spack.util.ordereddict import OrderedDict
+
 
 # Hacked yaml for configuration files preserves line numbers.
 import spack.util.spack_yaml as syaml

lib/spack/spack/error.py

@@ -25,10 +25,11 @@
 from __future__ import print_function
 
 import sys
+import inspect
 
 import llnl.util.tty as tty
+
 import spack
-import inspect
 
 
 class SpackError(Exception):

lib/spack/spack/operating_systems/cnl.py

@@ -26,8 +26,6 @@
 
 import llnl.util.tty as tty
 
-import spack.spec
-import spack.compilers
 from spack.architecture import OperatingSystem
 from spack.util.multiproc import parmap
 from spack.util.module_cmd import get_module_cmd
@@ -58,6 +56,9 @@ def _detect_crayos_version(self):
         return latest_version
 
     def find_compilers(self, *paths):
+        # function-local so that cnl doesn't depend on spack.config
+        import spack.compilers
+
         types = spack.compilers.all_compiler_types()
         compiler_lists = parmap(
             lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types)
@@ -68,6 +69,9 @@ def find_compilers(self, *paths):
         return clist
 
     def find_compiler(self, cmp_cls, *paths):
+        # function-local so that cnl doesn't depend on spack.config
+        import spack.spec
+
         compilers = []
         if cmp_cls.PrgEnv:
             if not cmp_cls.PrgEnv_compiler:

lib/spack/spack/util/path.py

@@ -23,6 +23,8 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 """Utilities for managing paths in Spack.
+
+TODO: this is really part of spack.config. Consolidate it.
 """
 import os
 import re

lib/spack/spack/version.py

@@ -51,6 +51,7 @@
 
 from spack.util.spack_yaml import syaml_dict
 
+
 __all__ = ['Version', 'VersionRange', 'VersionList', 'ver']
 
 # Valid version characters