source_cache, misc_cache, verify_ssl, checksum, & dirty in config.yaml

- Added new preferences to config.yaml:
  - source_cache
  - misc_cache
  - verify_ssl
  - checksum
  - dirty

etc/spack/defaults/config.yaml

@@ -35,6 +35,8 @@ config:
   # A value of $spack/var/spack/stage indicates that Spack should run
   # builds directly inside its install directory without staging them in
   # temporary space.
+  #
+  # The build stage can be purged with `spack purge --stage`.
   build_stage:
     - $tempdir
     - /nfs/tmp2/$user
@@ -42,11 +44,12 @@ config:
 
 
   # Cache directory already downloaded source tarballs and archived
-  # repositories. This can be purged with spack purge
+  # repositories. This can be purged with `spack purge --downloads`.
   source_cache: $spack/var/spack/cache
 
 
   # Cache directory for miscellaneous files, like the package index.
+  # This can be purged with `spack purge --misc-cache`
   misc_cache: ~/.spack/cache
 
 

lib/spack/spack/__init__.py

@@ -30,7 +30,10 @@
 from llnl.util.filesystem import *
 import llnl.util.tty as tty
 
-# This lives in $prefix/lib/spack/spack/__file__
+#-----------------------------------------------------------------------------
+# Variables describing how Spack is laid out in its prefix.
+#-----------------------------------------------------------------------------
+# This file lives in $prefix/lib/spack/spack/__file__
 spack_root = ancestor(__file__, 4)
 
 # The spack script itself
@@ -49,82 +52,100 @@
 stage_path     = join_path(var_path, "stage")
 repos_path     = join_path(var_path, "repos")
 share_path     = join_path(spack_root, "share", "spack")
-cache_path     = join_path(var_path, "cache")
+
+# Paths to built-in Spack repositories.
+packages_path      = join_path(repos_path, "builtin")
+mock_packages_path = join_path(repos_path, "builtin.mock")
 
 # User configuration location
 user_config_path = os.path.expanduser('~/.spack')
 
-import spack.fetch_strategy
-fetch_cache = spack.fetch_strategy.FsCache(cache_path)
-
-from spack.file_cache import FileCache
-user_cache_path = join_path(user_config_path, 'cache')
-user_cache = FileCache(user_cache_path)
-
 prefix = spack_root
 opt_path       = join_path(prefix, "opt")
 etc_path       = join_path(prefix, "etc")
 
-#
+
-# Set up the default packages database.
+#-----------------------------------------------------------------------------
-#
+# Initial imports (only for use in this file -- see __all__ below.)
+#-----------------------------------------------------------------------------
+# These imports depend on the paths above, or on each other
+# Group them here so it's easy to understand the order.
+# TODO: refactor this stuff to be more init order agnostic.
 import spack.repository
+import spack.error
+import spack.config
+import spack.fetch_strategy
+from spack.file_cache import FileCache
+from spack.preferred_packages import PreferredPackages
+from spack.abi import ABI
+from spack.concretize import DefaultConcretizer
+from spack.version import Version
+from spack.util.path import canonicalize_path
+
+
+#-----------------------------------------------------------------------------
+# Initialize various data structures & objects at the core of Spack.
+#-----------------------------------------------------------------------------
+# Version information
+spack_version = Version("0.9.1")
+
+
+# Set up the default packages database.
 try:
     repo = spack.repository.RepoPath()
     sys.meta_path.append(repo)
 except spack.error.SpackError, e:
     tty.die('while initializing Spack RepoPath:', e.message)
 
-#
-# Paths to built-in Spack repositories.
-#
-packages_path      = join_path(repos_path, "builtin")
-mock_packages_path = join_path(repos_path, "builtin.mock")
 
-#
+# PreferredPackages controls preference sort order during concretization.
-# This controls how packages are sorted when trying to choose
+# More preferred packages are sorted first.
-# the most preferred package.  More preferred packages are sorted
-# first.
-#
-from spack.preferred_packages import PreferredPackages
 pkgsort = PreferredPackages()
 
-#
+
-# This tests ABI compatibility between packages
+# Tests ABI compatibility between packages
-#
-from spack.abi import ABI
 abi = ABI()
 
-#
+
 # This controls how things are concretized in spack.
 # Replace it with a subclass if you want different
 # policies.
-#
-from spack.concretize import DefaultConcretizer
 concretizer = DefaultConcretizer()
 
-# Version information
+#-----------------------------------------------------------------------------
-from spack.version import Version
+# config.yaml options
-spack_version = Version("0.9.1")
+#-----------------------------------------------------------------------------
+_config = spack.config.get_config('config')
 
-#
-# Executables used by Spack
-#
-from spack.util.executable import Executable, which
 
-# User's editor from the environment
+# Path where downloaded source code is cached
-editor = Executable(os.environ.get("EDITOR", "vi"))
+cache_path = canonicalize_path(
+    _config.get('source_cache', join_path(var_path, "cache")))
+fetch_cache = spack.fetch_strategy.FsCache(cache_path)
+
+
+# cache for miscellaneous stuff.
+misc_cache_path = canonicalize_path(
+    _config.get('misc_cache', join_path(user_config_path, 'cache')))
+misc_cache = FileCache(misc_cache_path)
+
 
 # If this is enabled, tools that use SSL should not verify
 # certifiates. e.g., curl should use the -k option.
-insecure = False
+insecure = not _config.get('verify_ssl', True)
 
-# Whether spack should allow installation of unsafe versions of
-# software.  "Unsafe" versions are ones it doesn't have a checksum
-# for.
-do_checksum = True
 
-#
+# Whether spack should allow installation of unsafe versions of software.
+# "Unsafe" versions are ones it doesn't have a checksum for.
+do_checksum = _config.get('checksum', True)
+
+
+# If this is True, spack will not clean the environment to remove
+# potentially harmful variables before builds.
+dirty = _config.get('dirty', False)
+
+
+#-----------------------------------------------------------------------------
 # When packages call 'from spack import *', this extra stuff is brought in.
 #
 # Spack internal code should call 'import spack' and accesses other
@@ -135,6 +156,7 @@
 #       packages should live.  This file is overloaded for spack core vs.
 #       for packages.
 #
+#-----------------------------------------------------------------------------
 __all__ = ['PackageBase',
            'Package',
            'CMakePackage',
@@ -165,6 +187,9 @@
 from spack.util.executable import *
 __all__ += spack.util.executable.__all__
 
+# User's editor from the environment
+editor = Executable(os.environ.get("EDITOR", "vi"))
+
 from spack.package import \
     install_dependency_symlinks, flatten_dependencies, \
     DependencyConflictError, InstallError, ExternalPackageError

lib/spack/spack/cmd/common/arguments.py

@@ -95,3 +95,21 @@ def __call__(self, parser, namespace, values, option_string=None):
         'help': 'Recursively traverse spec dependencies'
     })
 _arguments['recurse_dependencies'] = parms
+
+parms = Bunch(
+    flags=('--clean',),
+    kwargs={
+        'action': 'store_false',
+        'dest': 'dirty',
+        'help': 'Clean environment before installing package.'
+    })
+_arguments['clean'] = parms
+
+parms = Bunch(
+    flags=('--dirty',),
+    kwargs={
+        'action': 'store_true',
+        'dest': 'dirty',
+        'help': 'Do NOT clean environment before installing.'
+    })
+_arguments['dirty'] = parms

lib/spack/spack/cmd/diy.py

@@ -30,6 +30,7 @@
 
 import spack
 import spack.cmd
+import spack.cmd.common.arguments as arguments
 from spack.cmd.edit import edit_package
 from spack.stage import DIYStage
 
@@ -52,9 +53,9 @@ def setup_parser(subparser):
     subparser.add_argument(
         'spec', nargs=argparse.REMAINDER,
         help="specs to use for install.  Must contain package AND version.")
-    subparser.add_argument(
+
-        '--dirty', action='store_true', dest='dirty',
+    cd_group = subparser.add_mutually_exclusive_group()
-        help="Install a package *without* cleaning the environment.")
+    arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
 
 
 def diy(self, args):

lib/spack/spack/cmd/install.py

@@ -34,6 +34,7 @@
 import llnl.util.tty as tty
 import spack
 import spack.cmd
+import spack.cmd.common.arguments as arguments
 from spack.build_environment import InstallError
 from spack.fetch_strategy import FetchError
 from spack.package import PackageBase
@@ -70,9 +71,10 @@ def setup_parser(subparser):
     subparser.add_argument(
         '--fake', action='store_true', dest='fake',
         help="Fake install. Just remove prefix and create a fake file.")
-    subparser.add_argument(
+
-        '--dirty', action='store_true', dest='dirty',
+    cd_group = subparser.add_mutually_exclusive_group()
-        help="Install a package *without* cleaning the environment.")
+    arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
+
     subparser.add_argument(
         'package',
         nargs=argparse.REMAINDER,

lib/spack/spack/cmd/purge.py

@@ -36,8 +36,8 @@ def setup_parser(subparser):
         '-d', '--downloads', action='store_true',
         help="Remove cached downloads.")
     subparser.add_argument(
-        '-u', '--user-cache', action='store_true',
+        '-m', '--misc-cache', action='store_true',
-        help="Remove caches in user home directory. Includes virtual indices.")
+        help="Remove long-lived caches, like the virtual package index.")
     subparser.add_argument(
         '-a', '--all', action='store_true',
         help="Remove all of the above.")
@@ -45,7 +45,7 @@ def setup_parser(subparser):
 
 def purge(parser, args):
     # Special case: no flags.
-    if not any((args.stage, args.downloads, args.user_cache, args.all)):
+    if not any((args.stage, args.downloads, args.misc_cache, args.all)):
         stage.purge()
         return
 
@@ -54,5 +54,5 @@ def purge(parser, args):
         stage.purge()
     if args.downloads or args.all:
         spack.fetch_cache.destroy()
-    if args.user_cache or args.all:
+    if args.misc_cache or args.all:
-        spack.user_cache.destroy()
+        spack.misc_cache.destroy()

lib/spack/spack/cmd/setup.py

@@ -32,6 +32,7 @@
 import spack
 import spack.cmd
 import spack.cmd.install as install
+import spack.cmd.common.arguments as arguments
 from llnl.util.filesystem import set_executable
 from spack import which
 from spack.cmd.edit import edit_package
@@ -50,9 +51,9 @@ def setup_parser(subparser):
     subparser.add_argument(
         'spec', nargs=argparse.REMAINDER,
         help="specs to use for install.  Must contain package AND version.")
-    subparser.add_argument(
+
-        '--dirty', action='store_true', dest='dirty',
+    cd_group = subparser.add_mutually_exclusive_group()
-        help="Install a package *without* cleaning the environment.")
+    arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
 
 
 def spack_transitive_include_path():

lib/spack/spack/config.py

@@ -298,6 +298,9 @@ def clear(self):
         """Empty cached config information."""
         self.sections = {}
 
+    def __repr__(self):
+        return '<ConfigScope: %s: %s>' % (self.name, self.path)
+
 #
 # Below are configuration scopes.
 #

lib/spack/spack/file_cache.py

@@ -113,7 +113,7 @@ def read_transaction(self, key):
         Returns a ReadTransaction context manager and opens the cache file for
         reading.  You can use it like this:
 
-           with spack.user_cache.read_transaction(key) as cache_file:
+           with file_cache_object.read_transaction(key) as cache_file:
                cache_file.read()
 
         """

lib/spack/spack/modules.py

@@ -675,7 +675,7 @@ class LmodModule(EnvModule):
     def __init__(self, spec=None):
         super(LmodModule, self).__init__(spec)
 
-        self.configuration = CONFIGURATION.get('lmod', {})
+        self.configuration = _module_config.get('lmod', {})
         hierarchy_tokens = self.configuration.get('hierarchical_scheme', [])
         # TODO : Check that the extra hierarchy tokens specified in the
         # TODO : configuration file are actually virtual dependencies

lib/spack/spack/package.py

@@ -1118,7 +1118,7 @@ def do_install(self,
                    run_tests=False,
                    fake=False,
                    explicit=False,
-                   dirty=False,
+                   dirty=None,
                    **kwargs):
         """Called by commands to install a package and its dependencies.
 
@@ -1165,6 +1165,10 @@ def do_install(self,
                         rec.explicit = True
                 return
 
+        # Dirty argument takes precedence over dirty config setting.
+        if dirty is None:
+            dirty = spack.dirty
+
         self._do_install_pop_kwargs(kwargs)
 
         tty.msg("Installing %s" % self.name)

lib/spack/spack/repository.py

@@ -620,12 +620,12 @@ def read():
 
         # Read the old ProviderIndex, or make a new one.
         key = self._cache_file
-        index_existed = spack.user_cache.init_entry(key)
+        index_existed = spack.misc_cache.init_entry(key)
         if index_existed and not self._needs_update:
-            with spack.user_cache.read_transaction(key) as f:
+            with spack.misc_cache.read_transaction(key) as f:
                 self._provider_index = ProviderIndex.from_yaml(f)
         else:
-            with spack.user_cache.write_transaction(key) as (old, new):
+            with spack.misc_cache.write_transaction(key) as (old, new):
                 if old:
                     self._provider_index = ProviderIndex.from_yaml(old)
                 else:
@@ -701,7 +701,7 @@ def _fast_package_check(self):
             self._all_package_names = []
 
             # Get index modification time.
-            index_mtime = spack.user_cache.mtime(self._cache_file)
+            index_mtime = spack.misc_cache.mtime(self._cache_file)
 
             for pkg_name in os.listdir(self.packages_path):
                 # Skip non-directories in the package root.

lib/spack/spack/test/config.py

@@ -221,7 +221,8 @@ def test_write_to_same_priority_file(self):
         self.check_config(b_comps, *self.b_comp_specs)
 
     def check_canonical(self, var, expected):
-        """ensure things are substituted properly and canonicalized."""
+        """Ensure that <expected> is substituted properly for <var> in strings
+           containing <var> in various positions."""
         path = '/foo/bar/baz'
 
         self.assertEqual(canonicalize_path(var + path),

lib/spack/spack/test/mock_packages_test.py

@@ -191,7 +191,7 @@
   misc_cache: ~/.spack/cache
   verify_ssl: true
   checksum: true
-  dirty: false
+  dirty: True
 """
 
 # these are written out to mock config files.
@@ -211,9 +211,6 @@ def initmock(self):
         self.db = RepoPath(spack.mock_packages_path)
         spack.repo.swap(self.db)
 
-        spack.config.clear_config_caches()
-        self.real_scopes = spack.config.config_scopes
-
         # Mock up temporary configuration directories
         self.temp_config = tempfile.mkdtemp()
         self.mock_site_config = os.path.join(self.temp_config, 'site')
@@ -227,6 +224,9 @@ def initmock(self):
 
         # TODO: Mocking this up is kind of brittle b/c ConfigScope
         # TODO: constructor modifies config_scopes.  Make it cleaner.
+        spack.config.clear_config_caches()
+        self.real_scopes = spack.config.config_scopes
+
         spack.config.config_scopes = OrderedDict()
         spack.config.ConfigScope('site', self.mock_site_config)
         spack.config.ConfigScope('user', self.mock_user_config)
@@ -261,6 +261,7 @@ def cleanmock(self):
         """Restore the real packages path after any test."""
         spack.repo.swap(self.db)
         spack.config.config_scopes = self.real_scopes
+
         shutil.rmtree(self.temp_config, ignore_errors=True)
         spack.config.clear_config_caches()
 

lib/spack/spack/test/modules.py

@@ -471,7 +471,7 @@ def test_no_hash(self):
         # Make sure that virtual providers (in the hierarchy) always
         # include a hash. Make sure that the module file for the spec
         # does not include a hash if hash_length is 0.
-        spack.modules.CONFIGURATION = self.configuration_no_hash
+        spack.modules._module_config = self.configuration_no_hash
         spec = spack.spec.Spec(mpileaks_spec_string)
         spec.concretize()
         module = spack.modules.LmodModule(spec)

lib/spack/spack/test/stage.py

@@ -46,9 +46,16 @@ def use_tmp(use_tmp):
        not use temporary space for stages.
     """
     # mock up config
-    path = _test_tmp_path if use_tmp else spack.stage_path
+    assert(_test_tmp_path is not None)
+
+    if use_tmp:
+        path = _test_tmp_path    # use temporary stage
+    else:
+        path = spack.stage_path  # Use Spack's stage dir (no links)
+
     spack.config.update_config(
         'config', {'build_stage': [path]}, scope='user')
+
     yield
 
 
@@ -59,13 +66,28 @@ def setUp(self):
            by the Stage class.  It doesn't actually create the Stage -- that
            is done by individual tests.
         """
+        super(StageTest, self).setUp()
+
         global _test_tmp_path
 
+        #
+        # Mock up a stage area that looks like this:
+        #
+        # TMPDIR/                    test_files_dir
+        #     tmp/                   test_tmp_path (where stage should be)
+        #     test-files/            archive_dir_path
+        #         README.txt         test_readme (contains "hello world!\n")
+        #     test-files.tar.gz      archive_url = file:///path/to/this
+        #
         self.test_files_dir = tempfile.mkdtemp()
         self.test_tmp_path  = os.path.realpath(
             os.path.join(self.test_files_dir, 'tmp'))
         _test_tmp_path = self.test_tmp_path
 
+        # set _test_tmp_path as the default test directory to use for stages.
+        spack.config.update_config(
+            'config', {'build_stage': [_test_tmp_path]}, scope='user')
+
         self.archive_dir = 'test-files'
         self.archive_name = self.archive_dir + '.tar.gz'
         archive_dir_path = os.path.join(self.test_files_dir,
@@ -99,6 +121,8 @@ def setUp(self):
 
     def tearDown(self):
         """Blows away the test environment directory."""
+        super(StageTest, self).tearDown()
+
         shutil.rmtree(self.test_files_dir, ignore_errors=True)
 
         # chdir back to original working dir
@@ -138,7 +162,7 @@ def check_setup(self, stage, stage_name):
             self.assertFalse(os.path.islink(target))
 
             # Make sure the directory is in the place we asked it to
-            # be (see setUp and tearDown)
+            # be (see setUp, tearDown, and use_tmp)
             self.assertTrue(target.startswith(self.test_tmp_path))
 
         else:

lib/spack/spack/util/path.py

@@ -50,18 +50,22 @@ def substitute_config_variables(path):
     - $spack     The Spack instance's prefix
     - $user      The current user's username
     - $tempdir   Default temporary directory returned by tempfile.gettempdir()
+
+    These are substituted case-insensitively into the path, and users can
+    use either ``$var`` or ``${var}`` syntax for the variables.
+
     """
     # Look up replacements for re.sub in the replacements dict.
     def repl(match):
         m = match.group(0).strip('${}')
-        return replacements.get(m, match.group(0))
+        return replacements.get(m.lower(), match.group(0))
 
     # Replace $var or ${var}.
     return re.sub(r'(\$\w+\b|\$\{\w+\})', repl, path)
 
 
 def canonicalize_path(path):
-    """Substitute $spack, expand user home, take abspath."""
+    """Substitute config vars, expand user home, take abspath."""
     path = substitute_config_variables(path)
     path = os.path.expanduser(path)
     path = os.path.abspath(path)