zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

source_cache, misc_cache, verify_ssl, checksum, & dirty in config.yaml

Browse Source 
- Added new preferences to config.yaml:
  - source_cache
  - misc_cache
  - verify_ssl
  - checksum
  - dirty
This commit is contained in:
Todd Gamblin 2016-10-28 00:57:51 -07:00
parent 22b14e0f23
commit da760a66de
17 changed files with 164 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
etc/spack/defaults/config.yaml
View File

@ -35,6 +35,8 @@ config:
# A value of $spack/var/spack/stage indicates that Spack should run
# builds directly inside its install directory without staging them in
# temporary space.
#
# The build stage can be purged with `spack purge --stage`.
build_stage:
- $tempdir
- /nfs/tmp2/$user
@ -42,11 +44,12 @@ config:
# Cache directory already downloaded source tarballs and archived
# repositories. This can be purged with spack purge
# repositories. This can be purged with `spack purge --downloads`.
source_cache: $spack/var/spack/cache
# Cache directory for miscellaneous files, like the package index.
# This can be purged with `spack purge --misc-cache`
misc_cache: ~/.spack/cache

115
lib/spack/spack/__init__.py
View File

@ -30,7 +30,10 @@
from llnl.util.filesystem import *
import llnl.util.tty as tty
# This lives in $prefix/lib/spack/spack/__file__
#-----------------------------------------------------------------------------
# Variables describing how Spack is laid out in its prefix.
#-----------------------------------------------------------------------------
# This file lives in $prefix/lib/spack/spack/__file__
spack_root = ancestor(__file__, 4)
# The spack script itself
@ -49,82 +52,100 @@
stage_path = join_path(var_path, "stage")
repos_path = join_path(var_path, "repos")
share_path = join_path(spack_root, "share", "spack")
cache_path = join_path(var_path, "cache")
# Paths to built-in Spack repositories.
packages_path = join_path(repos_path, "builtin")
mock_packages_path = join_path(repos_path, "builtin.mock")
# User configuration location
user_config_path = os.path.expanduser('~/.spack')
import spack.fetch_strategy
fetch_cache = spack.fetch_strategy.FsCache(cache_path)
from spack.file_cache import FileCache
user_cache_path = join_path(user_config_path, 'cache')
user_cache = FileCache(user_cache_path)
prefix = spack_root
opt_path = join_path(prefix, "opt")
etc_path = join_path(prefix, "etc")
#
# Set up the default packages database.
#
#-----------------------------------------------------------------------------
# Initial imports (only for use in this file -- see __all__ below.)
#-----------------------------------------------------------------------------
# These imports depend on the paths above, or on each other
# Group them here so it's easy to understand the order.
# TODO: refactor this stuff to be more init order agnostic.
import spack.repository
import spack.error
import spack.config
import spack.fetch_strategy
from spack.file_cache import FileCache
from spack.preferred_packages import PreferredPackages
from spack.abi import ABI
from spack.concretize import DefaultConcretizer
from spack.version import Version
from spack.util.path import canonicalize_path
#-----------------------------------------------------------------------------
# Initialize various data structures & objects at the core of Spack.
#-----------------------------------------------------------------------------
# Version information
spack_version = Version("0.9.1")
# Set up the default packages database.
try:
repo = spack.repository.RepoPath()
sys.meta_path.append(repo)
except spack.error.SpackError, e:
tty.die('while initializing Spack RepoPath:', e.message)
#
# Paths to built-in Spack repositories.
#
packages_path = join_path(repos_path, "builtin")
mock_packages_path = join_path(repos_path, "builtin.mock")
#
# This controls how packages are sorted when trying to choose
# the most preferred package. More preferred packages are sorted
# first.
#
from spack.preferred_packages import PreferredPackages
# PreferredPackages controls preference sort order during concretization.
# More preferred packages are sorted first.
pkgsort = PreferredPackages()
#
# This tests ABI compatibility between packages
#
from spack.abi import ABI
# Tests ABI compatibility between packages
abi = ABI()
#
# This controls how things are concretized in spack.
# Replace it with a subclass if you want different
# policies.
#
from spack.concretize import DefaultConcretizer
concretizer = DefaultConcretizer()
# Version information
from spack.version import Version
spack_version = Version("0.9.1")
#-----------------------------------------------------------------------------
# config.yaml options
#-----------------------------------------------------------------------------
_config = spack.config.get_config('config')
#
# Executables used by Spack
#
from spack.util.executable import Executable, which
# User's editor from the environment
editor = Executable(os.environ.get("EDITOR", "vi"))
# Path where downloaded source code is cached
cache_path = canonicalize_path(
_config.get('source_cache', join_path(var_path, "cache")))
fetch_cache = spack.fetch_strategy.FsCache(cache_path)
# cache for miscellaneous stuff.
misc_cache_path = canonicalize_path(
_config.get('misc_cache', join_path(user_config_path, 'cache')))
misc_cache = FileCache(misc_cache_path)
# If this is enabled, tools that use SSL should not verify
# certifiates. e.g., curl should use the -k option.
insecure = False
insecure = not _config.get('verify_ssl', True)
# Whether spack should allow installation of unsafe versions of
# software. "Unsafe" versions are ones it doesn't have a checksum
# for.
do_checksum = True
#
# Whether spack should allow installation of unsafe versions of software.
# "Unsafe" versions are ones it doesn't have a checksum for.
do_checksum = _config.get('checksum', True)
# If this is True, spack will not clean the environment to remove
# potentially harmful variables before builds.
dirty = _config.get('dirty', False)
#-----------------------------------------------------------------------------
# When packages call 'from spack import *', this extra stuff is brought in.
#
# Spack internal code should call 'import spack' and accesses other
@ -135,6 +156,7 @@
# packages should live. This file is overloaded for spack core vs.
# for packages.
#
#-----------------------------------------------------------------------------
__all__ = ['PackageBase',
'Package',
'CMakePackage',
@ -165,6 +187,9 @@
from spack.util.executable import *
__all__ += spack.util.executable.__all__
# User's editor from the environment
editor = Executable(os.environ.get("EDITOR", "vi"))
from spack.package import \
install_dependency_symlinks, flatten_dependencies, \
DependencyConflictError, InstallError, ExternalPackageError

18
lib/spack/spack/cmd/common/arguments.py
View File

@ -95,3 +95,21 @@ def __call__(self, parser, namespace, values, option_string=None):
'help': 'Recursively traverse spec dependencies'
})
_arguments['recurse_dependencies'] = parms
parms = Bunch(
flags=('--clean',),
kwargs={
'action': 'store_false',
'dest': 'dirty',
'help': 'Clean environment before installing package.'
})
_arguments['clean'] = parms
parms = Bunch(
flags=('--dirty',),
kwargs={
'action': 'store_true',
'dest': 'dirty',
'help': 'Do NOT clean environment before installing.'
})
_arguments['dirty'] = parms

7
lib/spack/spack/cmd/diy.py
View File

@ -30,6 +30,7 @@
import spack
import spack.cmd
import spack.cmd.common.arguments as arguments
from spack.cmd.edit import edit_package
from spack.stage import DIYStage
@ -52,9 +53,9 @@ def setup_parser(subparser):
subparser.add_argument(
'spec', nargs=argparse.REMAINDER,
help="specs to use for install. Must contain package AND version.")
subparser.add_argument(
'--dirty', action='store_true', dest='dirty',
help="Install a package *without* cleaning the environment.")
cd_group = subparser.add_mutually_exclusive_group()
arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
def diy(self, args):

8
lib/spack/spack/cmd/install.py
View File

@ -34,6 +34,7 @@
import llnl.util.tty as tty
import spack
import spack.cmd
import spack.cmd.common.arguments as arguments
from spack.build_environment import InstallError
from spack.fetch_strategy import FetchError
from spack.package import PackageBase
@ -70,9 +71,10 @@ def setup_parser(subparser):
subparser.add_argument(
'--fake', action='store_true', dest='fake',
help="Fake install. Just remove prefix and create a fake file.")
subparser.add_argument(
'--dirty', action='store_true', dest='dirty',
help="Install a package *without* cleaning the environment.")
cd_group = subparser.add_mutually_exclusive_group()
arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
subparser.add_argument(
'package',
nargs=argparse.REMAINDER,

10
lib/spack/spack/cmd/purge.py
View File

@ -36,8 +36,8 @@ def setup_parser(subparser):
'-d', '--downloads', action='store_true',
help="Remove cached downloads.")
subparser.add_argument(
'-u', '--user-cache', action='store_true',
help="Remove caches in user home directory. Includes virtual indices.")
'-m', '--misc-cache', action='store_true',
help="Remove long-lived caches, like the virtual package index.")
subparser.add_argument(
'-a', '--all', action='store_true',
help="Remove all of the above.")
@ -45,7 +45,7 @@ def setup_parser(subparser):
def purge(parser, args):
# Special case: no flags.
if not any((args.stage, args.downloads, args.user_cache, args.all)):
if not any((args.stage, args.downloads, args.misc_cache, args.all)):
stage.purge()
return
@ -54,5 +54,5 @@ def purge(parser, args):
stage.purge()
if args.downloads or args.all:
spack.fetch_cache.destroy()
if args.user_cache or args.all:
spack.user_cache.destroy()
if args.misc_cache or args.all:
spack.misc_cache.destroy()

7
lib/spack/spack/cmd/setup.py
View File

@ -32,6 +32,7 @@
import spack
import spack.cmd
import spack.cmd.install as install
import spack.cmd.common.arguments as arguments
from llnl.util.filesystem import set_executable
from spack import which
from spack.cmd.edit import edit_package
@ -50,9 +51,9 @@ def setup_parser(subparser):
subparser.add_argument(
'spec', nargs=argparse.REMAINDER,
help="specs to use for install. Must contain package AND version.")
subparser.add_argument(
'--dirty', action='store_true', dest='dirty',
help="Install a package *without* cleaning the environment.")
cd_group = subparser.add_mutually_exclusive_group()
arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
def spack_transitive_include_path():

3
lib/spack/spack/config.py
View File

@ -298,6 +298,9 @@ def clear(self):
"""Empty cached config information."""
self.sections = {}
def __repr__(self):
return '<ConfigScope: %s: %s>' % (self.name, self.path)
#
# Below are configuration scopes.
#

2
lib/spack/spack/file_cache.py
View File

@ -113,7 +113,7 @@ def read_transaction(self, key):
Returns a ReadTransaction context manager and opens the cache file for
reading. You can use it like this:
with spack.user_cache.read_transaction(key) as cache_file:
with file_cache_object.read_transaction(key) as cache_file:
cache_file.read()
"""

2
lib/spack/spack/modules.py
View File

@ -675,7 +675,7 @@ class LmodModule(EnvModule):
def __init__(self, spec=None):
super(LmodModule, self).__init__(spec)
self.configuration = CONFIGURATION.get('lmod', {})
self.configuration = _module_config.get('lmod', {})
hierarchy_tokens = self.configuration.get('hierarchical_scheme', [])
# TODO : Check that the extra hierarchy tokens specified in the
# TODO : configuration file are actually virtual dependencies

6
lib/spack/spack/package.py
View File

@ -1118,7 +1118,7 @@ def do_install(self,
run_tests=False,
fake=False,
explicit=False,
dirty=False,
dirty=None,
**kwargs):
"""Called by commands to install a package and its dependencies.
@ -1165,6 +1165,10 @@ def do_install(self,
rec.explicit = True
return
# Dirty argument takes precedence over dirty config setting.
if dirty is None:
dirty = spack.dirty
self._do_install_pop_kwargs(kwargs)
tty.msg("Installing %s" % self.name)

8
lib/spack/spack/repository.py
View File

@ -620,12 +620,12 @@ def read():
# Read the old ProviderIndex, or make a new one.
key = self._cache_file
index_existed = spack.user_cache.init_entry(key)
index_existed = spack.misc_cache.init_entry(key)
if index_existed and not self._needs_update:
with spack.user_cache.read_transaction(key) as f:
with spack.misc_cache.read_transaction(key) as f:
self._provider_index = ProviderIndex.from_yaml(f)
else:
with spack.user_cache.write_transaction(key) as (old, new):
with spack.misc_cache.write_transaction(key) as (old, new):
if old:
self._provider_index = ProviderIndex.from_yaml(old)
else:
@ -701,7 +701,7 @@ def _fast_package_check(self):
self._all_package_names = []
# Get index modification time.
index_mtime = spack.user_cache.mtime(self._cache_file)
index_mtime = spack.misc_cache.mtime(self._cache_file)
for pkg_name in os.listdir(self.packages_path):
# Skip non-directories in the package root.

3
lib/spack/spack/test/config.py
View File

@ -221,7 +221,8 @@ def test_write_to_same_priority_file(self):
self.check_config(b_comps, *self.b_comp_specs)
def check_canonical(self, var, expected):
"""ensure things are substituted properly and canonicalized."""
"""Ensure that <expected> is substituted properly for <var> in strings
containing <var> in various positions."""
path = '/foo/bar/baz'
self.assertEqual(canonicalize_path(var + path),

9
lib/spack/spack/test/mock_packages_test.py
View File

@ -191,7 +191,7 @@
misc_cache: ~/.spack/cache
verify_ssl: true
checksum: true
dirty: false
dirty: True
"""
# these are written out to mock config files.
@ -211,9 +211,6 @@ def initmock(self):
self.db = RepoPath(spack.mock_packages_path)
spack.repo.swap(self.db)
spack.config.clear_config_caches()
self.real_scopes = spack.config.config_scopes
# Mock up temporary configuration directories
self.temp_config = tempfile.mkdtemp()
self.mock_site_config = os.path.join(self.temp_config, 'site')
@ -227,6 +224,9 @@ def initmock(self):
# TODO: Mocking this up is kind of brittle b/c ConfigScope
# TODO: constructor modifies config_scopes. Make it cleaner.
spack.config.clear_config_caches()
self.real_scopes = spack.config.config_scopes
spack.config.config_scopes = OrderedDict()
spack.config.ConfigScope('site', self.mock_site_config)
spack.config.ConfigScope('user', self.mock_user_config)
@ -261,6 +261,7 @@ def cleanmock(self):
"""Restore the real packages path after any test."""
spack.repo.swap(self.db)
spack.config.config_scopes = self.real_scopes
shutil.rmtree(self.temp_config, ignore_errors=True)
spack.config.clear_config_caches()

2
lib/spack/spack/test/modules.py
View File

@ -471,7 +471,7 @@ def test_no_hash(self):
# Make sure that virtual providers (in the hierarchy) always
# include a hash. Make sure that the module file for the spec
# does not include a hash if hash_length is 0.
spack.modules.CONFIGURATION = self.configuration_no_hash
spack.modules._module_config = self.configuration_no_hash
spec = spack.spec.Spec(mpileaks_spec_string)
spec.concretize()
module = spack.modules.LmodModule(spec)

28
lib/spack/spack/test/stage.py
View File

@ -46,9 +46,16 @@ def use_tmp(use_tmp):
not use temporary space for stages.
"""
# mock up config
path = _test_tmp_path if use_tmp else spack.stage_path
assert(_test_tmp_path is not None)
if use_tmp:
path = _test_tmp_path # use temporary stage
else:
path = spack.stage_path # Use Spack's stage dir (no links)
spack.config.update_config(
'config', {'build_stage': [path]}, scope='user')
yield
@ -59,13 +66,28 @@ def setUp(self):
by the Stage class. It doesn't actually create the Stage -- that
is done by individual tests.
"""
super(StageTest, self).setUp()
global _test_tmp_path
#
# Mock up a stage area that looks like this:
#
# TMPDIR/ test_files_dir
# tmp/ test_tmp_path (where stage should be)
# test-files/ archive_dir_path
# README.txt test_readme (contains "hello world!\n")
# test-files.tar.gz archive_url = file:///path/to/this
#
self.test_files_dir = tempfile.mkdtemp()
self.test_tmp_path = os.path.realpath(
os.path.join(self.test_files_dir, 'tmp'))
_test_tmp_path = self.test_tmp_path
# set _test_tmp_path as the default test directory to use for stages.
spack.config.update_config(
'config', {'build_stage': [_test_tmp_path]}, scope='user')
self.archive_dir = 'test-files'
self.archive_name = self.archive_dir + '.tar.gz'
archive_dir_path = os.path.join(self.test_files_dir,
@ -99,6 +121,8 @@ def setUp(self):
def tearDown(self):
"""Blows away the test environment directory."""
super(StageTest, self).tearDown()
shutil.rmtree(self.test_files_dir, ignore_errors=True)
# chdir back to original working dir
@ -138,7 +162,7 @@ def check_setup(self, stage, stage_name):
self.assertFalse(os.path.islink(target))
# Make sure the directory is in the place we asked it to
# be (see setUp and tearDown)
# be (see setUp, tearDown, and use_tmp)
self.assertTrue(target.startswith(self.test_tmp_path))
else:

8
lib/spack/spack/util/path.py
View File

@ -50,18 +50,22 @@ def substitute_config_variables(path):
- $spack The Spack instance's prefix
- $user The current user's username
- $tempdir Default temporary directory returned by tempfile.gettempdir()
These are substituted case-insensitively into the path, and users can
use either ``$var`` or ``${var}`` syntax for the variables.
"""
# Look up replacements for re.sub in the replacements dict.
def repl(match):
m = match.group(0).strip('${}')
return replacements.get(m, match.group(0))
return replacements.get(m.lower(), match.group(0))
# Replace $var or ${var}.
return re.sub(r'(\$\w+\b|\$\{\w+\})', repl, path)
def canonicalize_path(path):
"""Substitute $spack, expand user home, take abspath."""
"""Substitute config vars, expand user home, take abspath."""
path = substitute_config_variables(path)
path = os.path.expanduser(path)
path = os.path.abspath(path)