zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

config: allow user to add configuration scopes on the command line.

Browse Source 
- Add command-line scope option to Spack

- Rework structure of main to allow configuration system to raise
  errors more naturally

Co-authored-by: Todd Gamblin <tgamblin@llnl.gov>
This commit is contained in:
Elizabeth Fischer
2016-12-29 20:06:21 -05:00
committed by Todd Gamblin
parent 2b0d944341
commit 52fbbdf5a1
4 changed files with 194 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
lib/spack/spack/config.py
View File

@@ -22,6 +22,8 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from __future__ import print_function
"""This module implements Spack's configuration file handling.
This implements Spack's configuration system, which handles merging
@@ -206,6 +208,19 @@ def __repr__(self):
return '<ConfigScope: %s: %s>' % (self.name, self.path)
class ImmutableConfigScope(ConfigScope):
"""A configuration scope that cannot be written to.
This is used for ConfigScopes passed on the command line.
"""
def write_section(self, section):
raise ConfigError("Cannot write to immutable scope %s" % self)
def __repr__(self):
return '<ImmutableConfigScope: %s: %s>' % (self.name, self.path)
class InternalConfigScope(ConfigScope):
"""An internal configuration scope that is not persisted to a file.
@@ -274,9 +289,8 @@ def pop_scope(self):
@property
def file_scopes(self):
"""List of scopes with an associated file (non-internal scopes)."""
return [s for s in self.scopes.values()
if not isinstance(s, InternalConfigScope)]
"""List of writable scopes with an associated file."""
return [s for s in self.scopes.values() if type(s) == ConfigScope]
def highest_precedence_scope(self):
"""Non-internal scope with highest precedence."""
@@ -455,17 +469,65 @@ def print_section(self, section, blame=False):
@contextmanager
def override(path, value):
"""Simple way to override config settings within a context."""
overrides = InternalConfigScope('overrides')
def override(path_or_scope, value=None):
"""Simple way to override config settings within a context.
config.push_scope(overrides)
config.set(path, value, scope='overrides')
Arguments:
path_or_scope (ConfigScope or str): scope or single option to override
value (object, optional): value for the single option
yield config
Temporarily push a scope on the current configuration, then remove it
after the context completes. If a single option is provided, create
an internal config scope for it and push/pop that scope.
scope = config.pop_scope()
assert scope is overrides
"""
if isinstance(path_or_scope, ConfigScope):
config.push_scope(path_or_scope)
yield config
config.pop_scope(path_or_scope)
else:
overrides = InternalConfigScope('overrides')
config.push_scope(overrides)
config.set(path_or_scope, value, scope='overrides')
yield config
scope = config.pop_scope()
assert scope is overrides
#: configuration scopes added on the command line
#: set by ``spack.main.main()``.
command_line_scopes = []
def _add_platform_scope(cfg, scope_type, name, path):
"""Add a platform-specific subdirectory for the current platform."""
platform = spack.architecture.platform().name
plat_name = '%s/%s' % (name, platform)
plat_path = os.path.join(path, platform)
cfg.push_scope(scope_type(plat_name, plat_path))
def _add_command_line_scopes(cfg, command_line_scopes):
"""Add additional scopes from the --config-scope argument.
Command line scopes are named after their position in the arg list.
"""
for i, path in enumerate(command_line_scopes):
# We ensure that these scopes exist and are readable, as they are
# provided on the command line by the user.
if not os.path.isdir(path):
raise ConfigError("config scope is not a directory: '%s'" % path)
elif not os.access(path, os.R_OK):
raise ConfigError("config scope is not readable: '%s'" % path)
# name based on order on the command line
name = 'cmd_scope_%d' % i
cfg.push_scope(ImmutableConfigScope(name, path))
_add_platform_scope(cfg, ImmutableConfigScope, name, path)
def _config():
@@ -485,16 +547,15 @@ def _config():
defaults = InternalConfigScope('_builtin', config_defaults)
cfg.push_scope(defaults)
# Each scope can have per-platfom overrides in subdirectories
platform = spack.architecture.platform().name
# add each scope and its platform-specific directory
for name, path in configuration_paths:
cfg.push_scope(ConfigScope(name, path))
plat_name = '%s/%s' % (name, platform)
plat_path = os.path.join(path, platform)
cfg.push_scope(ConfigScope(plat_name, plat_path))
# Each scope can have per-platfom overrides in subdirectories
_add_platform_scope(cfg, ConfigScope, name, path)
# add command-line scopes
_add_command_line_scopes(cfg, command_line_scopes)
# we make a special scope for spack commands so that they can
# override configuration options.

8
lib/spack/spack/error.py
View File

@@ -30,6 +30,11 @@
import llnl.util.tty as tty
#: whether we should write stack traces or short error messages
#: this is module-scoped because it needs to be set very early
debug = False
class SpackError(Exception):
"""This is the superclass for all Spack errors.
Subclasses can be found in the modules they have to do with.
@@ -72,8 +77,7 @@ def print_context(self):
sys.stderr.write('\n')
# stack trace, etc. in debug mode.
import spack.config
if spack.config.get('config:debug'):
if debug:
if self.traceback:
# exception came from a build child, already got
# traceback in child, so print it.

113
lib/spack/spack/main.py
View File

@@ -292,7 +292,9 @@ def add_command(self, cmd_name):
subparser = self.subparsers.add_parser(
cmd_name, help=module.description, description=module.description)
module.setup_parser(subparser)
return module
# return the callable function for the command
return spack.cmd.get_command(cmd_name)
def format_help(self, level='short'):
if self.prog == 'spack':
@@ -328,6 +330,9 @@ def make_argument_parser(**kwargs):
'--color', action='store', default='auto',
choices=('always', 'never', 'auto'),
help="when to colorize output (default: auto)")
parser.add_argument(
'-C', '--config-scope', dest='config_scopes', action='append',
metavar='DIRECTORY', help="use an additional configuration scope")
parser.add_argument(
'-d', '--debug', action='store_true',
help="write out debug logs during compile")
@@ -379,15 +384,18 @@ def setup_main_options(args):
tty.set_debug(args.debug)
tty.set_stacktrace(args.stacktrace)
# debug must be set first so that it can even affect behvaior of
# errors raised by spack.config.
if args.debug:
spack.error.debug = True
spack.util.debug.register_interrupt_handler()
spack.config.set('config:debug', True, scope='command_line')
# override lock configuration if passed on command line
if args.locks is not None:
spack.util.lock.check_lock_safety(spack.paths.prefix)
spack.config.set('config:locks', False, scope='command_line')
if args.debug:
spack.util.debug.register_interrupt_handler()
spack.config.set('config:debug', True, scope='command_line')
if args.mock:
rp = spack.repo.RepoPath(spack.paths.mock_packages_path)
spack.repo.set_path(rp)
@@ -414,7 +422,7 @@ def allows_unknown_args(command):
return (argcount == 3 and varnames[2] == 'unknown_args')
def _invoke_spack_command(command, parser, args, unknown_args):
def _invoke_command(command, parser, args, unknown_args):
"""Run a spack command *without* setting spack global options."""
if allows_unknown_args(command):
return_val = command(parser, args, unknown_args)
@@ -438,16 +446,15 @@ class SpackCommand(object):
Use this to invoke Spack commands directly from Python and check
their output.
"""
def __init__(self, command):
"""Create a new SpackCommand that invokes ``command`` when called.
def __init__(self, command_name):
"""Create a new SpackCommand that invokes ``command_name`` when called.
Args:
command (str): name of the command to invoke
command_name (str): name of the command to invoke
"""
self.parser = make_argument_parser()
self.parser.add_command(command)
self.command_name = command
self.command = spack.cmd.get_command(command)
self.command = self.parser.add_command(command_name)
self.command_name = command_name
def __call__(self, *argv, **kwargs):
"""Invoke this SpackCommand.
@@ -477,7 +484,7 @@ def __call__(self, *argv, **kwargs):
out = StringIO()
try:
with log_output(out):
self.returncode = _invoke_spack_command(
self.returncode = _invoke_command(
self.command, self.parser, args, unknown)
except SystemExit as e:
@@ -497,30 +504,6 @@ def __call__(self, *argv, **kwargs):
return out.getvalue()
def _main(command, parser, args, unknown_args):
"""Run a spack command *and* set spack globaloptions."""
# many operations will fail without a working directory.
set_working_dir()
# only setup main options in here, after the real parse (we'll get it
# wrong if we do it after the initial, partial parse)
setup_main_options(args)
spack.hooks.pre_run()
# Now actually execute the command
try:
return _invoke_spack_command(command, parser, args, unknown_args)
except SpackError as e:
e.die() # gracefully die on any SpackErrors
except Exception as e:
if spack.config.get('config:debug'):
raise
tty.die(str(e))
except KeyboardInterrupt:
sys.stderr.write('\n')
tty.die("Keyboard interrupt.")
def _profile_wrapper(command, parser, args, unknown_args):
import cProfile
@@ -543,7 +526,7 @@ def _profile_wrapper(command, parser, args, unknown_args):
# make a profiler and run the code.
pr = cProfile.Profile()
pr.enable()
return _main(command, parser, args, unknown_args)
return _invoke_command(command, parser, args, unknown_args)
finally:
pr.disable()
@@ -609,6 +592,10 @@ def main(argv=None):
parser.add_argument('command', nargs=argparse.REMAINDER)
args, unknown = parser.parse_known_args(argv)
# make spack.config aware of any command line configuration scopes
if args.config_scopes:
spack.config.command_line_scopes = args.config_scopes
if args.print_shell_vars:
print_setup_info(*args.print_shell_vars.split(','))
return 0
@@ -631,31 +618,51 @@ def main(argv=None):
parser.print_help()
return 1
# Try to load the particular command the caller asked for. If there
# is no module for it, just die.
cmd_name = args.command[0]
try:
parser.add_command(cmd_name)
except ImportError:
if spack.config.get('config:debug'):
raise
tty.die("Unknown command: %s" % args.command[0])
# ensure options on spack command come before everything
setup_main_options(args)
# Re-parse with the proper sub-parser added.
args, unknown = parser.parse_known_args()
# Try to load the particular command the caller asked for. If there
# is no module for it, just die.
cmd_name = args.command[0]
try:
command = parser.add_command(cmd_name)
except ImportError:
if spack.config.get('config:debug'):
raise
tty.die("Unknown command: %s" % args.command[0])
# now we can actually execute the command.
command = spack.cmd.get_command(cmd_name)
try:
# Re-parse with the proper sub-parser added.
args, unknown = parser.parse_known_args()
# many operations will fail without a working directory.
set_working_dir()
# pre-run hooks happen after we know we have a valid working dir
spack.hooks.pre_run()
# now we can actually execute the command.
if args.spack_profile or args.sorted_profile:
_profile_wrapper(command, parser, args, unknown)
elif args.pdb:
import pdb
pdb.runctx('_main(command, parser, args, unknown)',
pdb.runctx('_invoke_command(command, parser, args, unknown)',
globals(), locals())
return 0
else:
return _main(command, parser, args, unknown)
return _invoke_command(command, parser, args, unknown)
except SpackError as e:
e.die() # gracefully die on any SpackErrors
except Exception as e:
if spack.config.get('config:debug'):
raise
tty.die(str(e))
except KeyboardInterrupt:
sys.stderr.write('\n')
tty.die("Keyboard interrupt.")
except SystemExit as e:
return e.code

50
lib/spack/spack/test/config.py
View File

@@ -27,6 +27,8 @@
import getpass
import tempfile
from llnl.util.filesystem import touch, mkdirp
import pytest
import yaml
@@ -614,3 +616,51 @@ def test_bad_config_section(config):
with pytest.raises(spack.config.ConfigSectionError):
spack.config.get('foobar')
def test_bad_command_line_scopes(tmpdir, config):
cfg = spack.config.Configuration()
with tmpdir.as_cwd():
with pytest.raises(spack.config.ConfigError):
spack.config._add_command_line_scopes(cfg, ['bad_path'])
touch('unreadable_file')
with pytest.raises(spack.config.ConfigError):
spack.config._add_command_line_scopes(cfg, ['unreadable_file'])
mkdirp('unreadable_dir')
with pytest.raises(spack.config.ConfigError):
try:
os.chmod('unreadable_dir', 0)
spack.config._add_command_line_scopes(cfg, ['unreadable_dir'])
finally:
os.chmod('unreadable_dir', 0o700) # so tmpdir can be removed
def test_add_command_line_scopes(tmpdir, mutable_config):
config_yaml = str(tmpdir.join('config.yaml'))
with open(config_yaml, 'w') as f:
f.write("""\
config:
verify_ssh: False
dirty: False
"""'')
spack.config._add_command_line_scopes(mutable_config, [str(tmpdir)])
def test_immuntable_scope(tmpdir):
config_yaml = str(tmpdir.join('config.yaml'))
with open(config_yaml, 'w') as f:
f.write("""\
config:
install_tree: dummy_tree_value
"""'')
scope = spack.config.ImmutableConfigScope('test', str(tmpdir))
data = scope.get_section('config')
assert data['config']['install_tree'] == 'dummy_tree_value'
with pytest.raises(spack.config.ConfigError):
scope.write_section('config')