zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

stacks: add option to link only roots or all specs, default to all

Browse Source
This commit is contained in:
Gregory Becker
2019-07-03 15:17:07 -07:00
committed by Todd Gamblin
parent 7ec89aa036
commit ec6e5b0fd3
3 changed files with 134 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
lib/spack/spack/environment.py
View File

@@ -78,8 +78,12 @@
env_schema_keys = ('spack', 'env')
# Magic names
# The name of the standalone spec list in the manifest yaml
user_speclist_name = 'specs'
def_view_name = 'default'
# The name of the default view (the view loaded on env.activate)
default_view_name = 'default'
# Default behavior to link all packages into views (vs. only root packages)
default_view_link = 'all'
def valid_env_name(name):
@@ -145,7 +149,7 @@ def activate(
cmds += 'export SPACK_OLD_PS1="${PS1}"; fi;\n'
cmds += 'export PS1="%s ${PS1}";\n' % prompt
if add_view and def_view_name in env.views:
if add_view and default_view_name in env.views:
cmds += env.add_default_view_to_shell(shell)
return cmds
@@ -187,7 +191,7 @@ def deactivate(shell='sh'):
cmds += 'unset SPACK_OLD_PS1; export SPACK_OLD_PS1;\n'
cmds += 'fi;\n'
if def_view_name in _active_environment.views:
if default_view_name in _active_environment.views:
cmds += _active_environment.rm_default_view_from_shell(shell)
tty.debug("Deactivated environmennt '%s'" % _active_environment.name)
@@ -414,7 +418,8 @@ def _eval_conditional(string):
class ViewDescriptor(object):
def __init__(self, root, projections={}, select=[], exclude=[]):
def __init__(self, root, projections={}, select=[], exclude=[],
link=default_view_link):
self.root = root
self.projections = projections
self.select = select
@@ -422,6 +427,7 @@ def __init__(self, root, projections={}, select=[], exclude=[]):
self.exclude = exclude
self.exclude_fn = lambda x: not any(x.satisfies(e)
for e in self.exclude)
self.link = link
def to_dict(self):
ret = {'root': self.root}
@@ -431,6 +437,8 @@ def to_dict(self):
ret['select'] = self.select
if self.exclude:
ret['exclude'] = self.exclude
if self.link != default_view_link:
ret['link'] = self.link
return ret
@staticmethod
@@ -438,19 +446,22 @@ def from_dict(d):
return ViewDescriptor(d['root'],
d.get('projections', {}),
d.get('select', []),
d.get('exclude', []))
d.get('exclude', []),
d.get('link', default_view_link))
def view(self):
return YamlFilesystemView(self.root, spack.store.layout,
ignore_conflicts=True,
projections=self.projections)
def regenerate(self, specs):
def regenerate(self, all_specs, roots):
specs_for_view = []
specs = all_specs if self.link == 'all' else roots
for spec in specs:
# The view does not store build deps, so if we want it to
# recognize environment specs (which do store build deps), then
# they need to be stripped
if spec.concrete: # Do not link unconcretized roots
specs_for_view.append(spack.spec.Spec.from_dict(
spec.to_dict(all_deps=False)
))
@@ -523,9 +534,9 @@ def __init__(self, path, init_file=None, with_view=None):
self.views = {}
elif with_view is True:
self.views = {
def_view_name: ViewDescriptor(self.view_path_default)}
default_view_name: ViewDescriptor(self.view_path_default)}
elif isinstance(with_view, six.string_types):
self.views = {def_view_name: ViewDescriptor(with_view)}
self.views = {default_view_name: ViewDescriptor(with_view)}
# If with_view is None, then defer to the view settings determined by
# the manifest file
@@ -556,9 +567,9 @@ def _read_manifest(self, f):
# enable_view can be boolean, string, or None
if enable_view is True or enable_view is None:
self.views = {
def_view_name: ViewDescriptor(self.view_path_default)}
default_view_name: ViewDescriptor(self.view_path_default)}
elif isinstance(enable_view, six.string_types):
self.views = {def_view_name: ViewDescriptor(enable_view)}
self.views = {default_view_name: ViewDescriptor(enable_view)}
elif enable_view:
self.views = dict((name, ViewDescriptor.from_dict(values))
for name, values in enable_view.items())
@@ -881,23 +892,24 @@ def default_view(self):
raise SpackEnvironmentError(
"{0} does not have a view enabled".format(self.name))
if def_view_name not in self.views:
if default_view_name not in self.views:
raise SpackEnvironmentError(
"{0} does not have a default view enabled".format(self.name))
return self.views[def_view_name]
return self.views[default_view_name]
def update_default_view(self, viewpath):
if def_view_name in self.views and self.default_view.root != viewpath:
name = default_view_name
if name in self.views and self.default_view.root != viewpath:
shutil.rmtree(self.default_view.root)
if viewpath:
if def_view_name in self.views:
if name in self.views:
self.default_view.root = viewpath
else:
self.views[def_view_name] = ViewDescriptor(viewpath)
self.views[name] = ViewDescriptor(viewpath)
else:
self.views.pop(def_view_name, None)
self.views.pop(name, None)
def regenerate_views(self):
if not self.views:
@@ -907,7 +919,7 @@ def regenerate_views(self):
specs = self._get_environment_specs()
for view in self.views.values():
view.regenerate(specs)
view.regenerate(specs, self.roots())
def _shell_vars(self):
updates = [
@@ -921,7 +933,7 @@ def _shell_vars(self):
('CMAKE_PREFIX_PATH', ['']),
]
path_updates = list()
if def_view_name in self.views:
if default_view_name in self.views:
for var, subdirs in updates:
paths = filter(lambda x: os.path.exists(x),
list(os.path.join(self.default_view.root, x)
@@ -1198,7 +1210,8 @@ def write(self):
[])
yaml_spec_list[:] = self.user_specs.yaml_list
if self.views and len(self.views) == 1 and def_view_name in self.views:
default_name = default_view_name
if self.views and len(self.views) == 1 and default_name in self.views:
path = self.default_view.root
if self.default_view == ViewDescriptor(self.view_path_default):
view = True

4
lib/spack/spack/schema/env.py
View File

@@ -96,6 +96,10 @@
'root': {
'type': 'string'
},
'link': {
'type': 'string',
'pattern': '(roots|all)',
},
'select': {
'type': 'array',
'items': {

110
lib/spack/spack/test/cmd/env.py
View File

@@ -32,10 +32,9 @@
find = SpackCommand('find')
def check_mpileaks_install(viewdir):
def check_mpileaks_and_deps_in_view(viewdir):
"""Check that the expected install directories exist."""
assert os.path.exists(str(viewdir.join('.spack', 'mpileaks')))
# Check that dependencies got in too
assert os.path.exists(str(viewdir.join('.spack', 'libdwarf')))
@@ -634,7 +633,7 @@ def test_env_updates_view_install(
add('mpileaks')
install('--fake')
check_mpileaks_install(view_dir)
check_mpileaks_and_deps_in_view(view_dir)
def test_env_without_view_install(
@@ -656,7 +655,7 @@ def test_env_without_view_install(
# After enabling the view, the specs should be linked into the environment
# view dir
check_mpileaks_install(view_dir)
check_mpileaks_and_deps_in_view(view_dir)
def test_env_config_view_default(
@@ -686,7 +685,7 @@ def test_env_updates_view_install_package(
with ev.read('test'):
install('--fake', 'mpileaks')
check_mpileaks_install(view_dir)
assert os.path.exists(str(view_dir.join('.spack/mpileaks')))
def test_env_updates_view_add_concretize(
@@ -698,7 +697,7 @@ def test_env_updates_view_add_concretize(
add('mpileaks')
concretize()
check_mpileaks_install(view_dir)
check_mpileaks_and_deps_in_view(view_dir)
def test_env_updates_view_uninstall(
@@ -708,7 +707,7 @@ def test_env_updates_view_uninstall(
with ev.read('test'):
install('--fake', 'mpileaks')
check_mpileaks_install(view_dir)
check_mpileaks_and_deps_in_view(view_dir)
with ev.read('test'):
uninstall('-ay')
@@ -725,7 +724,7 @@ def test_env_updates_view_uninstall_referenced_elsewhere(
add('mpileaks')
concretize()
check_mpileaks_install(view_dir)
check_mpileaks_and_deps_in_view(view_dir)
with ev.read('test'):
uninstall('-ay')
@@ -742,7 +741,7 @@ def test_env_updates_view_remove_concretize(
add('mpileaks')
concretize()
check_mpileaks_install(view_dir)
check_mpileaks_and_deps_in_view(view_dir)
with ev.read('test'):
remove('mpileaks')
@@ -758,7 +757,7 @@ def test_env_updates_view_force_remove(
with ev.read('test'):
install('--fake', 'mpileaks')
check_mpileaks_install(view_dir)
check_mpileaks_and_deps_in_view(view_dir)
with ev.read('test'):
remove('-f', 'mpileaks')
@@ -1123,7 +1122,7 @@ def test_stack_combinatorial_view(tmpdir, mock_fetch, mock_packages,
install()
test = ev.read('test')
for _, spec in test.concretized_specs():
for spec in test._get_environment_specs():
assert os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
@@ -1156,7 +1155,7 @@ def test_stack_view_select(tmpdir, mock_fetch, mock_packages,
install()
test = ev.read('test')
for _, spec in test.concretized_specs():
for spec in test._get_environment_specs():
if spec.satisfies('%gcc'):
assert os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
@@ -1194,7 +1193,7 @@ def test_stack_view_exclude(tmpdir, mock_fetch, mock_packages,
install()
test = ev.read('test')
for _, spec in test.concretized_specs():
for spec in test._get_environment_specs():
if not spec.satisfies('callpath'):
assert os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
@@ -1233,7 +1232,88 @@ def test_stack_view_select_and_exclude(tmpdir, mock_fetch, mock_packages,
install()
test = ev.read('test')
for _, spec in test.concretized_specs():
for spec in test._get_environment_specs():
if spec.satisfies('%gcc') and not spec.satisfies('callpath'):
assert os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
else:
assert not os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
def test_view_link_roots(tmpdir, mock_fetch, mock_packages, mock_archive,
install_mockery):
filename = str(tmpdir.join('spack.yaml'))
viewdir = str(tmpdir.join('view'))
with open(filename, 'w') as f:
f.write("""\
env:
definitions:
- packages: [mpileaks, callpath]
- compilers: ['%%gcc', '%%clang']
specs:
- matrix:
- [$packages]
- [$compilers]
view:
combinatorial:
root: %s
select: ['%%gcc']
exclude: [callpath]
link: 'roots'
projections:
'all': '{name}/{version}-{compiler.name}'""" % viewdir)
with tmpdir.as_cwd():
env('create', 'test', './spack.yaml')
with ev.read('test'):
install()
test = ev.read('test')
for spec in test._get_environment_specs():
if spec in test.roots() and (spec.satisfies('%gcc') and
not spec.satisfies('callpath')):
assert os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
else:
assert not os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
def test_view_link_all(tmpdir, mock_fetch, mock_packages, mock_archive,
install_mockery):
filename = str(tmpdir.join('spack.yaml'))
viewdir = str(tmpdir.join('view'))
with open(filename, 'w') as f:
f.write("""\
env:
definitions:
- packages: [mpileaks, callpath]
- compilers: ['%%gcc', '%%clang']
specs:
- matrix:
- [$packages]
- [$compilers]
view:
combinatorial:
root: %s
select: ['%%gcc']
exclude: [callpath]
link: 'all'
projections:
'all': '{name}/{version}-{compiler.name}'""" % viewdir)
with tmpdir.as_cwd():
env('create', 'test', './spack.yaml')
with ev.read('test'):
install()
test = ev.read('test')
for spec in test._get_environment_specs():
if spec.satisfies('%gcc') and not spec.satisfies('callpath'):
assert os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
@@ -1342,7 +1422,7 @@ def test_stack_view_multiple_views(tmpdir, mock_fetch, mock_packages,
assert os.path.join(default_viewdir, 'bin') in shell
test = ev.read('test')
for _, spec in test.concretized_specs():
for spec in test._get_environment_specs():
if not spec.satisfies('callpath%gcc'):
assert os.path.exists(
os.path.join(combin_viewdir, spec.name, '%s-%s' %