zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

stacks: update view management for multiple/combinatorial views

Browse Source 
This adds notion of a default view, other views in environments
This commit is contained in:
Gregory Becker
2019-04-15 19:07:45 -07:00
committed by Todd Gamblin
parent d0bfe0d6a8
commit cebf1fd668
9 changed files with 406 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lib/spack/spack/cmd/env.py
View File

@@ -315,11 +315,11 @@ def env_view(args):
if args.view_path:
view_path = args.view_path
else:
view_path = env.default_view_path
env.update_view(view_path)
view_path = env.create_view_path
env.update_default_view(view_path)
env.write()
elif args.action == ViewAction.disable:
env.update_view(None)
env.update_default_view(None)
env.write()
else:
tty.msg("No active environment")

139
lib/spack/spack/environment.py
View File

@@ -141,7 +141,7 @@ def activate(
cmds += 'export SPACK_OLD_PS1="${PS1}"; fi;\n'
cmds += 'export PS1="%s ${PS1}";\n' % prompt
if add_view and env._view_path:
if add_view and env.default_view_path:
cmds += env.add_view_to_shell(shell)
return cmds
@@ -183,7 +183,7 @@ def deactivate(shell='sh'):
cmds += 'unset SPACK_OLD_PS1; export SPACK_OLD_PS1;\n'
cmds += 'fi;\n'
if _active_environment._view_path:
if _active_environment.default_view_path:
cmds += _active_environment.rm_view_from_shell(shell)
tty.debug("Deactivated environmennt '%s'" % _active_environment.name)
@@ -467,9 +467,9 @@ def __init__(self, path, init_file=None, with_view=None):
self._read_manifest(default_manifest_yaml)
if with_view is False:
self._view_path = None
self.views = None
elif isinstance(with_view, six.string_types):
self._view_path = with_view
self.views = {'default': self.create_view_descriptor(with_view)}
# If with_view is None, then defer to the view settings determined by
# the manifest file
@@ -499,27 +499,18 @@ def _read_manifest(self, f):
enable_view = config_dict(self.yaml).get('view')
# enable_view can be boolean, string, or None
if enable_view is True or enable_view is None:
self._view_path = self.default_view_path
self.views = {'default': self.create_view_descriptor()}
elif isinstance(enable_view, six.string_types):
self._view_path = enable_view
self.views = {'default': self.create_view_descriptor(enable_view)}
elif enable_view:
self.views = enable_view
else:
self._view_path = None
self.views = None
@property
def user_specs(self):
return self.read_specs['specs']
enable_view = config_dict(self.yaml).get('view')
# enable_view can be true/false, a string, or None (if the manifest did
# not specify it)
if enable_view is True or enable_view is None:
self._view_path = self.default_view_path
elif isinstance(enable_view, six.string_types):
self._view_path = enable_view
else:
# enable_view is False
self._view_path = None
def _set_user_specs_from_lockfile(self):
"""Copy user_specs from a read-in lockfile."""
self.read_specs = {
@@ -582,8 +573,13 @@ def repos_path(self):
def log_path(self):
return os.path.join(self.path, env_subdir_name, 'logs')
def create_view_descriptor(self, root=None):
if root is None:
root = self.create_view_path
return {'root': root, 'projections': {}}
@property
def default_view_path(self):
def create_view_path(self):
return os.path.join(self.env_subdir_path, 'view')
@property
@@ -824,26 +820,62 @@ def _install(self, spec, **install_args):
os.remove(build_log_link)
os.symlink(spec.package.build_log_path, build_log_link)
def view(self):
if not self._view_path:
@property
def all_views(self):
if not self.views:
raise SpackEnvironmentError(
"{0} does not have a view enabled".format(self.name))
return YamlFilesystemView(
self._view_path, spack.store.layout, ignore_conflicts=True)
return [YamlFilesystemView(view['root'], spack.store.layout,
ignore_conflicts=True,
projections=view['projections'])
for view in self.views.values()]
def update_view(self, view_path):
if self._view_path and self._view_path != view_path:
shutil.rmtree(self._view_path)
@property
def default_view(self):
if not self.views:
raise SpackEnvironmentError(
"{0} does not have a view enabled".format(self.name))
self._view_path = view_path
if 'default' not in self.views:
raise SpackEnvironmentError(
"{0} does not have a default view enabled".format(self.name))
def regenerate_view(self):
if not self._view_path:
return YamlFilesystemView(self.default_view_path,
spack.store.layout, ignore_confligs=True)
@property
def default_view_path(self):
if not self.views or 'default' not in self.views:
return None
return self.views['default']['root']
def update_default_view(self, viewpath):
if self.default_view_path and self.default_view_path != viewpath:
shutil.rmtree(self.default_view_path)
if viewpath:
if self.default_view_path:
self.views['default']['root'] = viewpath
elif self.views:
self.views['default'] = self.create_view_descriptor(viewpath)
else:
self.views = {'default': self.create_view_descriptor(viewpath)}
else:
self.views.pop('default', None)
def regenerate_views(self):
if not self.views:
tty.debug("Skip view update, this environment does not"
" maintain a view")
return
for name, view in zip(self.views.keys(), self.all_views):
self.regenerate_view(name, view)
def regenerate_view(self, name, view):
view_descriptor = self.views[name]
specs_for_view = []
for spec in self._get_environment_specs():
# The view does not store build deps, so if we want it to
@@ -852,13 +884,23 @@ def regenerate_view(self):
specs_for_view.append(spack.spec.Spec.from_dict(
spec.to_dict(all_deps=False)
))
select = view_descriptor.get('select', [])
if select:
select_fn = lambda x: any(x.satisfies(s) for s in select)
specs_for_view = list(filter(select_fn, specs_for_view))
exclude = view_descriptor.get('exclude', [])
if exclude:
exclude_fn = lambda x: not any(x.satisfies(e) for e in exclude)
specs_for_view = list(filter(exclude_fn, specs_for_view))
installed_specs_for_view = set(s for s in specs_for_view
if s.package.installed)
view = self.view()
view.clean()
specs_in_view = set(view.get_all_specs())
tty.msg("Updating view at {0}".format(self._view_path))
tty.msg("Updating view at {0}".format(view_descriptor['root']))
rm_specs = specs_in_view - installed_specs_for_view
view.remove_specs(*rm_specs, with_dependents=False)
@@ -880,7 +922,7 @@ def _shell_vars(self):
path_updates = list()
for var, subdirs in updates:
paths = filter(lambda x: os.path.exists(x),
list(os.path.join(self._view_path, x)
list(os.path.join(self.default_view_path, x)
for x in subdirs))
path_updates.append((var, paths))
return path_updates
@@ -945,7 +987,7 @@ def install_all(self, args=None):
os.remove(build_log_link)
os.symlink(spec.package.build_log_path, build_log_link)
self.regenerate_view()
self.regenerate_views()
def all_specs_by_hash(self):
"""Map of hashes to spec for all specs in this environment."""
@@ -1014,8 +1056,7 @@ def _get_environment_specs(self, recurse_dependencies=True):
If these specs appear under different user_specs, only one copy
is added to the list returned.
"""
package_to_spec = {}
spec_list = list()
spec_set = set()
for spec_hash in self.concretized_order:
spec = self.specs_by_hash[spec_hash]
@@ -1023,17 +1064,9 @@ def _get_environment_specs(self, recurse_dependencies=True):
specs = (spec.traverse(deptype=('link', 'run'))
if recurse_dependencies else (spec,))
for dep in specs:
prior = package_to_spec.get(dep.name)
if prior and prior != dep:
tty.debug("{0} takes priority over {1}"
.format(package_to_spec[dep.name].format(),
dep.format()))
else:
package_to_spec[dep.name] = dep
spec_list.append(dep)
spec_set.update(specs)
return spec_list
return list(spec_set)
def _to_lockfile_dict(self):
"""Create a dictionary to store a lockfile for this environment."""
@@ -1158,10 +1191,16 @@ def write(self):
yaml_spec_list = config_dict(self.yaml).setdefault('specs', [])
yaml_spec_list[:] = self.user_specs.yaml_list
if self._view_path == self.default_view_path:
view = True
elif self._view_path:
view = self._view_path
if self.views and len(self.views) == 1 and self.default_view_path:
path = self.default_view_path
if self.views['default'] == self.create_view_descriptor():
view = True
elif self.views['default'] == self.create_view_descriptor(path):
view = path
else:
view = self.views
elif self.views:
view = self.views
else:
view = False
@@ -1179,7 +1218,7 @@ def write(self):
# TODO: for operations that just add to the env (install etc.) this
# could just call update_view
self.regenerate_view()
self.regenerate_views()
def __enter__(self):
self._previous_active = _active_environment

13
lib/spack/spack/filesystem_view.py
View File

@@ -210,9 +210,16 @@ def __init__(self, root, layout, **kwargs):
with open(projections_path, 'w') as f:
f.write(s_yaml.dump({'projections': self.projections}))
else:
msg = 'View at %s has projections file' % self._root
msg += ' and was passed projections manually.'
raise ConflictingProjectionsError(msg)
# Ensure projections are the same from each source
# Read projections file from view
with open(projections_path, 'r') as f:
projections_data = s_yaml.load(f)
spack.config.validate(projections_data,
spack.schema.projections.schema)
if self.projections != projections_data['projections']:
msg = 'View at %s has projections file' % self._root
msg += ' which does not match projections passed manually.'
raise ConflictingProjectionsError(msg)
self.extensions_layout = YamlViewExtensionsLayout(self, layout)

47
lib/spack/spack/schema/env.py
View File

@@ -67,9 +67,6 @@
'type': 'string'
},
},
'view': {
'type': ['boolean', 'string']
},
'definitions': {
'type': 'array',
'default': [],
@@ -81,7 +78,7 @@
}
},
'patternProperties': {
'^(?!when$)\w*': spec_list_schema
r'^(?!when$)\w*': spec_list_schema
}
}
},
@@ -91,25 +88,29 @@
{'type': 'boolean'},
{'type': 'string'},
{'type': 'object',
'required': ['root'],
'additionalProperties': False,
'properties': {
'root': {
'type': 'string'
},
'select': {
'type': 'array',
'items': {
'type': 'string'
}
},
'exclude': {
'type': 'array',
'items': {
'type': 'string'
}
},
'projections': projections_scheme
'patternProperties': {
r'\w+': {
'required': ['root'],
'additionalProperties': False,
'properties': {
'root': {
'type': 'string'
},
'select': {
'type': 'array',
'items': {
'type': 'string'
}
},
'exclude': {
'type': 'array',
'items': {
'type': 'string'
}
},
'projections': projections_scheme
}
}
}
}
]

267
lib/spack/spack/test/cmd/env.py
View File

@@ -45,6 +45,13 @@ def check_viewdir_removal(viewdir):
os.listdir(str(viewdir.join('.spack'))) == ['projections.yaml'])
@pytest.fixture()
def env_deactivate():
yield
spack.environment._active_environment = None
os.environ.pop('SPACK_ENV', None)
def test_add():
e = ev.create('test')
e.add('mpileaks')
@@ -637,7 +644,7 @@ def test_env_without_view_install(
test_env = ev.read('test')
with pytest.raises(spack.environment.SpackEnvironmentError):
test_env.view()
test_env.default_view
view_dir = tmpdir.mkdir('view')
@@ -668,7 +675,7 @@ def test_env_config_view_default(
e = ev.read('test')
# Try retrieving the view object
view = e.view()
view = e.default_view
assert view.get_spec('mpileaks')
@@ -1088,3 +1095,259 @@ def test_stack_definition_conditional_add_write(tmpdir):
assert 'callpath' in packages_lists[1]['packages']
assert 'zmpi' in packages_lists[0]['packages']
assert 'zmpi' not in packages_lists[1]['packages']
def test_stack_combinatorial_view(tmpdir, mock_fetch, mock_packages,
mock_archive, install_mockery):
filename = str(tmpdir.join('spack.yaml'))
viewdir = str(tmpdir.join('view'))
with open(filename, 'w') as f:
f.write("""\
env:
definitions:
- packages: [mpileaks, callpath]
- compilers: ['%%gcc', '%%clang']
specs:
- matrix:
- [$packages]
- [$compilers]
view:
combinatorial:
root: %s
projections:
'all': '${package}/${version}-${compilername}'""" % viewdir)
with tmpdir.as_cwd():
env('create', 'test', './spack.yaml')
with ev.read('test'):
install()
test = ev.read('test')
for _, spec in test.concretized_specs():
assert os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
def test_stack_view_select(tmpdir, mock_fetch, mock_packages,
mock_archive, install_mockery):
filename = str(tmpdir.join('spack.yaml'))
viewdir = str(tmpdir.join('view'))
with open(filename, 'w') as f:
f.write("""\
env:
definitions:
- packages: [mpileaks, callpath]
- compilers: ['%%gcc', '%%clang']
specs:
- matrix:
- [$packages]
- [$compilers]
view:
combinatorial:
root: %s
select: ['%%gcc']
projections:
'all': '${package}/${version}-${compilername}'""" % viewdir)
with tmpdir.as_cwd():
env('create', 'test', './spack.yaml')
with ev.read('test'):
install()
test = ev.read('test')
for _, spec in test.concretized_specs():
if spec.satisfies('%gcc'):
assert os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
else:
assert not os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
def test_stack_view_exclude(tmpdir, mock_fetch, mock_packages,
mock_archive, install_mockery):
filename = str(tmpdir.join('spack.yaml'))
viewdir = str(tmpdir.join('view'))
with open(filename, 'w') as f:
f.write("""\
env:
definitions:
- packages: [mpileaks, callpath]
- compilers: ['%%gcc', '%%clang']
specs:
- matrix:
- [$packages]
- [$compilers]
view:
combinatorial:
root: %s
exclude: [callpath]
projections:
'all': '${package}/${version}-${compilername}'""" % viewdir)
with tmpdir.as_cwd():
env('create', 'test', './spack.yaml')
with ev.read('test'):
install()
test = ev.read('test')
for _, spec in test.concretized_specs():
if not spec.satisfies('callpath'):
assert os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
else:
assert not os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
def test_stack_view_select_and_exclude(tmpdir, mock_fetch, mock_packages,
mock_archive, install_mockery):
filename = str(tmpdir.join('spack.yaml'))
viewdir = str(tmpdir.join('view'))
with open(filename, 'w') as f:
f.write("""\
env:
definitions:
- packages: [mpileaks, callpath]
- compilers: ['%%gcc', '%%clang']
specs:
- matrix:
- [$packages]
- [$compilers]
view:
combinatorial:
root: %s
select: ['%%gcc']
exclude: [callpath]
projections:
'all': '${package}/${version}-${compilername}'""" % viewdir)
with tmpdir.as_cwd():
env('create', 'test', './spack.yaml')
with ev.read('test'):
install()
test = ev.read('test')
for _, spec in test.concretized_specs():
if spec.satisfies('%gcc') and not spec.satisfies('callpath'):
assert os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
else:
assert not os.path.exists(
os.path.join(viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
def test_stack_view_activate_from_default(tmpdir, mock_fetch, mock_packages,
mock_archive, install_mockery,
env_deactivate):
filename = str(tmpdir.join('spack.yaml'))
viewdir = str(tmpdir.join('view'))
with open(filename, 'w') as f:
f.write("""\
env:
definitions:
- packages: [mpileaks, cmake]
- compilers: ['%%gcc', '%%clang']
specs:
- matrix:
- [$packages]
- [$compilers]
view:
default:
root: %s
select: ['%%gcc']""" % viewdir)
with tmpdir.as_cwd():
env('create', 'test', './spack.yaml')
with ev.read('test'):
install()
shell = env('activate', '--sh', 'test')
assert 'PATH' in shell
assert os.path.join(viewdir, 'bin') in shell
def test_stack_view_no_activate_without_default(tmpdir, mock_fetch,
mock_packages, mock_archive,
install_mockery,
env_deactivate):
filename = str(tmpdir.join('spack.yaml'))
viewdir = str(tmpdir.join('view'))
with open(filename, 'w') as f:
f.write("""\
env:
definitions:
- packages: [mpileaks, cmake]
- compilers: ['%%gcc', '%%clang']
specs:
- matrix:
- [$packages]
- [$compilers]
view:
not-default:
root: %s
select: ['%%gcc']""" % viewdir)
with tmpdir.as_cwd():
env('create', 'test', './spack.yaml')
with ev.read('test'):
install()
shell = env('activate', '--sh', 'test')
assert 'PATH' not in shell
assert viewdir not in shell
def test_stack_view_multiple_views(tmpdir, mock_fetch, mock_packages,
mock_archive, install_mockery,
env_deactivate):
filename = str(tmpdir.join('spack.yaml'))
default_viewdir = str(tmpdir.join('default-view'))
combin_viewdir = str(tmpdir.join('combinatorial-view'))
with open(filename, 'w') as f:
f.write("""\
env:
definitions:
- packages: [mpileaks, cmake]
- compilers: ['%%gcc', '%%clang']
specs:
- matrix:
- [$packages]
- [$compilers]
view:
default:
root: %s
select: ['%%gcc']
combinatorial:
root: %s
exclude: [callpath %%gcc]
projections:
'all': '${package}/${version}-${compilername}'""" % (default_viewdir,
combin_viewdir))
with tmpdir.as_cwd():
env('create', 'test', './spack.yaml')
with ev.read('test'):
install()
shell = env('activate', '--sh', 'test')
assert 'PATH' in shell
assert os.path.join(default_viewdir, 'bin') in shell
test = ev.read('test')
for _, spec in test.concretized_specs():
if not spec.satisfies('callpath%gcc'):
assert os.path.exists(
os.path.join(combin_viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))
else:
assert not os.path.exists(
os.path.join(combin_viewdir, spec.name, '%s-%s' %
(spec.version, spec.compiler.name)))

1
lib/spack/spack/test/concretize.py
View File

@@ -15,6 +15,7 @@
from spack.spec import ConflictsInSpecError, SpecError
from spack.version import ver
from spack.test.conftest import MockPackage, MockPackageMultiRepo
import spack.compilers
def check_spec(abstract, concrete):

12
lib/spack/spack/test/conftest.py
View File

@@ -109,6 +109,18 @@ def no_chdir():
assert os.getcwd() == original_wd
@pytest.fixture(scope='function', autouse=True)
def reset_compiler_cache():
"""Ensure that the compiler cache is not shared across Spack tests
This cache can cause later tests to fail if left in a state incompatible
with the new configuration. Since tests can make almost unlimited changes
to their setup, default to not use the compiler cache across tests."""
spack.compilers._compiler_cache = {}
yield
spack.compilers._compiler_cache = {}
@pytest.fixture(scope='session', autouse=True)
def mock_stage(tmpdir_factory):
"""Mocks up a fake stage directory for use by tests."""

2
var/spack/repos/builtin.mock/packages/mpich/package.py
View File

@@ -28,4 +28,4 @@ class Mpich(Package):
provides('mpi@:1', when='@:1')
def install(self, spec, prefix):
pass
touch(prefix.mpich)

2
var/spack/repos/builtin.mock/packages/mpileaks/package.py
View File

@@ -27,7 +27,7 @@ class Mpileaks(Package):
libs = None
def install(self, spec, prefix):
pass
touch(prefix.mpileaks)
def setup_environment(self, senv, renv):
renv.set('FOOBAR', self.name)