zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into features/shared

Browse Source
This commit is contained in:
Carson Woods
2020-01-27 15:28:28 -05:00
parent 7c1083916a 9ffa053f18
commit cdcd3dcedd
47 changed files with 559 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
lib/spack/docs/basic_usage.rst
View File

@@ -929,11 +929,13 @@ in GNU Autotools. If all flags are set, the order is
Compiler environment variables and additional RPATHs
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
In the exceptional case a compiler requires setting special environment
variables, like an explicit library load path. These can bet set in an
extra section in the compiler configuration (the supported environment
modification commands are: ``set``, ``unset``, ``append-path``, and
``prepend-path``). The user can also specify additional ``RPATHs`` that the
Sometimes compilers require setting special environment variables to
operate correctly. Spack handles these cases by allowing custom environment
modifications in the ``environment`` attribute of the compiler configuration
section. See also the :ref:`configuration_environment_variables` section
of the configuration files docs for more information.
It is also possible to specify additional ``RPATHs`` that the
compiler will add to all executables generated by that compiler. This is
useful for forcing certain compilers to RPATH their own runtime libraries, so
that executables will run without the need to set ``LD_LIBRARY_PATH``.
@@ -950,28 +952,19 @@ that executables will run without the need to set ``LD_LIBRARY_PATH``.
fc: /opt/gcc/bin/gfortran
environment:
unset:
BAD_VARIABLE: # The colon is required but the value must be empty
- BAD_VARIABLE
set:
GOOD_VARIABLE_NUM: 1
GOOD_VARIABLE_STR: good
prepend-path:
prepend_path:
PATH: /path/to/binutils
append-path:
append_path:
LD_LIBRARY_PATH: /opt/gcc/lib
extra_rpaths:
- /path/to/some/compiler/runtime/directory
- /path/to/some/other/compiler/runtime/directory
.. note::
The section `environment` is interpreted as an ordered dictionary, which
means two things. First, environment modification are applied in the order
they are specified in the configuration file. Second, you cannot express
environment modifications that require mixing different commands, i.e. you
cannot `set` one variable, than `prepend-path` to another one, and than
again `set` a third one.
^^^^^^^^^^^^^^^^^^^^^^^
Architecture specifiers
^^^^^^^^^^^^^^^^^^^^^^^

27
lib/spack/docs/configuration.rst
View File

@@ -427,6 +427,33 @@ home directory, and ``~user`` will expand to a specified user's home
directory. The ``~`` must appear at the beginning of the path, or Spack
will not expand it.
.. _configuration_environment_variables:
-------------------------
Environment Modifications
-------------------------
Spack allows to prescribe custom environment modifications in a few places
within its configuration files. Every time these modifications are allowed
they are specified as a dictionary, like in the following example:
.. code-block:: yaml
environment:
set:
LICENSE_FILE: '/path/to/license'
unset:
- CPATH
- LIBRARY_PATH
append_path:
PATH: '/new/bin/dir'
The possible actions that are permitted are ``set``, ``unset``, ``append_path``,
``prepend_path`` and finally ``remove_path``. They all require a dictionary
of variable names mapped to the values used for the modification.
The only exception is ``unset`` that requires just a list of variable names.
No particular order is ensured on the execution of each of these modifications.
----------------------------
Seeing Spack's Configuration
----------------------------

18
lib/spack/spack/binary_distribution.py
View File

@@ -663,7 +663,7 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
_cached_specs = None
def get_specs(force=False):
def get_specs(force=False, use_arch=False):
"""
Get spec.yaml's for build caches available on mirror
"""
@@ -690,7 +690,13 @@ def get_specs(force=False):
for file in files:
if re.search('spec.yaml', file):
link = url_util.join(fetch_url_build_cache, file)
urls.add(link)
if use_arch and re.search('%s-%s' %
(spack.architecture.platform,
spack.architecture.os),
file):
urls.add(link)
else:
urls.add(link)
else:
tty.msg("Finding buildcaches at %s" %
url_util.format(fetch_url_build_cache))
@@ -698,7 +704,13 @@ def get_specs(force=False):
url_util.join(fetch_url_build_cache, 'index.html'))
for link in links:
if re.search("spec.yaml", link):
urls.add(link)
if use_arch and re.search('%s-%s' %
(spack.architecture.platform,
spack.architecture.os),
link):
urls.add(link)
else:
urls.add(link)
_cached_specs = []
for link in urls:

18
lib/spack/spack/build_environment.py
View File

@@ -39,7 +39,6 @@
import sys
import traceback
import types
from six import iteritems
from six import StringIO
import llnl.util.tty as tty
@@ -52,6 +51,7 @@
import spack.config
import spack.main
import spack.paths
import spack.schema.environment
import spack.store
from spack.util.string import plural
from spack.util.environment import (
@@ -342,21 +342,7 @@ def set_build_environment_variables(pkg, env, dirty):
# Set environment variables if specified for
# the given compiler
compiler = pkg.compiler
environment = compiler.environment
for command, variable in iteritems(environment):
if command == 'set':
for name, value in iteritems(variable):
env.set(name, value)
elif command == 'unset':
for name, _ in iteritems(variable):
env.unset(name)
elif command == 'prepend-path':
for name, value in iteritems(variable):
env.prepend_path(name, value)
elif command == 'append-path':
for name, value in iteritems(variable):
env.append_path(name, value)
env.extend(spack.schema.environment.parse(compiler.environment))
if compiler.extra_rpaths:
extra_rpaths = ':'.join(compiler.extra_rpaths)

21
lib/spack/spack/cmd/config.py
View File

@@ -56,6 +56,8 @@ def setup_parser(subparser):
'--print-file', action='store_true',
help="print the file name that would be edited")
sp.add_parser('list', help='list configuration sections')
def _get_scope_and_section(args):
"""Extract config scope and section from arguments."""
@@ -83,7 +85,6 @@ def config_get(args):
With no arguments and an active environment, print the contents of
the environment's manifest file (spack.yaml).
"""
scope, section = _get_scope_and_section(args)
@@ -113,7 +114,6 @@ def config_edit(args):
With no arguments and an active environment, edit the spack.yaml for
the active environment.
"""
scope, section = _get_scope_and_section(args)
if not scope and not section:
@@ -127,8 +127,19 @@ def config_edit(args):
editor(config_file)
def config_list(args):
"""List the possible configuration sections.
Used primarily for shell tab completion scripts.
"""
print(' '.join(list(spack.config.section_schemas)))
def config(parser, args):
action = {'get': config_get,
'blame': config_blame,
'edit': config_edit}
action = {
'get': config_get,
'blame': config_blame,
'edit': config_edit,
'list': config_list,
}
action[args.config_command](args)

2
lib/spack/spack/cmd/list.py
View File

@@ -23,7 +23,7 @@
from spack.version import VersionList
if sys.version_info > (3, 1):
from html import escape
from html import escape # novm
else:
from cgi import escape

33
lib/spack/spack/modules/common.py
View File

@@ -28,26 +28,24 @@
Each of the four classes needs to be sub-classed when implementing a new
module type.
"""
import collections
import copy
import datetime
import inspect
import os.path
import re
import collections
import six
import llnl.util.filesystem
import llnl.util.tty as tty
import spack.paths
import spack.build_environment as build_environment
import spack.util.environment
import spack.tengine as tengine
import spack.util.path
import spack.util.environment
import spack.error
import spack.util.spack_yaml as syaml
import spack.paths
import spack.schema.environment
import spack.tengine as tengine
import spack.util.environment
import spack.util.file_permissions as fp
import spack.util.path
import spack.util.spack_yaml as syaml
#: config section for this file
@@ -417,22 +415,7 @@ def env(self):
"""List of environment modifications that should be done in the
module.
"""
env_mods = spack.util.environment.EnvironmentModifications()
actions = self.conf.get('environment', {})
def process_arglist(arglist):
if method == 'unset':
for x in arglist:
yield (x,)
else:
for x in six.iteritems(arglist):
yield x
for method, arglist in actions.items():
for args in process_arglist(arglist):
getattr(env_mods, method)(*args)
return env_mods
return spack.schema.environment.parse(self.conf.get('environment', {}))
@property
def suffixes(self):

2
lib/spack/spack/package.py
View File

@@ -1507,7 +1507,7 @@ def _update_explicit_entry_in_db(self, rec, explicit):
def try_install_from_binary_cache(self, explicit):
tty.msg('Searching for binary cache of %s' % self.name)
specs = binary_distribution.get_specs()
specs = binary_distribution.get_specs(use_arch=True)
binary_spec = spack.spec.Spec.from_dict(self.spec.to_dict())
binary_spec._mark_concrete()
if binary_spec not in specs:

47
lib/spack/spack/schema/compilers.py
View File

@@ -8,7 +8,7 @@
.. literalinclude:: _spack_root/lib/spack/spack/schema/compilers.py
:lines: 13-
"""
import spack.schema.environment
#: Properties for inclusion in other schemas
properties = {
@@ -68,50 +68,7 @@
{'type': 'boolean'}
]
},
'environment': {
'type': 'object',
'default': {},
'additionalProperties': False,
'properties': {
'set': {
'type': 'object',
'patternProperties': {
# Variable name
r'\w[\w-]*': {
'anyOf': [{'type': 'string'},
{'type': 'number'}]
}
}
},
'unset': {
'type': 'object',
'patternProperties': {
# Variable name
r'\w[\w-]*': {'type': 'null'}
}
},
'prepend-path': {
'type': 'object',
'patternProperties': {
# Variable name
r'\w[\w-]*': {
'anyOf': [{'type': 'string'},
{'type': 'number'}]
}
}
},
'append-path': {
'type': 'object',
'patternProperties': {
# Variable name
r'\w[\w-]*': {
'anyOf': [{'type': 'string'},
{'type': 'number'}]
}
}
}
}
},
'environment': spack.schema.environment.definition,
'extra_rpaths': {
'type': 'array',
'default': [],

58
lib/spack/spack/schema/environment.py Normal file
View File

@@ -0,0 +1,58 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Schema for environment modifications. Meant for inclusion in other
schemas.
"""
array_of_strings_or_num = {
'type': 'array', 'default': [], 'items':
{'anyOf': [{'type': 'string'}, {'type': 'number'}]}
}
dictionary_of_strings_or_num = {
'type': 'object', 'patternProperties':
{r'\w[\w-]*': {'anyOf': [{'type': 'string'}, {'type': 'number'}]}}
}
definition = {
'type': 'object',
'default': {},
'additionalProperties': False,
'properties': {
'set': dictionary_of_strings_or_num,
'unset': array_of_strings_or_num,
'prepend_path': dictionary_of_strings_or_num,
'append_path': dictionary_of_strings_or_num,
'remove_path': dictionary_of_strings_or_num
}
}
def parse(config_obj):
"""Returns an EnvironmentModifications object containing the modifications
parsed from input.
Args:
config_obj: a configuration dictionary conforming to the
schema definition for environment modifications
"""
import spack.util.environment as ev
try:
from collections import Sequence # novm
except ImportError:
from collections.abc import Sequence # novm
env = ev.EnvironmentModifications()
for command, variable in config_obj.items():
# Distinguish between commands that take only a name as argument
# (e.g. unset) and commands that take a name and a value.
if isinstance(variable, Sequence):
for name in variable:
getattr(env, command)(name)
else:
for name, value in variable.items():
getattr(env, command)(name, value)
return env

14
lib/spack/spack/schema/modules.py
View File

@@ -8,6 +8,8 @@
.. literalinclude:: _spack_root/lib/spack/spack/schema/modules.py
:lines: 13-
"""
import spack.schema.environment
#: Matches a spec or a multi-valued variant but not another
#: valid keyword.
@@ -66,17 +68,7 @@
}
}
},
'environment': {
'type': 'object',
'default': {},
'additionalProperties': False,
'properties': {
'set': dictionary_of_strings,
'unset': array_of_strings,
'prepend_path': dictionary_of_strings,
'append_path': dictionary_of_strings
}
}
'environment': spack.schema.environment.definition
}
}

130
lib/spack/spack/test/build_environment.py
View File

@@ -14,7 +14,6 @@
from spack.paths import build_env_path
from spack.build_environment import dso_suffix, _static_to_shared_library
from spack.util.executable import Executable
from spack.util.spack_yaml import syaml_dict, syaml_str
from spack.util.environment import EnvironmentModifications
from llnl.util.filesystem import LibraryList, HeaderList
@@ -65,6 +64,18 @@ def build_environment(working_env):
del os.environ[name]
@pytest.fixture
def ensure_env_variables(config, mock_packages, monkeypatch, working_env):
"""Returns a function that takes a dictionary and updates os.environ
for the test lifetime accordingly. Plugs-in mock config and repo.
"""
def _ensure(env_mods):
for name, value in env_mods.items():
monkeypatch.setenv(name, value)
return _ensure
def test_static_to_shared_library(build_environment):
os.environ['SPACK_TEST_COMMAND'] = 'dump-args'
@@ -119,79 +130,58 @@ def _set_wrong_cc(x):
assert os.environ['ANOTHER_VAR'] == 'THIS_IS_SET'
@pytest.mark.usefixtures('config', 'mock_packages')
def test_compiler_config_modifications(monkeypatch, working_env):
s = spack.spec.Spec('cmake')
s.concretize()
pkg = s.package
@pytest.mark.parametrize('initial,modifications,expected', [
# Set and unset variables
({'SOME_VAR_STR': '', 'SOME_VAR_NUM': '0'},
{'set': {'SOME_VAR_STR': 'SOME_STR', 'SOME_VAR_NUM': 1}},
{'SOME_VAR_STR': 'SOME_STR', 'SOME_VAR_NUM': '1'}),
({'SOME_VAR_STR': ''},
{'unset': ['SOME_VAR_STR']},
{'SOME_VAR_STR': None}),
({}, # Set a variable that was not defined already
{'set': {'SOME_VAR_STR': 'SOME_STR'}},
{'SOME_VAR_STR': 'SOME_STR'}),
# Append and prepend to the same variable
({'EMPTY_PATH_LIST': '/path/middle'},
{'prepend_path': {'EMPTY_PATH_LIST': '/path/first'},
'append_path': {'EMPTY_PATH_LIST': '/path/last'}},
{'EMPTY_PATH_LIST': '/path/first:/path/middle:/path/last'}),
# Append and prepend from empty variables
({'EMPTY_PATH_LIST': '', 'SOME_VAR_STR': ''},
{'prepend_path': {'EMPTY_PATH_LIST': '/path/first'},
'append_path': {'SOME_VAR_STR': '/path/last'}},
{'EMPTY_PATH_LIST': '/path/first', 'SOME_VAR_STR': '/path/last'}),
({}, # Same as before but on variables that were not defined
{'prepend_path': {'EMPTY_PATH_LIST': '/path/first'},
'append_path': {'SOME_VAR_STR': '/path/last'}},
{'EMPTY_PATH_LIST': '/path/first', 'SOME_VAR_STR': '/path/last'}),
# Remove a path from a list
({'EMPTY_PATH_LIST': '/path/first:/path/middle:/path/last'},
{'remove_path': {'EMPTY_PATH_LIST': '/path/middle'}},
{'EMPTY_PATH_LIST': '/path/first:/path/last'}),
({'EMPTY_PATH_LIST': '/only/path'},
{'remove_path': {'EMPTY_PATH_LIST': '/only/path'}},
{'EMPTY_PATH_LIST': ''}),
])
def test_compiler_config_modifications(
initial, modifications, expected, ensure_env_variables, monkeypatch
):
# Set the environment as per prerequisites
ensure_env_variables(initial)
os.environ['SOME_VAR_STR'] = ''
os.environ['SOME_VAR_NUM'] = '0'
os.environ['PATH_LIST'] = '/path/third:/path/forth'
os.environ['EMPTY_PATH_LIST'] = ''
os.environ.pop('NEW_PATH_LIST', None)
# Monkeypatch a pkg.compiler.environment with the required modifications
pkg = spack.spec.Spec('cmake').concretized().package
monkeypatch.setattr(pkg.compiler, 'environment', modifications)
env_mod = syaml_dict()
set_cmd = syaml_dict()
env_mod[syaml_str('set')] = set_cmd
set_cmd[syaml_str('SOME_VAR_STR')] = syaml_str('SOME_STR')
set_cmd[syaml_str('SOME_VAR_NUM')] = 1
monkeypatch.setattr(pkg.compiler, 'environment', env_mod)
# Trigger the modifications
spack.build_environment.setup_package(pkg, False)
assert os.environ['SOME_VAR_STR'] == 'SOME_STR'
assert os.environ['SOME_VAR_NUM'] == str(1)
env_mod = syaml_dict()
unset_cmd = syaml_dict()
env_mod[syaml_str('unset')] = unset_cmd
unset_cmd[syaml_str('SOME_VAR_STR')] = None
monkeypatch.setattr(pkg.compiler, 'environment', env_mod)
assert 'SOME_VAR_STR' in os.environ
spack.build_environment.setup_package(pkg, False)
assert 'SOME_VAR_STR' not in os.environ
env_mod = syaml_dict()
set_cmd = syaml_dict()
env_mod[syaml_str('set')] = set_cmd
append_cmd = syaml_dict()
env_mod[syaml_str('append-path')] = append_cmd
unset_cmd = syaml_dict()
env_mod[syaml_str('unset')] = unset_cmd
prepend_cmd = syaml_dict()
env_mod[syaml_str('prepend-path')] = prepend_cmd
set_cmd[syaml_str('EMPTY_PATH_LIST')] = syaml_str('/path/middle')
append_cmd[syaml_str('PATH_LIST')] = syaml_str('/path/last')
append_cmd[syaml_str('EMPTY_PATH_LIST')] = syaml_str('/path/last')
append_cmd[syaml_str('NEW_PATH_LIST')] = syaml_str('/path/last')
unset_cmd[syaml_str('SOME_VAR_NUM')] = None
prepend_cmd[syaml_str('PATH_LIST')] = syaml_str('/path/first:/path/second')
prepend_cmd[syaml_str('EMPTY_PATH_LIST')] = syaml_str('/path/first')
prepend_cmd[syaml_str('NEW_PATH_LIST')] = syaml_str('/path/first')
prepend_cmd[syaml_str('SOME_VAR_NUM')] = syaml_str('/8')
assert 'SOME_VAR_NUM' in os.environ
monkeypatch.setattr(pkg.compiler, 'environment', env_mod)
spack.build_environment.setup_package(pkg, False)
# Check that the order of modifications is respected and the
# variable was unset before it was prepended.
assert os.environ['SOME_VAR_NUM'] == '/8'
expected = '/path/first:/path/second:/path/third:/path/forth:/path/last'
assert os.environ['PATH_LIST'] == expected
expected = '/path/first:/path/middle:/path/last'
assert os.environ['EMPTY_PATH_LIST'] == expected
expected = '/path/first:/path/last'
assert os.environ['NEW_PATH_LIST'] == expected
# Check they were applied
for name, value in expected.items():
if value is not None:
assert os.environ[name] == value
continue
assert name not in os.environ
@pytest.mark.regression('9107')

6
lib/spack/spack/test/cmd/config.py
View File

@@ -91,3 +91,9 @@ def test_config_edit_fails_correctly_with_no_env(mutable_mock_env_path):
def test_config_get_fails_correctly_with_no_env(mutable_mock_env_path):
output = config('get', fail_on_error=False)
assert "requires a section argument or an active environment" in output
def test_config_list():
output = config('list')
assert 'compilers' in output
assert 'packages' in output

3
lib/spack/spack/test/data/config/config.yaml
View File

@@ -12,3 +12,6 @@ config:
verify_ssl: true
checksum: true
dirty: false
module_roots:
tcl: $spack/share/spack/modules
lmod: $spack/share/spack/lmod

11
lib/spack/spack/util/environment.py
View File

@@ -17,7 +17,6 @@
import llnl.util.tty as tty
import spack.util.executable as executable
from spack.util.module_cmd import py_cmd
from llnl.util.lang import dedupe
@@ -919,8 +918,14 @@ def _source_single_file(file_and_args, environment):
source_file.extend(x for x in file_and_args)
source_file = ' '.join(source_file)
dump_environment = 'PYTHONHOME="{0}" "{1}" -c "{2}"'.format(
sys.prefix, sys.executable, py_cmd)
# If the environment contains 'python' use it, if not
# go with sys.executable. Below we just need a working
# Python interpreter, not necessarily sys.executable.
python_cmd = executable.which('python3', 'python', 'python2')
python_cmd = python_cmd.name if python_cmd else sys.executable
dump_cmd = 'import os, json; print(json.dumps(dict(os.environ)))'
dump_environment = python_cmd + ' -c "{0}"'.format(dump_cmd)
# Try to source the file
source_file_arguments = ' '.join([

5
lib/spack/spack/util/module_cmd.py
View File

@@ -18,7 +18,7 @@
# This list is not exhaustive. Currently we only use load and unload
# If we need another option that changes the environment, add it here.
module_change_commands = ['load', 'swap', 'unload', 'purge', 'use', 'unuse']
py_cmd = 'import os; import json; print(json.dumps(dict(os.environ)))'
py_cmd = "'import os;import json;print(json.dumps(dict(os.environ)))'"
# This is just to enable testing. I hate it but we can't find a better way
_test_mode = False
@@ -32,8 +32,7 @@ def module(*args):
if args[0] in module_change_commands:
# Do the module manipulation, then output the environment in JSON
# and read the JSON back in the parent process to update os.environ
module_cmd += ' > /dev/null; PYTHONHOME="{0}" "{1}" -c "{2}"'.format(
sys.prefix, sys.executable, py_cmd)
module_cmd += ' >/dev/null;' + sys.executable + ' -c %s' % py_cmd
module_p = subprocess.Popen(module_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,

6
lib/spack/spack/util/web.py
View File

@@ -21,7 +21,7 @@
try:
# Python 2 had these in the HTMLParser package.
from HTMLParser import HTMLParser, HTMLParseError
from HTMLParser import HTMLParser, HTMLParseError # novm
except ImportError:
# In Python 3, things moved to html.parser
from html.parser import HTMLParser
@@ -80,7 +80,7 @@ class NonDaemonPool(multiprocessing.pool.Pool):
Process = NonDaemonProcess
else:
class NonDaemonContext(type(multiprocessing.get_context())):
class NonDaemonContext(type(multiprocessing.get_context())): # novm
Process = NonDaemonProcess
class NonDaemonPool(multiprocessing.pool.Pool):
@@ -128,7 +128,7 @@ def read_from_url(url, accept_content_type=None):
warn_no_ssl_cert_checking()
else:
# User wants SSL verification, and it *can* be provided.
context = ssl.create_default_context()
context = ssl.create_default_context() # novm
else:
# User has explicitly indicated that they do not want SSL
# verification.