zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

flake 8 : fixed checks

Browse Source
This commit is contained in:
alalazo 2016-05-11 17:02:36 +02:00
parent 78ae5d7723
commit bb4b6c8ee2
6 changed files with 391 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/spack/spack/cmd/module.py
View File

@ -32,18 +32,21 @@
from spack.modules import module_types from spack.modules import module_types
from spack.util.string import * from spack.util.string import *
description ="Manipulate modules and dotkits." description = "Manipulate modules and dotkits."
def setup_parser(subparser): def setup_parser(subparser):
sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='module_command') sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='module_command')
refresh_parser = sp.add_parser('refresh', help='Regenerate all module files.') sp.add_parser('refresh', help='Regenerate all module files.')
find_parser = sp.add_parser('find', help='Find module files for packages.') find_parser = sp.add_parser('find', help='Find module files for packages.')
find_parser.add_argument( find_parser.add_argument('module_type',
'module_type', help="Type of module to find file for. [" + '|'.join(module_types) + "]") help="Type of module to find file for. [" +
find_parser.add_argument('spec', nargs='+', help='spec to find a module file for.') '|'.join(module_types) + "]")
find_parser.add_argument('spec',
nargs='+',
help='spec to find a module file for.')
def module_find(mtype, spec_array): def module_find(mtype, spec_array):
@ -53,7 +56,8 @@ def module_find(mtype, spec_array):
should type to use that package's module. should type to use that package's module.
""" """
if mtype not in module_types: if mtype not in module_types:
tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) tty.die("Invalid module type: '%s'. Options are %s" %
(mtype, comma_or(module_types)))
specs = spack.cmd.parse_specs(spec_array) specs = spack.cmd.parse_specs(spec_array)
if len(specs) > 1: if len(specs) > 1:

58
lib/spack/spack/environment.py
View File

@ -1,7 +1,7 @@
import os
import os.path
import collections import collections
import inspect import inspect
import os
import os.path
class NameModifier(object): class NameModifier(object):
@ -26,7 +26,8 @@ def execute(self):
class UnsetEnv(NameModifier): class UnsetEnv(NameModifier):
def execute(self): def execute(self):
os.environ.pop(self.name, None) # Avoid throwing if the variable was not set # Avoid throwing if the variable was not set
os.environ.pop(self.name, None)
class SetPath(NameValueModifier): class SetPath(NameValueModifier):
@ -55,7 +56,9 @@ class RemovePath(NameValueModifier):
def execute(self): def execute(self):
environment_value = os.environ.get(self.name, '') environment_value = os.environ.get(self.name, '')
directories = environment_value.split(':') if environment_value else [] directories = environment_value.split(':') if environment_value else []
directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)] directories = [os.path.normpath(x)
for x in directories
if x != os.path.normpath(self.value)]
os.environ[self.name] = ':'.join(directories) os.environ[self.name] = ':'.join(directories)
@ -63,7 +66,8 @@ class EnvironmentModifications(object):
""" """
Keeps track of requests to modify the current environment. Keeps track of requests to modify the current environment.
Each call to a method to modify the environment stores the extra information on the caller in the request: Each call to a method to modify the environment stores the extra
information on the caller in the request:
- 'filename' : filename of the module where the caller is defined - 'filename' : filename of the module where the caller is defined
- 'lineno': line number where the request occurred - 'lineno': line number where the request occurred
- 'context' : line of code that issued the request that failed - 'context' : line of code that issued the request that failed
@ -71,10 +75,10 @@ class EnvironmentModifications(object):
def __init__(self, other=None): def __init__(self, other=None):
""" """
Initializes a new instance, copying commands from other if it is not None Initializes a new instance, copying commands from other if not None
Args: Args:
other: another instance of EnvironmentModifications from which (optional) other: another instance of EnvironmentModifications (optional)
""" """
self.env_modifications = [] self.env_modifications = []
if other is not None: if other is not None:
@ -93,7 +97,8 @@ def extend(self, other):
@staticmethod @staticmethod
def _check_other(other): def _check_other(other):
if not isinstance(other, EnvironmentModifications): if not isinstance(other, EnvironmentModifications):
raise TypeError('other must be an instance of EnvironmentModifications') raise TypeError(
'other must be an instance of EnvironmentModifications')
def _get_outside_caller_attributes(self): def _get_outside_caller_attributes(self):
stack = inspect.stack() stack = inspect.stack()
@ -101,12 +106,10 @@ def _get_outside_caller_attributes(self):
_, filename, lineno, _, context, index = stack[2] _, filename, lineno, _, context, index = stack[2]
context = context[index].strip() context = context[index].strip()
except Exception: except Exception:
filename, lineno, context = 'unknown file', 'unknown line', 'unknown context' filename = 'unknown file'
args = { lineno = 'unknown line'
'filename': filename, context = 'unknown context'
'lineno': lineno, args = {'filename': filename, 'lineno': lineno, 'context': context}
'context': context
}
return args return args
def set(self, name, value, **kwargs): def set(self, name, value, **kwargs):
@ -170,7 +173,8 @@ def prepend_path(self, name, path, **kwargs):
def remove_path(self, name, path, **kwargs): def remove_path(self, name, path, **kwargs):
""" """
Stores in the current object a request to remove a path from a path list Stores in the current object a request to remove a path from a path
list
Args: Args:
name: name of the path list in the environment name: name of the path list in the environment
@ -185,7 +189,8 @@ def group_by_name(self):
Returns a dict of the modifications grouped by variable name Returns a dict of the modifications grouped by variable name
Returns: Returns:
dict mapping the environment variable name to the modifications to be done on it dict mapping the environment variable name to the modifications to
be done on it
""" """
modifications = collections.defaultdict(list) modifications = collections.defaultdict(list)
for item in self: for item in self:
@ -203,7 +208,7 @@ def apply_modifications(self):
Applies the modifications and clears the list Applies the modifications and clears the list
""" """
modifications = self.group_by_name() modifications = self.group_by_name()
# Apply the modifications to the environment variables one variable at a time # Apply modifications one variable at a time
for name, actions in sorted(modifications.items()): for name, actions in sorted(modifications.items()):
for x in actions: for x in actions:
x.execute() x.execute()
@ -224,13 +229,19 @@ def concatenate_paths(paths):
def set_or_unset_not_first(variable, changes, errstream): def set_or_unset_not_first(variable, changes, errstream):
""" """
Check if we are going to set or unset something after other modifications have already been requested Check if we are going to set or unset something after other modifications
have already been requested
""" """
indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]] indexes = [ii
for ii, item in enumerate(changes)
if ii != 0 and type(item) in [SetEnv, UnsetEnv]]
if indexes: if indexes:
good = '\t \t{context} at {filename}:{lineno}' good = '\t \t{context} at {filename}:{lineno}'
nogood = '\t--->\t{context} at {filename}:{lineno}' nogood = '\t--->\t{context} at {filename}:{lineno}'
errstream('Suspicious requests to set or unset the variable \'{var}\' found'.format(var=variable)) message = 'Suspicious requests to set or unset the variable \'{var}\' found' # NOQA: ignore=E501
errstream(
message.format(
var=variable))
for ii, item in enumerate(changes): for ii, item in enumerate(changes):
print_format = nogood if ii in indexes else good print_format = nogood if ii in indexes else good
errstream(print_format.format(**item.args)) errstream(print_format.format(**item.args))
@ -238,8 +249,8 @@ def set_or_unset_not_first(variable, changes, errstream):
def validate(env, errstream): def validate(env, errstream):
""" """
Validates the environment modifications to check for the presence of suspicious patterns. Prompts a warning for Validates the environment modifications to check for the presence of
everything that was found suspicious patterns. Prompts a warning for everything that was found
Current checks: Current checks:
- set or unset variables after other changes on the same variable - set or unset variables after other changes on the same variable
@ -254,7 +265,8 @@ def validate(env, errstream):
def filter_environment_blacklist(env, variables): def filter_environment_blacklist(env, variables):
""" """
Generator that filters out any change to environment variables present in the input list Generator that filters out any change to environment variables present in
the input list
Args: Args:
env: list of environment modifications env: list of environment modifications

196
lib/spack/spack/modules.py
View File

@ -23,36 +23,35 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
############################################################################## ##############################################################################
""" """
This module contains code for creating environment modules, which can include dotkits, tcl modules, lmod, and others. This module contains code for creating environment modules, which can include
dotkits, tcl modules, lmod, and others.
The various types of modules are installed by post-install hooks and removed after an uninstall by post-uninstall hooks. The various types of modules are installed by post-install hooks and removed
This class consolidates the logic for creating an abstract description of the information that module systems need. after an uninstall by post-uninstall hooks. This class consolidates the logic
Currently that includes a number of directories to be appended to paths in the user's environment: for creating an abstract description of the information that module systems
need.
* /bin directories to be appended to PATH This module also includes logic for coming up with unique names for the module
* /lib* directories for LD_LIBRARY_PATH files so that they can be found by the various shell-support files in
* /include directories for CPATH $SPACK/share/spack/setup-env.*.
* /man* and /share/man* directories for MANPATH
* the package prefix for CMAKE_PREFIX_PATH
This module also includes logic for coming up with unique names for the module files so that they can be found by the Each hook in hooks/ implements the logic for writing its specific type of
various shell-support files in $SPACK/share/spack/setup-env.*. module file.
Each hook in hooks/ implements the logic for writing its specific type of module file.
""" """
import copy import copy
import datetime import datetime
import os import os
import os.path import os.path
import re import re
import textwrap
import string import string
import textwrap
import llnl.util.tty as tty import llnl.util.tty as tty
import spack import spack
import spack.config import spack.config
from llnl.util.filesystem import join_path, mkdirp from llnl.util.filesystem import join_path, mkdirp
from spack.build_environment import parent_class_modules, set_module_variables_for_package from spack.build_environment import parent_class_modules
from spack.build_environment import set_module_variables_for_package
from spack.environment import * from spack.environment import *
__all__ = ['EnvModule', 'Dotkit', 'TclModule'] __all__ = ['EnvModule', 'Dotkit', 'TclModule']
@ -67,30 +66,26 @@ def print_help():
""" """
For use by commands to tell user how to activate shell support. For use by commands to tell user how to activate shell support.
""" """
tty.msg("This command requires spack's shell integration.", tty.msg("This command requires spack's shell integration.", "",
"",
"To initialize spack's shell commands, you must run one of", "To initialize spack's shell commands, you must run one of",
"the commands below. Choose the right command for your shell.", "the commands below. Choose the right command for your shell.",
"", "", "For bash and zsh:",
"For bash and zsh:", " . %s/setup-env.sh" % spack.share_path, "",
" . %s/setup-env.sh" % spack.share_path, "For csh and tcsh:", " setenv SPACK_ROOT %s" % spack.prefix,
"", " source %s/setup-env.csh" % spack.share_path, "")
"For csh and tcsh:",
" setenv SPACK_ROOT %s" % spack.prefix,
" source %s/setup-env.csh" % spack.share_path,
"")
def inspect_path(prefix): def inspect_path(prefix):
""" """
Inspects the prefix of an installation to search for common layouts. Issues a request to modify the environment Inspects the prefix of an installation to search for common layouts. Issues
accordingly when an item is found. a request to modify the environment accordingly when an item is found.
Args: Args:
prefix: prefix of the installation prefix: prefix of the installation
Returns: Returns:
instance of EnvironmentModifications containing the requested modifications instance of EnvironmentModifications containing the requested
modifications
""" """
env = EnvironmentModifications() env = EnvironmentModifications()
# Inspect the prefix to check for the existence of common directories # Inspect the prefix to check for the existence of common directories
@ -105,18 +100,21 @@ def inspect_path(prefix):
def dependencies(spec, request='all'): def dependencies(spec, request='all'):
""" """
Returns the list of dependent specs for a given spec, according to the given request Returns the list of dependent specs for a given spec, according to the
given request
Args: Args:
spec: target spec spec: target spec
request: either 'none', 'direct' or 'all' request: either 'none', 'direct' or 'all'
Returns: Returns:
empty list if 'none', direct dependency list if 'direct', all dependencies if 'all' empty list if 'none', direct dependency list if 'direct', all
dependencies if 'all'
""" """
if request not in ('none', 'direct', 'all'): if request not in ('none', 'direct', 'all'):
raise tty.error("Wrong value for argument 'request' : should be one of ('none', 'direct', 'all') " message = "Wrong value for argument 'request' : "
" [current value is '%s']" % request) message += "should be one of ('none', 'direct', 'all')"
raise tty.error(message + " [current value is '%s']" % request)
if request == 'none': if request == 'none':
return [] return []
@ -124,12 +122,19 @@ def dependencies(spec, request='all'):
if request == 'direct': if request == 'direct':
return [xx for _, xx in spec.dependencies.items()] return [xx for _, xx in spec.dependencies.items()]
# FIXME : during module file creation nodes seem to be visited multiple times even if cover='nodes' # FIXME : during module file creation nodes seem to be visited multiple
# FIXME : is given. This work around permits to get a unique list of spec anyhow. # FIXME : times even if cover='nodes' is given. This work around permits
# FIXME : Possibly we miss a merge step among nodes that refer to the same package. # FIXME : to get a unique list of spec anyhow. Do we miss a merge
# FIXME : step among nodes that refer to the same package?
seen = set() seen = set()
seen_add = seen.add seen_add = seen.add
l = [xx for xx in sorted(spec.traverse(order='post', depth=True, cover='nodes', root=False), reverse=True)] l = [xx
for xx in sorted(
spec.traverse(order='post',
depth=True,
cover='nodes',
root=False),
reverse=True)]
return [xx for ii, xx in l if not (xx in seen or seen_add(xx))] return [xx for ii, xx in l if not (xx in seen or seen_add(xx))]
@ -146,7 +151,8 @@ def update_dictionary_extending_lists(target, update):
def parse_config_options(module_generator): def parse_config_options(module_generator):
""" """
Parse the configuration file and returns a bunch of items that will be needed during module file generation Parse the configuration file and returns a bunch of items that will be
needed during module file generation
Args: Args:
module_generator: module generator for a given spec module_generator: module generator for a given spec
@ -154,11 +160,14 @@ def parse_config_options(module_generator):
Returns: Returns:
autoloads: list of specs to be autoloaded autoloads: list of specs to be autoloaded
prerequisites: list of specs to be marked as prerequisite prerequisites: list of specs to be marked as prerequisite
filters: list of environment variables whose modification is blacklisted in module files filters: list of environment variables whose modification is
env: list of custom environment modifications to be applied in the module file blacklisted in module files
env: list of custom environment modifications to be applied in the
module file
""" """
# Get the configuration for this kind of generator # Get the configuration for this kind of generator
module_configuration = copy.deepcopy(CONFIGURATION.get(module_generator.name, {})) module_configuration = copy.deepcopy(CONFIGURATION.get(
module_generator.name, {}))
##### #####
# Merge all the rules # Merge all the rules
@ -179,9 +188,12 @@ def parse_config_options(module_generator):
##### #####
# Automatic loading loads # Automatic loading loads
module_file_actions['autoload'] = dependencies(module_generator.spec, module_file_actions.get('autoload', 'none')) module_file_actions['autoload'] = dependencies(
module_generator.spec, module_file_actions.get('autoload', 'none'))
# Prerequisites # Prerequisites
module_file_actions['prerequisites'] = dependencies(module_generator.spec, module_file_actions.get('prerequisites', 'none')) module_file_actions['prerequisites'] = dependencies(
module_generator.spec, module_file_actions.get('prerequisites',
'none'))
# Environment modifications # Environment modifications
environment_actions = module_file_actions.pop('environment', {}) environment_actions = module_file_actions.pop('environment', {})
env = EnvironmentModifications() env = EnvironmentModifications()
@ -189,7 +201,7 @@ def parse_config_options(module_generator):
def process_arglist(arglist): def process_arglist(arglist):
if method == 'unset': if method == 'unset':
for x in arglist: for x in arglist:
yield (x,) yield (x, )
else: else:
for x in arglist.iteritems(): for x in arglist.iteritems():
yield x yield x
@ -198,19 +210,13 @@ def process_arglist(arglist):
for args in process_arglist(arglist): for args in process_arglist(arglist):
getattr(env, method)(*args) getattr(env, method)(*args)
# for item in arglist:
# if method == 'unset':
# args = [item]
# else:
# args = item.split(',')
# getattr(env, method)(*args)
return module_file_actions, env return module_file_actions, env
def filter_blacklisted(specs, module_name): def filter_blacklisted(specs, module_name):
""" """
Given a sequence of specs, filters the ones that are blacklisted in the module configuration file. Given a sequence of specs, filters the ones that are blacklisted in the
module configuration file.
Args: Args:
specs: sequence of spec instances specs: sequence of spec instances
@ -233,7 +239,8 @@ class EnvModule(object):
class __metaclass__(type): class __metaclass__(type):
def __init__(cls, name, bases, dict): def __init__(cls, name, bases, dict):
type.__init__(cls, name, bases, dict) type.__init__(cls, name, bases, dict)
if cls.name != 'env_module' and cls.name in CONFIGURATION['enable']: if cls.name != 'env_module' and cls.name in CONFIGURATION[
'enable']:
module_types[cls.name] = cls module_types[cls.name] = cls
def __init__(self, spec=None): def __init__(self, spec=None):
@ -249,7 +256,8 @@ def __init__(self, spec=None):
# long description is the docstring with reduced whitespace. # long description is the docstring with reduced whitespace.
self.long_description = None self.long_description = None
if self.spec.package.__doc__: if self.spec.package.__doc__:
self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) self.long_description = re.sub(r'\s+', ' ',
self.spec.package.__doc__)
@property @property
def naming_scheme(self): def naming_scheme(self):
@ -271,12 +279,14 @@ def tokens(self):
@property @property
def use_name(self): def use_name(self):
""" """
Subclasses should implement this to return the name the module command uses to refer to the package. Subclasses should implement this to return the name the module command
uses to refer to the package.
""" """
naming_tokens = self.tokens naming_tokens = self.tokens
naming_scheme = self.naming_scheme naming_scheme = self.naming_scheme
name = naming_scheme.format(**naming_tokens) name = naming_scheme.format(**naming_tokens)
name += '-' + self.spec.dag_hash() # Always append the hash to make the module file unique name += '-' + self.spec.dag_hash(
) # Always append the hash to make the module file unique
# Not everybody is working on linux... # Not everybody is working on linux...
parts = name.split('/') parts = name.split('/')
name = join_path(*parts) name = join_path(*parts)
@ -296,8 +306,12 @@ def category(self):
@property @property
def blacklisted(self): def blacklisted(self):
configuration = CONFIGURATION.get(self.name, {}) configuration = CONFIGURATION.get(self.name, {})
whitelist_matches = [x for x in configuration.get('whitelist', []) if self.spec.satisfies(x)] whitelist_matches = [x
blacklist_matches = [x for x in configuration.get('blacklist', []) if self.spec.satisfies(x)] for x in configuration.get('whitelist', [])
if self.spec.satisfies(x)]
blacklist_matches = [x
for x in configuration.get('blacklist', [])
if self.spec.satisfies(x)]
if whitelist_matches: if whitelist_matches:
message = '\tWHITELIST : %s [matches : ' % self.spec.cshort_spec message = '\tWHITELIST : %s [matches : ' % self.spec.cshort_spec
for rule in whitelist_matches: for rule in whitelist_matches:
@ -327,7 +341,8 @@ def write(self):
""" """
if self.blacklisted: if self.blacklisted:
return return
tty.debug("\tWRITE : %s [%s]" % (self.spec.cshort_spec, self.file_name)) tty.debug("\tWRITE : %s [%s]" %
(self.spec.cshort_spec, self.file_name))
module_dir = os.path.dirname(self.file_name) module_dir = os.path.dirname(self.file_name)
if not os.path.exists(module_dir): if not os.path.exists(module_dir):
@ -337,11 +352,12 @@ def write(self):
# installation prefix # installation prefix
env = inspect_path(self.spec.prefix) env = inspect_path(self.spec.prefix)
# Let the extendee/dependency modify their extensions/dependencies before asking for # Let the extendee/dependency modify their extensions/dependencies
# package-specific modifications # before asking for package-specific modifications
spack_env = EnvironmentModifications() spack_env = EnvironmentModifications()
# TODO : the code down below is quite similar to build_environment.setup_package and needs to be # TODO : the code down below is quite similar to
# TODO : factored out to a single place # TODO : build_environment.setup_package and needs to be factored out
# TODO : to a single place
for item in dependencies(self.spec, 'all'): for item in dependencies(self.spec, 'all'):
package = self.spec[item.name].package package = self.spec[item.name].package
modules = parent_class_modules(package.__class__) modules = parent_class_modules(package.__class__)
@ -358,14 +374,18 @@ def write(self):
# Parse configuration file # Parse configuration file
module_configuration, conf_env = parse_config_options(self) module_configuration, conf_env = parse_config_options(self)
env.extend(conf_env) env.extend(conf_env)
filters = module_configuration.get('filter', {}).get('environment_blacklist',{}) filters = module_configuration.get('filter', {}).get(
'environment_blacklist', {})
# Build up the module file content # Build up the module file content
module_file_content = self.header module_file_content = self.header
for x in filter_blacklisted(module_configuration.pop('autoload', []), self.name): for x in filter_blacklisted(
module_configuration.pop('autoload', []), self.name):
module_file_content += self.autoload(x) module_file_content += self.autoload(x)
for x in filter_blacklisted(module_configuration.pop('prerequisites', []), self.name): for x in filter_blacklisted(
module_configuration.pop('prerequisites', []), self.name):
module_file_content += self.prerequisite(x) module_file_content += self.prerequisite(x)
for line in self.process_environment_command(filter_environment_blacklist(env, filters)): for line in self.process_environment_command(
filter_environment_blacklist(env, filters)):
module_file_content += line module_file_content += line
for line in self.module_specific_content(module_configuration): for line in self.module_specific_content(module_configuration):
module_file_content += line module_file_content += line
@ -392,10 +412,13 @@ def prerequisite(self, spec):
def process_environment_command(self, env): def process_environment_command(self, env):
for command in env: for command in env:
try: try:
yield self.environment_modifications_formats[type(command)].format(**command.args) yield self.environment_modifications_formats[type(
command)].format(**command.args)
except KeyError: except KeyError:
tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command))) message = 'Cannot handle command of type {command} : skipping request' # NOQA: ignore=E501
tty.warn('{context} at {filename}:{lineno}'.format(**command.args)) details = '{context} at {filename}:{lineno}'
tty.warn(message.format(command=type(command)))
tty.warn(details.format(**command.args))
@property @property
def file_name(self): def file_name(self):
@ -408,9 +431,12 @@ def remove(self):
if os.path.exists(mod_file): if os.path.exists(mod_file):
try: try:
os.remove(mod_file) # Remove the module file os.remove(mod_file) # Remove the module file
os.removedirs(os.path.dirname(mod_file)) # Remove all the empty directories from the leaf up os.removedirs(
os.path.dirname(mod_file)
) # Remove all the empty directories from the leaf up
except OSError: except OSError:
pass # removedirs throws OSError on first non-empty directory found # removedirs throws OSError on first non-empty directory found
pass
class Dotkit(EnvModule): class Dotkit(EnvModule):
@ -424,13 +450,12 @@ class Dotkit(EnvModule):
autoload_format = 'dk_op {module_file}\n' autoload_format = 'dk_op {module_file}\n'
prerequisite_format = None # TODO : does something like prerequisite exist for dotkit? default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' # NOQA: ignore=E501
default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}'
@property @property
def file_name(self): def file_name(self):
return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name) return join_path(Dotkit.path, self.spec.architecture,
'%s.dk' % self.use_name)
@property @property
def header(self): def header(self):
@ -474,7 +499,7 @@ class TclModule(EnvModule):
prerequisite_format = 'prereq {module_file}\n' prerequisite_format = 'prereq {module_file}\n'
default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' default_naming_format = '{name}-{version}-{compiler.name}-{compiler.version}' # NOQA: ignore=E501
@property @property
def file_name(self): def file_name(self):
@ -482,9 +507,10 @@ def file_name(self):
@property @property
def header(self): def header(self):
timestamp = datetime.datetime.now()
# TCL Modulefile header # TCL Modulefile header
header = '#%Module1.0\n' header = '#%Module1.0\n'
header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % datetime.datetime.now() header += '## Module file created by spack (https://github.com/LLNL/spack) on %s\n' % timestamp # NOQA: ignore=E501
header += '##\n' header += '##\n'
header += '## %s\n' % self.spec.short_spec header += '## %s\n' % self.spec.short_spec
header += '##\n' header += '##\n'
@ -509,16 +535,18 @@ def module_specific_content(self, configuration):
f = string.Formatter() f = string.Formatter()
for item in conflict_format: for item in conflict_format:
line = 'conflict ' + item + '\n' line = 'conflict ' + item + '\n'
if len([x for x in f.parse(line)]) > 1: # We do have placeholder to substitute if len([x for x in f.parse(line)
for naming_dir, conflict_dir in zip(self.naming_scheme.split('/'), item.split('/')): ]) > 1: # We do have placeholder to substitute
for naming_dir, conflict_dir in zip(
self.naming_scheme.split('/'), item.split('/')):
if naming_dir != conflict_dir: if naming_dir != conflict_dir:
message = 'conflict scheme does not match naming scheme [{spec}]\n\n' message = 'conflict scheme does not match naming scheme [{spec}]\n\n' # NOQA: ignore=E501
message += 'naming scheme : "{nformat}"\n' message += 'naming scheme : "{nformat}"\n'
message += 'conflict scheme : "{cformat}"\n\n' message += 'conflict scheme : "{cformat}"\n\n'
message += '** You may want to check your `modules.yaml` configuration file **\n' message += '** You may want to check your `modules.yaml` configuration file **\n' # NOQA: ignore=E501
tty.error( tty.error(message.format(spec=self.spec,
message.format(spec=self.spec, nformat=self.naming_scheme, cformat=item) nformat=self.naming_scheme,
) cformat=item))
raise SystemExit('Module generation aborted.') raise SystemExit('Module generation aborted.')
line = line.format(**naming_tokens) line = line.format(**naming_tokens)
yield line yield line

353
lib/spack/spack/package.py
View File

@ -37,7 +37,6 @@
import re import re
import textwrap import textwrap
import time import time
import glob
import llnl.util.tty as tty import llnl.util.tty as tty
import spack import spack
@ -62,7 +61,6 @@
from spack.util.executable import ProcessError from spack.util.executable import ProcessError
from spack.version import * from spack.version import *
from urlparse import urlparse from urlparse import urlparse
"""Allowed URL schemes for spack packages.""" """Allowed URL schemes for spack packages."""
_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
@ -305,26 +303,21 @@ class SomePackage(Package):
# #
"""By default we build in parallel. Subclasses can override this.""" """By default we build in parallel. Subclasses can override this."""
parallel = True parallel = True
"""# jobs to use for parallel make. If set, overrides default of ncpus.""" """# jobs to use for parallel make. If set, overrides default of ncpus."""
make_jobs = None make_jobs = None
"""Most packages are NOT extendable. Set to True if you want extensions."""
"""Most packages are NOT extendable. Set to True if you want extensions."""
extendable = False extendable = False
"""List of prefix-relative file paths (or a single path). If these do """List of prefix-relative file paths (or a single path). If these do
not exist after install, or if they exist but are not files, not exist after install, or if they exist but are not files,
sanity checks fail. sanity checks fail.
""" """
sanity_check_is_file = [] sanity_check_is_file = []
"""List of prefix-relative directory paths (or a single path). If """List of prefix-relative directory paths (or a single path). If
these do not exist after install, or if they exist but are not these do not exist after install, or if they exist but are not
directories, sanity checks will fail. directories, sanity checks will fail.
""" """
sanity_check_is_dir = [] sanity_check_is_dir = []
def __init__(self, spec): def __init__(self, spec):
# this determines how the package should be built. # this determines how the package should be built.
self.spec = spec self.spec = spec
@ -336,23 +329,24 @@ def __init__(self, spec):
self.name = self.name[self.name.rindex('.') + 1:] self.name = self.name[self.name.rindex('.') + 1:]
# Allow custom staging paths for packages # Allow custom staging paths for packages
self.path=None self.path = None
# Sanity check attributes required by Spack directives. # Sanity check attributes required by Spack directives.
spack.directives.ensure_dicts(type(self)) spack.directives.ensure_dicts(type(self))
# Check versions in the versions dict. # Check versions in the versions dict.
for v in self.versions: for v in self.versions:
assert(isinstance(v, Version)) assert (isinstance(v, Version))
# Check version descriptors # Check version descriptors
for v in sorted(self.versions): for v in sorted(self.versions):
assert(isinstance(self.versions[v], dict)) assert (isinstance(self.versions[v], dict))
# Version-ize the keys in versions dict # Version-ize the keys in versions dict
try: try:
self.versions = dict((Version(v), h) for v,h in self.versions.items()) self.versions = dict((Version(v), h)
except ValueError, e: for v, h in self.versions.items())
except ValueError as e:
raise ValueError("In package %s: %s" % (self.name, e.message)) raise ValueError("In package %s: %s" % (self.name, e.message))
# stage used to build this package. # stage used to build this package.
@ -366,9 +360,9 @@ def __init__(self, spec):
# This makes self.url behave sanely. # This makes self.url behave sanely.
if self.spec.versions.concrete: if self.spec.versions.concrete:
# TODO: this is a really roundabout way of determining the type # TODO: this is a really roundabout way of determining the type
# TODO: of fetch to do. figure out a more sane fetch strategy/package # TODO: of fetch to do. figure out a more sane fetch
# TODO: init order (right now it's conflated with stage, package, and # TODO: strategy/package init order (right now it's conflated with
# TODO: the tests make assumptions) # TODO: stage, package, and the tests make assumptions)
f = fs.for_package_version(self, self.version) f = fs.for_package_version(self, self.version)
if isinstance(f, fs.URLFetchStrategy): if isinstance(f, fs.URLFetchStrategy):
self.url = self.url_for_version(self.spec.version) self.url = self.url_for_version(self.spec.version)
@ -387,14 +381,12 @@ def __init__(self, spec):
if self.is_extension: if self.is_extension:
spack.repo.get(self.extendee_spec)._check_extendable() spack.repo.get(self.extendee_spec)._check_extendable()
@property @property
def version(self): def version(self):
if not self.spec.versions.concrete: if not self.spec.versions.concrete:
raise ValueError("Can only get of package with concrete version.") raise ValueError("Can only get of package with concrete version.")
return self.spec.versions[0] return self.spec.versions[0]
@memoized @memoized
def version_urls(self): def version_urls(self):
"""Return a list of URLs for different versions of this """Return a list of URLs for different versions of this
@ -407,7 +399,6 @@ def version_urls(self):
version_urls[v] = args['url'] version_urls[v] = args['url']
return version_urls return version_urls
def nearest_url(self, version): def nearest_url(self, version):
"""Finds the URL for the next lowest version with a URL. """Finds the URL for the next lowest version with a URL.
If there is no lower version with a URL, uses the If there is no lower version with a URL, uses the
@ -424,10 +415,11 @@ def nearest_url(self, version):
url = version_urls[v] url = version_urls[v]
return url return url
# TODO: move this out of here and into some URL extrapolation module? # TODO: move this out of here and into some URL extrapolation module?
def url_for_version(self, version): def url_for_version(self, version):
"""Returns a URL that you can download a new version of this package from.""" """
Returns a URL that you can download a new version of this package from.
"""
if not isinstance(version, Version): if not isinstance(version, Version):
version = Version(version) version = Version(version)
@ -441,14 +433,17 @@ def url_for_version(self, version):
return version_urls[version] return version_urls[version]
# If we have no idea, try to substitute the version. # If we have no idea, try to substitute the version.
return spack.url.substitute_version(self.nearest_url(version), return spack.url.substitute_version(
self.url_version(version)) self.nearest_url(version), self.url_version(version))
def _make_resource_stage(self, root_stage, fetcher, resource): def _make_resource_stage(self, root_stage, fetcher, resource):
resource_stage_folder = self._resource_stage(resource) resource_stage_folder = self._resource_stage(resource)
resource_mirror = join_path(self.name, os.path.basename(fetcher.url)) resource_mirror = join_path(self.name, os.path.basename(fetcher.url))
stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource, stage = ResourceStage(resource.fetcher,
name=resource_stage_folder, mirror_path=resource_mirror, root=root_stage,
resource=resource,
name=resource_stage_folder,
mirror_path=resource_mirror,
path=self.path) path=self.path)
return stage return stage
@ -474,7 +469,8 @@ def _make_stage(self):
else: else:
# Construct resource stage # Construct resource stage
resource = resources[ii - 1] # ii == 0 is root! resource = resources[ii - 1] # ii == 0 is root!
stage = self._make_resource_stage(composite_stage[0], fetcher, resource) stage = self._make_resource_stage(composite_stage[0], fetcher,
resource)
# Append the item to the composite # Append the item to the composite
composite_stage.append(stage) composite_stage.append(stage)
@ -492,13 +488,11 @@ def stage(self):
self._stage = self._make_stage() self._stage = self._make_stage()
return self._stage return self._stage
@stage.setter @stage.setter
def stage(self, stage): def stage(self, stage):
"""Allow a stage object to be set to override the default.""" """Allow a stage object to be set to override the default."""
self._stage = stage self._stage = stage
def _make_fetcher(self): def _make_fetcher(self):
# Construct a composite fetcher that always contains at least # Construct a composite fetcher that always contains at least
# one element (the root package). In case there are resources # one element (the root package). In case there are resources
@ -515,7 +509,8 @@ def _make_fetcher(self):
@property @property
def fetcher(self): def fetcher(self):
if not self.spec.versions.concrete: if not self.spec.versions.concrete:
raise ValueError("Can only get a fetcher for a package with concrete versions.") raise ValueError(
"Can only get a fetcher for a package with concrete versions.")
if not self._fetcher: if not self._fetcher:
self._fetcher = self._make_fetcher() self._fetcher = self._make_fetcher()
return self._fetcher return self._fetcher
@ -524,10 +519,11 @@ def fetcher(self):
def fetcher(self, f): def fetcher(self, f):
self._fetcher = f self._fetcher = f
@property @property
def extendee_spec(self): def extendee_spec(self):
"""Spec of the extendee of this package, or None if it is not an extension.""" """
Spec of the extendee of this package, or None if it is not an extension
"""
if not self.extendees: if not self.extendees:
return None return None
@ -549,10 +545,11 @@ def extendee_spec(self):
spec, kwargs = self.extendees[name] spec, kwargs = self.extendees[name]
return spec return spec
@property @property
def extendee_args(self): def extendee_args(self):
"""Spec of the extendee of this package, or None if it is not an extension.""" """
Spec of the extendee of this package, or None if it is not an extension
"""
if not self.extendees: if not self.extendees:
return None return None
@ -560,7 +557,6 @@ def extendee_args(self):
name = next(iter(self.extendees)) name = next(iter(self.extendees))
return self.extendees[name][1] return self.extendees[name][1]
@property @property
def is_extension(self): def is_extension(self):
# if it is concrete, it's only an extension if it actually # if it is concrete, it's only an extension if it actually
@ -571,22 +567,20 @@ def is_extension(self):
# If not, then it's an extension if it *could* be an extension # If not, then it's an extension if it *could* be an extension
return bool(self.extendees) return bool(self.extendees)
def extends(self, spec): def extends(self, spec):
if not spec.name in self.extendees: if spec.name not in self.extendees:
return False return False
s = self.extendee_spec s = self.extendee_spec
return s and s.satisfies(spec) return s and s.satisfies(spec)
@property @property
def activated(self): def activated(self):
if not self.is_extension: if not self.is_extension:
raise ValueError("is_extension called on package that is not an extension.") raise ValueError(
"is_extension called on package that is not an extension.")
exts = spack.install_layout.extension_map(self.extendee_spec) exts = spack.install_layout.extension_map(self.extendee_spec)
return (self.name in exts) and (exts[self.name] == self.spec) return (self.name in exts) and (exts[self.name] == self.spec)
def preorder_traversal(self, visited=None, **kwargs): def preorder_traversal(self, visited=None, **kwargs):
"""This does a preorder traversal of the package's dependence DAG.""" """This does a preorder traversal of the package's dependence DAG."""
virtual = kwargs.get("virtual", False) virtual = kwargs.get("virtual", False)
@ -605,36 +599,35 @@ def preorder_traversal(self, visited=None, **kwargs):
spec = self.dependencies[name] spec = self.dependencies[name]
# currently, we do not descend into virtual dependencies, as this # currently, we do not descend into virtual dependencies, as this
# makes doing a sensible traversal much harder. We just assume that # makes doing a sensible traversal much harder. We just assume
# ANY of the virtual deps will work, which might not be true (due to # that ANY of the virtual deps will work, which might not be true
# conflicts or unsatisfiable specs). For now this is ok but we might # (due to conflicts or unsatisfiable specs). For now this is ok
# want to reinvestigate if we start using a lot of complicated virtual # but we might want to reinvestigate if we start using a lot of
# dependencies # complicated virtual dependencies
# TODO: reinvestigate this. # TODO: reinvestigate this.
if spec.virtual: if spec.virtual:
if virtual: if virtual:
yield spec yield spec
continue continue
for pkg in spack.repo.get(name).preorder_traversal(visited, **kwargs): for pkg in spack.repo.get(name).preorder_traversal(visited,
**kwargs):
yield pkg yield pkg
def provides(self, vpkg_name): def provides(self, vpkg_name):
"""True if this package provides a virtual package with the specified name.""" """
True if this package provides a virtual package with the specified name
"""
return any(s.name == vpkg_name for s in self.provided) return any(s.name == vpkg_name for s in self.provided)
def virtual_dependencies(self, visited=None): def virtual_dependencies(self, visited=None):
for spec in sorted(set(self.preorder_traversal(virtual=True))): for spec in sorted(set(self.preorder_traversal(virtual=True))):
yield spec yield spec
@property @property
def installed(self): def installed(self):
return os.path.isdir(self.prefix) return os.path.isdir(self.prefix)
@property @property
def installed_dependents(self): def installed_dependents(self):
"""Return a list of the specs of all installed packages that depend """Return a list of the specs of all installed packages that depend
@ -651,60 +644,62 @@ def installed_dependents(self):
dependents.append(spec) dependents.append(spec)
return dependents return dependents
@property @property
def prefix(self): def prefix(self):
"""Get the prefix into which this package should be installed.""" """Get the prefix into which this package should be installed."""
return self.spec.prefix return self.spec.prefix
@property @property
def compiler(self): def compiler(self):
"""Get the spack.compiler.Compiler object used to build this package.""" """Get the spack.compiler.Compiler object used to build this package"""
if not self.spec.concrete: if not self.spec.concrete:
raise ValueError("Can only get a compiler for a concrete package.") raise ValueError("Can only get a compiler for a concrete package.")
return spack.compilers.compiler_for_spec(self.spec.compiler) return spack.compilers.compiler_for_spec(self.spec.compiler)
def url_version(self, version): def url_version(self, version):
"""Given a version, this returns a string that should be substituted into the """
package's URL to download that version. Given a version, this returns a string that should be substituted
By default, this just returns the version string. Subclasses may need to into the package's URL to download that version.
override this, e.g. for boost versions where you need to ensure that there
are _'s in the download URL. By default, this just returns the version string. Subclasses may need
to override this, e.g. for boost versions where you need to ensure that
there are _'s in the download URL.
""" """
return str(version) return str(version)
def remove_prefix(self): def remove_prefix(self):
"""Removes the prefix for a package along with any empty parent directories.""" """
Removes the prefix for a package along with any empty parent
directories
"""
spack.install_layout.remove_install_directory(self.spec) spack.install_layout.remove_install_directory(self.spec)
def do_fetch(self, mirror_only=False): def do_fetch(self, mirror_only=False):
"""Creates a stage directory and downloads the tarball for this package. """
Working directory will be set to the stage directory. Creates a stage directory and downloads the tarball for this package.
Working directory will be set to the stage directory.
""" """
if not self.spec.concrete: if not self.spec.concrete:
raise ValueError("Can only fetch concrete packages.") raise ValueError("Can only fetch concrete packages.")
start_time = time.time() start_time = time.time()
if spack.do_checksum and not self.version in self.versions: if spack.do_checksum and self.version not in self.versions:
tty.warn("There is no checksum on file to fetch %s safely." tty.warn("There is no checksum on file to fetch %s safely." %
% self.spec.format('$_$@')) self.spec.format('$_$@'))
# Ask the user whether to skip the checksum if we're # Ask the user whether to skip the checksum if we're
# interactive, but just fail if non-interactive. # interactive, but just fail if non-interactive.
checksum_msg = "Add a checksum or use --no-checksum to skip this check." checksum_msg = "Add a checksum or use --no-checksum to skip this check." # NOQA: ignore=E501
ignore_checksum = False ignore_checksum = False
if sys.stdout.isatty(): if sys.stdout.isatty():
ignore_checksum = tty.get_yes_or_no(" Fetch anyway?", default=False) ignore_checksum = tty.get_yes_or_no(" Fetch anyway?",
default=False)
if ignore_checksum: if ignore_checksum:
tty.msg("Fetching with no checksum.", checksum_msg) tty.msg("Fetching with no checksum.", checksum_msg)
if not ignore_checksum: if not ignore_checksum:
raise FetchError( raise FetchError("Will not fetch %s" %
"Will not fetch %s" % self.spec.format('$_$@'), checksum_msg) self.spec.format('$_$@'), checksum_msg)
self.stage.fetch(mirror_only) self.stage.fetch(mirror_only)
@ -723,7 +718,6 @@ def do_stage(self, mirror_only=False):
self.stage.expand_archive() self.stage.expand_archive()
self.stage.chdir_to_source() self.stage.chdir_to_source()
def do_patch(self): def do_patch(self):
"""Calls do_stage(), then applied patches to the expanded tarball if they """Calls do_stage(), then applied patches to the expanded tarball if they
haven't been applied already.""" haven't been applied already."""
@ -743,10 +737,10 @@ def do_patch(self):
# Construct paths to special files in the archive dir used to # Construct paths to special files in the archive dir used to
# keep track of whether patches were successfully applied. # keep track of whether patches were successfully applied.
archive_dir = self.stage.source_path archive_dir = self.stage.source_path
good_file = join_path(archive_dir, '.spack_patched') good_file = join_path(archive_dir, '.spack_patched')
no_patches_file = join_path(archive_dir, '.spack_no_patches') no_patches_file = join_path(archive_dir, '.spack_no_patches')
bad_file = join_path(archive_dir, '.spack_patch_failed') bad_file = join_path(archive_dir, '.spack_patch_failed')
# If we encounter an archive that failed to patch, restage it # If we encounter an archive that failed to patch, restage it
# so that we can apply all the patches again. # so that we can apply all the patches again.
@ -801,13 +795,11 @@ def do_patch(self):
else: else:
touch(no_patches_file) touch(no_patches_file)
@property @property
def namespace(self): def namespace(self):
namespace, dot, module = self.__module__.rpartition('.') namespace, dot, module = self.__module__.rpartition('.')
return namespace return namespace
def do_fake_install(self): def do_fake_install(self):
"""Make a fake install directory contaiing a 'fake' file in bin.""" """Make a fake install directory contaiing a 'fake' file in bin."""
mkdirp(self.prefix.bin) mkdirp(self.prefix.bin)
@ -815,15 +807,15 @@ def do_fake_install(self):
mkdirp(self.prefix.lib) mkdirp(self.prefix.lib)
mkdirp(self.prefix.man1) mkdirp(self.prefix.man1)
def _get_needed_resources(self): def _get_needed_resources(self):
resources = [] resources = []
# Select the resources that are needed for this build # Select the resources that are needed for this build
for when_spec, resource_list in self.resources.items(): for when_spec, resource_list in self.resources.items():
if when_spec in self.spec: if when_spec in self.spec:
resources.extend(resource_list) resources.extend(resource_list)
# Sorts the resources by the length of the string representing their destination. Since any nested resource # Sorts the resources by the length of the string representing their
# must contain another resource's name in its path, it seems that should work # destination. Since any nested resource must contain another
# resource's name in its path, it seems that should work
resources = sorted(resources, key=lambda res: len(res.destination)) resources = sorted(resources, key=lambda res: len(res.destination))
return resources return resources
@ -832,10 +824,14 @@ def _resource_stage(self, resource):
resource_stage_folder = '-'.join(pieces) resource_stage_folder = '-'.join(pieces)
return resource_stage_folder return resource_stage_folder
def do_install(self, def do_install(self,
keep_prefix=False, keep_stage=False, ignore_deps=False, keep_prefix=False,
skip_patch=False, verbose=False, make_jobs=None, fake=False): keep_stage=False,
ignore_deps=False,
skip_patch=False,
verbose=False,
make_jobs=None,
fake=False):
"""Called by commands to install a package and its dependencies. """Called by commands to install a package and its dependencies.
Package implementations should override install() to describe Package implementations should override install() to describe
@ -846,18 +842,20 @@ def do_install(self,
keep_stage -- By default, stage is destroyed only if there are no keep_stage -- By default, stage is destroyed only if there are no
exceptions during build. Set to True to keep the stage exceptions during build. Set to True to keep the stage
even with exceptions. even with exceptions.
ignore_deps -- Do not install dependencies before installing this package. ignore_deps -- Don't install dependencies before installing this
package
fake -- Don't really build -- install fake stub files instead. fake -- Don't really build -- install fake stub files instead.
skip_patch -- Skip patch stage of build if True. skip_patch -- Skip patch stage of build if True.
verbose -- Display verbose build output (by default, suppresses it) verbose -- Display verbose build output (by default, suppresses it)
make_jobs -- Number of make jobs to use for install. Default is ncpus. make_jobs -- Number of make jobs to use for install. Default is ncpus
""" """
if not self.spec.concrete: if not self.spec.concrete:
raise ValueError("Can only install concrete packages.") raise ValueError("Can only install concrete packages.")
# No installation needed if package is external # No installation needed if package is external
if self.spec.external: if self.spec.external:
tty.msg("%s is externally installed in %s" % (self.name, self.spec.external)) tty.msg("%s is externally installed in %s" %
(self.name, self.spec.external))
return return
# Ensure package is not already installed # Ensure package is not already installed
@ -869,9 +867,13 @@ def do_install(self,
# First, install dependencies recursively. # First, install dependencies recursively.
if not ignore_deps: if not ignore_deps:
self.do_install_dependencies( self.do_install_dependencies(keep_prefix=keep_prefix,
keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps, keep_stage=keep_stage,
fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs) ignore_deps=ignore_deps,
fake=fake,
skip_patch=skip_patch,
verbose=verbose,
make_jobs=make_jobs)
# Set parallelism before starting build. # Set parallelism before starting build.
self.make_jobs = make_jobs self.make_jobs = make_jobs
@ -899,35 +901,41 @@ def build_process():
self.do_fake_install() self.do_fake_install()
else: else:
# Do the real install in the source directory. # Do the real install in the source directory.
self.stage.chdir_to_source() self.stage.chdir_to_source()
# Save the build environment in a file before building. # Save the build environment in a file before building.
env_path = join_path(os.getcwd(), 'spack-build.env') env_path = join_path(os.getcwd(), 'spack-build.env')
try: try:
# Redirect I/O to a build log (and optionally to the terminal) # Redirect I/O to a build log (and optionally to
# the terminal)
log_path = join_path(os.getcwd(), 'spack-build.out') log_path = join_path(os.getcwd(), 'spack-build.out')
log_file = open(log_path, 'w') log_file = open(log_path, 'w')
with log_output(log_file, verbose, sys.stdout.isatty(), True): with log_output(log_file, verbose, sys.stdout.isatty(),
True):
dump_environment(env_path) dump_environment(env_path)
self.install(self.spec, self.prefix) self.install(self.spec, self.prefix)
except ProcessError as e: except ProcessError as e:
# Annotate ProcessErrors with the location of the build log. # Annotate ProcessErrors with the location of
e.build_log = log_path # the build log
raise e e.build_log = log_path
raise e
# Ensure that something was actually installed. # Ensure that something was actually installed.
self.sanity_check_prefix() self.sanity_check_prefix()
# Copy provenance into the install directory on success # Copy provenance into the install directory on success
log_install_path = spack.install_layout.build_log_path(self.spec) log_install_path = spack.install_layout.build_log_path(
env_install_path = spack.install_layout.build_env_path(self.spec) self.spec)
packages_dir = spack.install_layout.build_packages_path(self.spec) env_install_path = spack.install_layout.build_env_path(
self.spec)
packages_dir = spack.install_layout.build_packages_path(
self.spec)
install(log_path, log_install_path) install(log_path, log_install_path)
install(env_path, env_install_path) install(env_path, env_install_path)
dump_packages(self.spec, packages_dir) dump_packages(self.spec, packages_dir)
# Run post install hooks before build stage is removed. # Run post install hooks before build stage is removed.
spack.hooks.post_install(self) spack.hooks.post_install(self)
@ -937,8 +945,9 @@ def build_process():
build_time = self._total_time - self._fetch_time build_time = self._total_time - self._fetch_time
tty.msg("Successfully installed %s" % self.name, tty.msg("Successfully installed %s" % self.name,
"Fetch: %s. Build: %s. Total: %s." "Fetch: %s. Build: %s. Total: %s." %
% (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) (_hms(self._fetch_time), _hms(build_time),
_hms(self._total_time)))
print_pkg(self.prefix) print_pkg(self.prefix)
try: try:
@ -953,16 +962,17 @@ def build_process():
tty.warn("Keeping install prefix in place despite error.", tty.warn("Keeping install prefix in place despite error.",
"Spack will think this package is installed. " + "Spack will think this package is installed. " +
"Manually remove this directory to fix:", "Manually remove this directory to fix:",
self.prefix, wrap=True) self.prefix,
wrap=True)
raise raise
# note: PARENT of the build process adds the new package to # note: PARENT of the build process adds the new package to
# the database, so that we don't need to re-read from file. # the database, so that we don't need to re-read from file.
spack.installed_db.add(self.spec, self.prefix) spack.installed_db.add(self.spec, self.prefix)
def sanity_check_prefix(self): def sanity_check_prefix(self):
"""This function checks whether install succeeded.""" """This function checks whether install succeeded."""
def check_paths(path_list, filetype, predicate): def check_paths(path_list, filetype, predicate):
if isinstance(path_list, basestring): if isinstance(path_list, basestring):
path_list = [path_list] path_list = [path_list]
@ -970,8 +980,9 @@ def check_paths(path_list, filetype, predicate):
for path in path_list: for path in path_list:
abs_path = os.path.join(self.prefix, path) abs_path = os.path.join(self.prefix, path)
if not predicate(abs_path): if not predicate(abs_path):
raise InstallError("Install failed for %s. No such %s in prefix: %s" raise InstallError(
% (self.name, filetype, path)) "Install failed for %s. No such %s in prefix: %s" %
(self.name, filetype, path))
check_paths(self.sanity_check_is_file, 'file', os.path.isfile) check_paths(self.sanity_check_is_file, 'file', os.path.isfile)
check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir) check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir)
@ -982,13 +993,11 @@ def check_paths(path_list, filetype, predicate):
raise InstallError( raise InstallError(
"Install failed for %s. Nothing was installed!" % self.name) "Install failed for %s. Nothing was installed!" % self.name)
def do_install_dependencies(self, **kwargs): def do_install_dependencies(self, **kwargs):
# Pass along paths of dependencies here # Pass along paths of dependencies here
for dep in self.spec.dependencies.values(): for dep in self.spec.dependencies.values():
dep.package.do_install(**kwargs) dep.package.do_install(**kwargs)
@property @property
def build_log_path(self): def build_log_path(self):
if self.installed: if self.installed:
@ -996,7 +1005,6 @@ def build_log_path(self):
else: else:
return join_path(self.stage.source_path, 'spack-build.out') return join_path(self.stage.source_path, 'spack-build.out')
@property @property
def module(self): def module(self):
"""Use this to add variables to the class's module's scope. """Use this to add variables to the class's module's scope.
@ -1037,7 +1045,6 @@ def setup_environment(self, spack_env, run_env):
""" """
pass pass
def setup_dependent_environment(self, spack_env, run_env, dependent_spec): def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
"""Set up the environment of packages that depend on this one. """Set up the environment of packages that depend on this one.
@ -1077,7 +1084,6 @@ def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
""" """
self.setup_environment(spack_env, run_env) self.setup_environment(spack_env, run_env)
def setup_dependent_package(self, module, dependent_spec): def setup_dependent_package(self, module, dependent_spec):
"""Set up Python module-scope variables for dependent packages. """Set up Python module-scope variables for dependent packages.
@ -1123,8 +1129,11 @@ def setup_dependent_package(self, module, dependent_spec):
pass pass
def install(self, spec, prefix): def install(self, spec, prefix):
"""Package implementations override this with their own build configuration.""" """
raise InstallError("Package %s provides no install method!" % self.name) Package implementations override this with their own configuration
"""
raise InstallError("Package %s provides no install method!" %
self.name)
def do_uninstall(self, force=False): def do_uninstall(self, force=False):
if not self.installed: if not self.installed:
@ -1146,12 +1155,10 @@ def do_uninstall(self, force=False):
# Once everything else is done, run post install hooks # Once everything else is done, run post install hooks
spack.hooks.post_uninstall(self) spack.hooks.post_uninstall(self)
def _check_extendable(self): def _check_extendable(self):
if not self.extendable: if not self.extendable:
raise ValueError("Package %s is not extendable!" % self.name) raise ValueError("Package %s is not extendable!" % self.name)
def _sanity_check_extension(self): def _sanity_check_extension(self):
if not self.is_extension: if not self.is_extension:
raise ActivationError("This package is not an extension.") raise ActivationError("This package is not an extension.")
@ -1160,12 +1167,13 @@ def _sanity_check_extension(self):
extendee_package._check_extendable() extendee_package._check_extendable()
if not extendee_package.installed: if not extendee_package.installed:
raise ActivationError("Can only (de)activate extensions for installed packages.") raise ActivationError(
"Can only (de)activate extensions for installed packages.")
if not self.installed: if not self.installed:
raise ActivationError("Extensions must first be installed.") raise ActivationError("Extensions must first be installed.")
if not self.extendee_spec.name in self.extendees: if self.extendee_spec.name not in self.extendees:
raise ActivationError("%s does not extend %s!" % (self.name, self.extendee.name)) raise ActivationError("%s does not extend %s!" %
(self.name, self.extendee.name))
def do_activate(self, force=False): def do_activate(self, force=False):
"""Called on an etension to invoke the extendee's activate method. """Called on an etension to invoke the extendee's activate method.
@ -1175,8 +1183,8 @@ def do_activate(self, force=False):
""" """
self._sanity_check_extension() self._sanity_check_extension()
spack.install_layout.check_extension_conflict( spack.install_layout.check_extension_conflict(self.extendee_spec,
self.extendee_spec, self.spec) self.spec)
# Activate any package dependencies that are also extensions. # Activate any package dependencies that are also extensions.
if not force: if not force:
@ -1188,9 +1196,8 @@ def do_activate(self, force=False):
self.extendee_spec.package.activate(self, **self.extendee_args) self.extendee_spec.package.activate(self, **self.extendee_args)
spack.install_layout.add_extension(self.extendee_spec, self.spec) spack.install_layout.add_extension(self.extendee_spec, self.spec)
tty.msg("Activated extension %s for %s" tty.msg("Activated extension %s for %s" %
% (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@")))
def activate(self, extension, **kwargs): def activate(self, extension, **kwargs):
"""Symlinks all files from the extension into extendee's install dir. """Symlinks all files from the extension into extendee's install dir.
@ -1201,6 +1208,7 @@ def activate(self, extension, **kwargs):
always executed. always executed.
""" """
def ignore(filename): def ignore(filename):
return (filename in spack.install_layout.hidden_file_paths or return (filename in spack.install_layout.hidden_file_paths or
kwargs.get('ignore', lambda f: False)(filename)) kwargs.get('ignore', lambda f: False)(filename))
@ -1212,7 +1220,6 @@ def ignore(filename):
tree.merge(self.prefix, ignore=ignore) tree.merge(self.prefix, ignore=ignore)
def do_deactivate(self, **kwargs): def do_deactivate(self, **kwargs):
"""Called on the extension to invoke extendee's deactivate() method.""" """Called on the extension to invoke extendee's deactivate() method."""
self._sanity_check_extension() self._sanity_check_extension()
@ -1230,7 +1237,7 @@ def do_deactivate(self, **kwargs):
for dep in aspec.traverse(): for dep in aspec.traverse():
if self.spec == dep: if self.spec == dep:
raise ActivationError( raise ActivationError(
"Cannot deactivate %s beacuse %s is activated and depends on it." "Cannot deactivate %s because %s is activated and depends on it." # NOQA: ignore=E501
% (self.spec.short_spec, aspec.short_spec)) % (self.spec.short_spec, aspec.short_spec))
self.extendee_spec.package.deactivate(self, **self.extendee_args) self.extendee_spec.package.deactivate(self, **self.extendee_args)
@ -1238,11 +1245,11 @@ def do_deactivate(self, **kwargs):
# redundant activation check -- makes SURE the spec is not # redundant activation check -- makes SURE the spec is not
# still activated even if something was wrong above. # still activated even if something was wrong above.
if self.activated: if self.activated:
spack.install_layout.remove_extension(self.extendee_spec, self.spec) spack.install_layout.remove_extension(self.extendee_spec,
self.spec)
tty.msg("Deactivated extension %s for %s"
% (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@")))
tty.msg("Deactivated extension %s for %s" %
(self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@")))
def deactivate(self, extension, **kwargs): def deactivate(self, extension, **kwargs):
"""Unlinks all files from extension out of this package's install dir. """Unlinks all files from extension out of this package's install dir.
@ -1253,6 +1260,7 @@ def deactivate(self, extension, **kwargs):
always executed. always executed.
""" """
def ignore(filename): def ignore(filename):
return (filename in spack.install_layout.hidden_file_paths or return (filename in spack.install_layout.hidden_file_paths or
kwargs.get('ignore', lambda f: False)(filename)) kwargs.get('ignore', lambda f: False)(filename))
@ -1260,17 +1268,14 @@ def ignore(filename):
tree = LinkTree(extension.prefix) tree = LinkTree(extension.prefix)
tree.unmerge(self.prefix, ignore=ignore) tree.unmerge(self.prefix, ignore=ignore)
def do_restage(self): def do_restage(self):
"""Reverts expanded/checked out source to a pristine state.""" """Reverts expanded/checked out source to a pristine state."""
self.stage.restage() self.stage.restage()
def do_clean(self): def do_clean(self):
"""Removes the package's build stage and source tarball.""" """Removes the package's build stage and source tarball."""
self.stage.destroy() self.stage.destroy()
def format_doc(self, **kwargs): def format_doc(self, **kwargs):
"""Wrap doc string at 72 characters and format nicely""" """Wrap doc string at 72 characters and format nicely"""
indent = kwargs.get('indent', 0) indent = kwargs.get('indent', 0)
@ -1285,7 +1290,6 @@ def format_doc(self, **kwargs):
results.write((" " * indent) + line + "\n") results.write((" " * indent) + line + "\n")
return results.getvalue() return results.getvalue()
@property @property
def all_urls(self): def all_urls(self):
urls = [] urls = []
@ -1297,7 +1301,6 @@ def all_urls(self):
urls.append(args['url']) urls.append(args['url'])
return urls return urls
def fetch_remote_versions(self): def fetch_remote_versions(self):
"""Try to find remote versions of this package using the """Try to find remote versions of this package using the
list_url and any other URLs described in the package file.""" list_url and any other URLs described in the package file."""
@ -1306,26 +1309,30 @@ def fetch_remote_versions(self):
try: try:
return spack.util.web.find_versions_of_archive( return spack.util.web.find_versions_of_archive(
*self.all_urls, list_url=self.list_url, list_depth=self.list_depth) *self.all_urls,
list_url=self.list_url,
list_depth=self.list_depth)
except spack.error.NoNetworkConnectionError as e: except spack.error.NoNetworkConnectionError as e:
tty.die("Package.fetch_versions couldn't connect to:", tty.die("Package.fetch_versions couldn't connect to:", e.url,
e.url, e.message) e.message)
@property @property
def rpath(self): def rpath(self):
"""Get the rpath this package links with, as a list of paths.""" """Get the rpath this package links with, as a list of paths."""
rpaths = [self.prefix.lib, self.prefix.lib64] rpaths = [self.prefix.lib, self.prefix.lib64]
rpaths.extend(d.prefix.lib for d in self.spec.traverse(root=False) rpaths.extend(d.prefix.lib
for d in self.spec.traverse(root=False)
if os.path.isdir(d.prefix.lib)) if os.path.isdir(d.prefix.lib))
rpaths.extend(d.prefix.lib64 for d in self.spec.traverse(root=False) rpaths.extend(d.prefix.lib64
for d in self.spec.traverse(root=False)
if os.path.isdir(d.prefix.lib64)) if os.path.isdir(d.prefix.lib64))
return rpaths return rpaths
@property @property
def rpath_args(self): def rpath_args(self):
"""Get the rpath args as a string, with -Wl,-rpath, for each element.""" """
Get the rpath args as a string, with -Wl,-rpath, for each element
"""
return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) return " ".join("-Wl,-rpath,%s" % p for p in self.rpath)
@ -1333,6 +1340,7 @@ def install_dependency_symlinks(pkg, spec, prefix):
"""Execute a dummy install and flatten dependencies""" """Execute a dummy install and flatten dependencies"""
flatten_dependencies(spec, prefix) flatten_dependencies(spec, prefix)
def flatten_dependencies(spec, flat_dir): def flatten_dependencies(spec, flat_dir):
"""Make each dependency of spec present in dir via symlink.""" """Make each dependency of spec present in dir via symlink."""
for dep in spec.traverse(root=False): for dep in spec.traverse(root=False):
@ -1341,13 +1349,13 @@ def flatten_dependencies(spec, flat_dir):
dep_path = spack.install_layout.path_for_spec(dep) dep_path = spack.install_layout.path_for_spec(dep)
dep_files = LinkTree(dep_path) dep_files = LinkTree(dep_path)
os.mkdir(flat_dir+'/'+name) os.mkdir(flat_dir + '/' + name)
conflict = dep_files.find_conflict(flat_dir+'/'+name) conflict = dep_files.find_conflict(flat_dir + '/' + name)
if conflict: if conflict:
raise DependencyConflictError(conflict) raise DependencyConflictError(conflict)
dep_files.merge(flat_dir+'/'+name) dep_files.merge(flat_dir + '/' + name)
def validate_package_url(url_string): def validate_package_url(url_string):
@ -1388,9 +1396,11 @@ def dump_packages(spec, path):
# Create a source repo and get the pkg directory out of it. # Create a source repo and get the pkg directory out of it.
try: try:
source_repo = spack.repository.Repo(source_repo_root) source_repo = spack.repository.Repo(source_repo_root)
source_pkg_dir = source_repo.dirname_for_package_name(node.name) source_pkg_dir = source_repo.dirname_for_package_name(
except RepoError as e: node.name)
tty.warn("Warning: Couldn't copy in provenance for %s" % node.name) except RepoError:
tty.warn("Warning: Couldn't copy in provenance for %s" %
node.name)
# Create a destination repository # Create a destination repository
dest_repo_root = join_path(path, node.namespace) dest_repo_root = join_path(path, node.namespace)
@ -1410,7 +1420,7 @@ def print_pkg(message):
"""Outputs a message with a package icon.""" """Outputs a message with a package icon."""
from llnl.util.tty.color import cwrite from llnl.util.tty.color import cwrite
cwrite('@*g{[+]} ') cwrite('@*g{[+]} ')
print message print(message)
def _hms(seconds): def _hms(seconds):
@ -1419,20 +1429,25 @@ def _hms(seconds):
h, m = divmod(m, 60) h, m = divmod(m, 60)
parts = [] parts = []
if h: parts.append("%dh" % h) if h:
if m: parts.append("%dm" % m) parts.append("%dh" % h)
if s: parts.append("%.2fs" % s) if m:
parts.append("%dm" % m)
if s:
parts.append("%.2fs" % s)
return ' '.join(parts) return ' '.join(parts)
class FetchError(spack.error.SpackError): class FetchError(spack.error.SpackError):
"""Raised when something goes wrong during fetch.""" """Raised when something goes wrong during fetch."""
def __init__(self, message, long_msg=None): def __init__(self, message, long_msg=None):
super(FetchError, self).__init__(message, long_msg) super(FetchError, self).__init__(message, long_msg)
class InstallError(spack.error.SpackError): class InstallError(spack.error.SpackError):
"""Raised when something goes wrong during install or uninstall.""" """Raised when something goes wrong during install or uninstall."""
def __init__(self, message, long_msg=None): def __init__(self, message, long_msg=None):
super(InstallError, self).__init__(message, long_msg) super(InstallError, self).__init__(message, long_msg)
@ -1443,21 +1458,24 @@ class ExternalPackageError(InstallError):
class PackageStillNeededError(InstallError): class PackageStillNeededError(InstallError):
"""Raised when package is still needed by another on uninstall.""" """Raised when package is still needed by another on uninstall."""
def __init__(self, spec, dependents): def __init__(self, spec, dependents):
super(PackageStillNeededError, self).__init__( super(PackageStillNeededError, self).__init__("Cannot uninstall %s" %
"Cannot uninstall %s" % spec) spec)
self.spec = spec self.spec = spec
self.dependents = dependents self.dependents = dependents
class PackageError(spack.error.SpackError): class PackageError(spack.error.SpackError):
"""Raised when something is wrong with a package definition.""" """Raised when something is wrong with a package definition."""
def __init__(self, message, long_msg=None): def __init__(self, message, long_msg=None):
super(PackageError, self).__init__(message, long_msg) super(PackageError, self).__init__(message, long_msg)
class PackageVersionError(PackageError): class PackageVersionError(PackageError):
"""Raised when a version URL cannot automatically be determined.""" """Raised when a version URL cannot automatically be determined."""
def __init__(self, version): def __init__(self, version):
super(PackageVersionError, self).__init__( super(PackageVersionError, self).__init__(
"Cannot determine a URL automatically for version %s" % version, "Cannot determine a URL automatically for version %s" % version,
@ -1466,6 +1484,7 @@ def __init__(self, version):
class VersionFetchError(PackageError): class VersionFetchError(PackageError):
"""Raised when a version URL cannot automatically be determined.""" """Raised when a version URL cannot automatically be determined."""
def __init__(self, cls): def __init__(self, cls):
super(VersionFetchError, self).__init__( super(VersionFetchError, self).__init__(
"Cannot fetch versions for package %s " % cls.__name__ + "Cannot fetch versions for package %s " % cls.__name__ +
@ -1474,12 +1493,15 @@ def __init__(self, cls):
class NoURLError(PackageError): class NoURLError(PackageError):
"""Raised when someone tries to build a URL for a package with no URLs.""" """Raised when someone tries to build a URL for a package with no URLs."""
def __init__(self, cls): def __init__(self, cls):
super(NoURLError, self).__init__( super(NoURLError, self).__init__(
"Package %s has no version with a URL." % cls.__name__) "Package %s has no version with a URL." % cls.__name__)
class ExtensionError(PackageError): pass class ExtensionError(PackageError):
pass
class ExtensionConflictError(ExtensionError): class ExtensionConflictError(ExtensionError):
@ -1495,7 +1517,8 @@ def __init__(self, msg, long_msg=None):
class DependencyConflictError(spack.error.SpackError): class DependencyConflictError(spack.error.SpackError):
"""Raised when the dependencies cannot be flattened as asked for.""" """Raised when the dependencies cannot be flattened as asked for."""
def __init__(self, conflict): def __init__(self, conflict):
super(DependencyConflictError, self).__init__( super(DependencyConflictError, self).__init__(
"%s conflicts with another file in the flattened directory." %( "%s conflicts with another file in the flattened directory." % (
conflict)) conflict))

70
lib/spack/spack/test/__init__.py
View File

@ -23,53 +23,23 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
############################################################################## ##############################################################################
import sys import sys
import unittest
import nose
from spack.test.tally_plugin import Tally
from llnl.util.filesystem import join_path
import llnl.util.tty as tty import llnl.util.tty as tty
from llnl.util.tty.colify import colify import nose
import spack import spack
from llnl.util.filesystem import join_path
from llnl.util.tty.colify import colify
from spack.test.tally_plugin import Tally
"""Names of tests to be included in Spack's test suite""" """Names of tests to be included in Spack's test suite"""
test_names = ['versions', test_names = ['versions', 'url_parse', 'url_substitution', 'packages', 'stage',
'url_parse', 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize',
'url_substitution', 'multimethod', 'install', 'package_sanity', 'config',
'packages', 'directory_layout', 'pattern', 'python_version', 'git_fetch',
'stage', 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate',
'spec_syntax', 'cc', 'link_tree', 'spec_yaml', 'optional_deps',
'spec_semantics', 'make_executable', 'configure_guess', 'lock', 'database',
'spec_dag', 'namespace_trie', 'yaml', 'sbang', 'environment',
'concretize', 'cmd.uninstall', 'cmd.test_install']
'multimethod',
'install',
'package_sanity',
'config',
'directory_layout',
'pattern',
'python_version',
'git_fetch',
'svn_fetch',
'hg_fetch',
'mirror',
'modules',
'url_extrapolate',
'cc',
'link_tree',
'spec_yaml',
'optional_deps',
'make_executable',
'configure_guess',
'lock',
'database',
'namespace_trie',
'yaml',
'sbang',
'environment',
'cmd.uninstall',
'cmd.test_install']
def list_tests(): def list_tests():
@ -80,8 +50,6 @@ def list_tests():
def run(names, outputDir, verbose=False): def run(names, outputDir, verbose=False):
"""Run tests with the supplied names. Names should be a list. If """Run tests with the supplied names. Names should be a list. If
it's empty, run ALL of Spack's tests.""" it's empty, run ALL of Spack's tests."""
verbosity = 1 if not verbose else 2
if not names: if not names:
names = test_names names = test_names
else: else:
@ -95,7 +63,7 @@ def run(names, outputDir, verbose=False):
tally = Tally() tally = Tally()
for test in names: for test in names:
module = 'spack.test.' + test module = 'spack.test.' + test
print module print(module)
tty.msg("Running test: %s" % test) tty.msg("Running test: %s" % test)
@ -105,15 +73,13 @@ def run(names, outputDir, verbose=False):
xmlOutputFname = "unittests-{0}.xml".format(test) xmlOutputFname = "unittests-{0}.xml".format(test)
xmlOutputPath = join_path(outputDir, xmlOutputFname) xmlOutputPath = join_path(outputDir, xmlOutputFname)
runOpts += ["--with-xunit", runOpts += ["--with-xunit",
"--xunit-file={0}".format(xmlOutputPath)] "--xunit-file={0}".format(xmlOutputPath)]
argv = [""] + runOpts + [module] argv = [""] + runOpts + [module]
result = nose.run(argv=argv, addplugins=[tally]) nose.run(argv=argv, addplugins=[tally])
succeeded = not tally.failCount and not tally.errorCount succeeded = not tally.failCount and not tally.errorCount
tty.msg("Tests Complete.", tty.msg("Tests Complete.", "%5d tests run" % tally.numberOfTestsRun,
"%5d tests run" % tally.numberOfTestsRun, "%5d failures" % tally.failCount, "%5d errors" % tally.errorCount)
"%5d failures" % tally.failCount,
"%5d errors" % tally.errorCount)
if succeeded: if succeeded:
tty.info("OK", format='g') tty.info("OK", format='g')

42
lib/spack/spack/test/modules.py
View File

@ -2,14 +2,18 @@
from contextlib import contextmanager from contextlib import contextmanager
import StringIO import StringIO
import spack.modules
from spack.test.mock_packages_test import MockPackagesTest
FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) FILE_REGISTRY = collections.defaultdict(StringIO.StringIO)
# Monkey-patch open to write module files to a StringIO instance # Monkey-patch open to write module files to a StringIO instance
@contextmanager @contextmanager
def mock_open(filename, mode): def mock_open(filename, mode):
if not mode == 'w': if not mode == 'w':
raise RuntimeError('test.modules : unexpected opening mode for monkey-patched open') raise RuntimeError(
'test.modules : unexpected opening mode for monkey-patched open')
FILE_REGISTRY[filename] = StringIO.StringIO() FILE_REGISTRY[filename] = StringIO.StringIO()
@ -20,7 +24,6 @@ def mock_open(filename, mode):
FILE_REGISTRY[filename] = handle.getvalue() FILE_REGISTRY[filename] = handle.getvalue()
handle.close() handle.close()
import spack.modules
configuration_autoload_direct = { configuration_autoload_direct = {
'enable': ['tcl'], 'enable': ['tcl'],
@ -47,7 +50,8 @@ def mock_open(filename, mode):
'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']}
}, },
'=x86-linux': { '=x86-linux': {
'environment': {'set': {'FOO': 'foo'}, 'unset': ['BAR']} 'environment': {'set': {'FOO': 'foo'},
'unset': ['BAR']}
} }
} }
} }
@ -72,15 +76,14 @@ def mock_open(filename, mode):
} }
} }
from spack.test.mock_packages_test import MockPackagesTest
class TclTests(MockPackagesTest): class TclTests(MockPackagesTest):
def setUp(self): def setUp(self):
super(TclTests, self).setUp() super(TclTests, self).setUp()
self.configuration_obj = spack.modules.CONFIGURATION self.configuration_obj = spack.modules.CONFIGURATION
spack.modules.open = mock_open spack.modules.open = mock_open
spack.modules.CONFIGURATION = None # Make sure that a non-mocked configuration will trigger an error # Make sure that a non-mocked configuration will trigger an error
spack.modules.CONFIGURATION = None
def tearDown(self): def tearDown(self):
del spack.modules.open del spack.modules.open
@ -98,7 +101,7 @@ def test_simple_case(self):
spack.modules.CONFIGURATION = configuration_autoload_direct spack.modules.CONFIGURATION = configuration_autoload_direct
spec = spack.spec.Spec('mpich@3.0.4=x86-linux') spec = spack.spec.Spec('mpich@3.0.4=x86-linux')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertTrue('module-whatis "mpich @3.0.4"' in content ) self.assertTrue('module-whatis "mpich @3.0.4"' in content)
def test_autoload(self): def test_autoload(self):
spack.modules.CONFIGURATION = configuration_autoload_direct spack.modules.CONFIGURATION = configuration_autoload_direct
@ -117,14 +120,22 @@ def test_alter_environment(self):
spack.modules.CONFIGURATION = configuration_alter_environment spack.modules.CONFIGURATION = configuration_alter_environment
spec = spack.spec.Spec('mpileaks=x86-linux') spec = spack.spec.Spec('mpileaks=x86-linux')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) self.assertEqual(
self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 1) len([x
for x in content
if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0)
self.assertEqual(
len([x for x in content if 'setenv FOO "foo"' in x]), 1)
self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1)
spec = spack.spec.Spec('libdwarf=x64-linux') spec = spack.spec.Spec('libdwarf=x64-linux')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0) self.assertEqual(
self.assertEqual(len([x for x in content if 'setenv FOO "foo"' in x]), 0) len([x
for x in content
if x.startswith('prepend-path CMAKE_PREFIX_PATH')]), 0)
self.assertEqual(
len([x for x in content if 'setenv FOO "foo"' in x]), 0)
self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0)
def test_blacklist(self): def test_blacklist(self):
@ -138,6 +149,9 @@ def test_conflicts(self):
spack.modules.CONFIGURATION = configuration_conflicts spack.modules.CONFIGURATION = configuration_conflicts
spec = spack.spec.Spec('mpileaks=x86-linux') spec = spack.spec.Spec('mpileaks=x86-linux')
content = self.get_modulefile_content(spec) content = self.get_modulefile_content(spec)
self.assertEqual(len([x for x in content if x.startswith('conflict')]), 2) self.assertEqual(
self.assertEqual(len([x for x in content if x == 'conflict mpileaks']), 1) len([x for x in content if x.startswith('conflict')]), 2)
self.assertEqual(len([x for x in content if x == 'conflict intel/14.0.1']), 1) self.assertEqual(
len([x for x in content if x == 'conflict mpileaks']), 1)
self.assertEqual(
len([x for x in content if x == 'conflict intel/14.0.1']), 1)