zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' of ssh://cz-stash.llnl.gov:7999/scale/spack into develop

Browse Source
This commit is contained in:
Gregory L. Lee 2015-06-04 15:49:57 -07:00
commit 96ec75f7fe
37 changed files with 555 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/spack/llnl/util/filesystem.py
View File

@ -152,7 +152,7 @@ def copy_mode(src, dest):
def install(src, dest):
"""Manually install a file to a particular location."""
tty.info("Installing %s to %s" % (src, dest))
tty.debug("Installing %s to %s" % (src, dest))
shutil.copy(src, dest)
set_install_permissions(dest)
copy_mode(src, dest)
@ -160,7 +160,7 @@ def install(src, dest):
def install_tree(src, dest, **kwargs):
"""Manually install a file to a particular location."""
tty.info("Installing %s to %s" % (src, dest))
tty.debug("Installing %s to %s" % (src, dest))
shutil.copytree(src, dest, **kwargs)
for s, d in traverse_tree(src, dest, follow_nonexisting=False):

8
lib/spack/llnl/util/tty/__init__.py
View File

@ -36,6 +36,14 @@
_verbose = False
indent = " "
def is_verbose():
return _verbose
def is_debug():
return _debug
def set_debug(flag):
global _debug
_debug = flag

8
lib/spack/llnl/util/tty/color.py
View File

@ -99,6 +99,10 @@ def __init__(self, message):
color_re = r'@(?:@|\.|([*_])?([a-zA-Z])?(?:{((?:[^}]|}})*)})?)'
# Force color even if stdout is not a tty.
_force_color = False
class match_to_ansi(object):
def __init__(self, color=True):
self.color = color
@ -162,7 +166,7 @@ def cwrite(string, stream=sys.stdout, color=None):
then it will be set based on stream.isatty().
"""
if color is None:
color = stream.isatty()
color = stream.isatty() or _force_color
stream.write(colorize(string, color=color))
@ -189,7 +193,7 @@ def write(self, string, **kwargs):
if raw:
color=True
else:
color = self._stream.isatty()
color = self._stream.isatty() or _force_color
raw_write(colorize(string, color=color))
def writelines(self, sequence, **kwargs):

178
lib/spack/llnl/util/tty/log.py Normal file
View File

@ -0,0 +1,178 @@
##############################################################################
# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""Utility classes for logging the output of blocks of code.
"""
import sys
import os
import re
import select
import inspect
import llnl.util.tty as tty
import llnl.util.tty.color as color
# Use this to strip escape sequences
_escape = re.compile(r'\x1b[^m]*m|\x1b\[?1034h')
def _strip(line):
"""Strip color and control characters from a line."""
return _escape.sub('', line)
class _SkipWithBlock():
"""Special exception class used to skip a with block."""
pass
class log_output(object):
"""Redirects output and error of enclosed block to a file.
Usage:
with log_output(open('logfile.txt', 'w')):
# do things ... output will be logged.
or:
with log_output(open('logfile.txt', 'w'), echo=True):
# do things ... output will be logged
# and also printed to stdout.
Closes the provided stream when done with the block.
If echo is True, also prints the output to stdout.
"""
def __init__(self, stream, echo=False, force_color=False, debug=False):
self.stream = stream
# various output options
self.echo = echo
self.force_color = force_color
self.debug = debug
def trace(self, frame, event, arg):
"""Jumps to __exit__ on the child process."""
raise _SkipWithBlock()
def __enter__(self):
"""Redirect output from the with block to a file.
This forks the with block as a separate process, with stdout
and stderr redirected back to the parent via a pipe. If
echo is set, also writes to standard out.
"""
# remember these values for later.
self._force_color = color._force_color
self._debug = tty._debug
read, write = os.pipe()
self.pid = os.fork()
if self.pid:
# Parent: read from child, skip the with block.
os.close(write)
read_file = os.fdopen(read, 'r', 0)
with self.stream as log_file:
while True:
rlist, w, x = select.select([read_file], [], [])
if not rlist:
break
line = read_file.readline()
if not line:
break
# Echo to stdout if requested.
if self.echo:
sys.stdout.write(line)
# Stripped output to log file.
log_file.write(_strip(line))
read_file.flush()
read_file.close()
# Set a trace function to skip the with block.
sys.settrace(lambda *args, **keys: None)
frame = inspect.currentframe(1)
frame.f_trace = self.trace
else:
# Child: redirect output, execute the with block.
os.close(read)
# Save old stdout and stderr
self._stdout = os.dup(sys.stdout.fileno())
self._stderr = os.dup(sys.stderr.fileno())
# redirect to the pipe.
os.dup2(write, sys.stdout.fileno())
os.dup2(write, sys.stderr.fileno())
if self.force_color:
color._force_color = True
if self.debug:
tty._debug = True
def __exit__(self, exc_type, exception, traceback):
"""Exits on child, handles skipping the with block on parent."""
# Child should just exit here.
if self.pid == 0:
# Flush the log to disk.
sys.stdout.flush()
sys.stderr.flush()
if exception:
# Restore stdout on the child if there's an exception,
# and let it be raised normally.
#
# This assumes that even if the exception is caught,
# the child will exit with a nonzero return code. If
# it doesn't, the child process will continue running.
#
# TODO: think about how this works outside install.
# TODO: ideally would propagate exception to parent...
os.dup2(self._stdout, sys.stdout.fileno())
os.dup2(self._stderr, sys.stderr.fileno())
return False
else:
# Die quietly if there was no exception.
os._exit(0)
else:
# If the child exited badly, parent also should exit.
pid, returncode = os.waitpid(self.pid, 0)
if returncode != 0:
os._exit(1)
# restore output options.
color._force_color = self._force_color
tty._debug = self._debug
# Suppresses exception if it's our own.
return exc_type is _SkipWithBlock

8
lib/spack/spack/build_environment.py
View File

@ -73,14 +73,14 @@ def __init__(self, name, jobs):
self.jobs = jobs
def __call__(self, *args, **kwargs):
parallel = kwargs.get('parallel', self.jobs > 1)
disable_parallel = env_flag(SPACK_NO_PARALLEL_MAKE)
disable = env_flag(SPACK_NO_PARALLEL_MAKE)
parallel = not disable and kwargs.get('parallel', self.jobs > 1)
if self.jobs > 1 and not disable_parallel:
if parallel:
jobs = "-j%d" % self.jobs
args = (jobs,) + args
super(MakeExecutable, self).__call__(*args, **kwargs)
return super(MakeExecutable, self).__call__(*args, **kwargs)
def set_compiler_environment_variables(pkg):

2
lib/spack/spack/cmd/edit.py
View File

@ -95,6 +95,7 @@ def edit(parser, args):
path = join_path(spack.cmd.command_path, name + ".py")
if not os.path.exists(path):
tty.die("No command named '%s'." % name)
spack.editor(path)
else:
# By default open the directory where packages or commands live.
@ -103,4 +104,3 @@ def edit(parser, args):
spack.editor(path)
else:
edit_package(name, args.force)

22
lib/spack/spack/cmd/find.py
View File

@ -50,20 +50,28 @@ def setup_parser(subparser):
subparser.add_argument(
'-l', '--long', action='store_true', dest='long',
help='Show dependency hashes as well as versions.')
subparser.add_argument(
'-L', '--very-long', action='store_true', dest='very_long',
help='Show dependency hashes as well as versions.')
subparser.add_argument(
'query_specs', nargs=argparse.REMAINDER,
help='optional specs to filter results')
def gray_hash(spec):
return colorize('@K{[%s]}' % spec.dag_hash(7))
def gray_hash(spec, length):
return colorize('@K{%s}' % spec.dag_hash(length))
def display_specs(specs, **kwargs):
mode = kwargs.get('mode', 'short')
hashes = kwargs.get('long', False)
hlen = 7
if kwargs.get('very_long', False):
hashes = True
hlen = None
# Make a dict with specs keyed by architecture and compiler.
index = index_by(specs, ('architecture', 'compiler'))
@ -87,6 +95,8 @@ def display_specs(specs, **kwargs):
format = " %-{}s%s".format(width)
for abbrv, spec in zip(abbreviated, specs):
if hashes:
print gray_hash(spec, hlen),
print format % (abbrv, spec.prefix)
elif mode == 'deps':
@ -95,13 +105,13 @@ def display_specs(specs, **kwargs):
format='$_$@$+',
color=True,
indent=4,
prefix=(lambda s: gray_hash(s)) if hashes else None)
prefix=(lambda s: gray_hash(s, hlen)) if hashes else None)
elif mode == 'short':
def fmt(s):
string = ""
if hashes:
string += gray_hash(s) + ' '
string += gray_hash(s, hlen) + ' '
string += s.format('$-_$@$+', color=True)
return string
colify(fmt(s) for s in specs)
@ -138,4 +148,6 @@ def find(parser, args):
if sys.stdout.isatty():
tty.msg("%d installed packages." % len(specs))
display_specs(specs, mode=args.mode, long=args.long)
display_specs(specs, mode=args.mode,
long=args.long,
very_long=args.very_long)

4
lib/spack/spack/cmd/install.py
View File

@ -47,6 +47,9 @@ def setup_parser(subparser):
subparser.add_argument(
'-n', '--no-checksum', action='store_true', dest='no_checksum',
help="Do not check packages against checksum")
subparser.add_argument(
'-v', '--verbose', action='store_true', dest='verbose',
help="Display verbose build output while installing.")
subparser.add_argument(
'--fake', action='store_true', dest='fake',
help="Fake install. Just remove the prefix and touch a fake file in it.")
@ -73,4 +76,5 @@ def install(parser, args):
keep_stage=args.keep_stage,
ignore_deps=args.ignore_deps,
make_jobs=args.jobs,
verbose=args.verbose,
fake=args.fake)

5
lib/spack/spack/cmd/location.py
View File

@ -54,6 +54,8 @@ def setup_parser(subparser):
help="Top-level packages directory for Spack.")
directories.add_argument(
'-s', '--stage-dir', action='store_true', help="Stage directory for a spec.")
directories.add_argument(
'-S', '--stages', action='store_true', help="Top level Stage directory.")
directories.add_argument(
'-b', '--build-dir', action='store_true',
help="Checked out or expanded source directory for a spec (requires it to be staged first).")
@ -72,6 +74,9 @@ def location(parser, args):
elif args.packages:
print spack.db.root
elif args.stages:
print spack.stage_path
else:
specs = spack.cmd.parse_specs(args.spec)
if not specs:

7
lib/spack/spack/directory_layout.py
View File

@ -35,7 +35,6 @@
from llnl.util.lang import memoized
from llnl.util.filesystem import join_path, mkdirp
import spack
from spack.spec import Spec
from spack.error import SpackError
@ -175,6 +174,7 @@ def __init__(self, root, **kwargs):
self.spec_file_name = 'spec.yaml'
self.extension_file_name = 'extensions.yaml'
self.build_log_name = 'build.out' # TODO: use config file.
# Cache of already written/read extension maps.
self._extension_maps = {}
@ -233,6 +233,11 @@ def metadata_path(self, spec):
return join_path(self.path_for_spec(spec), self.metadata_dir)
def build_log_path(self, spec):
return join_path(self.path_for_spec(spec), self.metadata_dir,
self.build_log_name)
def create_install_directory(self, spec):
_check_concrete(spec)

7
lib/spack/spack/error.py
View File

@ -30,7 +30,12 @@ class SpackError(Exception):
def __init__(self, message, long_message=None):
super(SpackError, self).__init__()
self.message = message
self.long_message = long_message
self._long_message = long_message
@property
def long_message(self):
return self._long_message
def __str__(self):

19
lib/spack/spack/fetch_strategy.py
View File

@ -220,13 +220,22 @@ def expand(self):
os.chdir(tarball_container)
decompress(self.archive_file)
# If the tarball *didn't* explode, move
# the expanded directory up & remove the protector directory.
# Check for an exploding tarball, i.e. one that doesn't expand
# to a single directory. If the tarball *didn't* explode,
# move contents up & remove the container directory.
#
# NOTE: The tar program on Mac OS X will encode HFS metadata
# in hidden files, which can end up *alongside* a single
# top-level directory. We ignore hidden files to accomodate
# these "semi-exploding" tarballs.
files = os.listdir(tarball_container)
if len(files) == 1:
expanded_dir = os.path.join(tarball_container, files[0])
non_hidden = filter(lambda f: not f.startswith('.'), files)
if len(non_hidden) == 1:
expanded_dir = os.path.join(tarball_container, non_hidden[0])
if os.path.isdir(expanded_dir):
shutil.move(expanded_dir, self.stage.path)
for f in files:
shutil.move(os.path.join(tarball_container, f),
os.path.join(self.stage.path, f))
os.rmdir(tarball_container)
# Set the wd back to the stage when done.

92
lib/spack/spack/package.py
View File

@ -36,7 +36,7 @@
import os
import re
import time
import inspect
import itertools
import subprocess
import platform as py_platform
import multiprocessing
@ -45,6 +45,7 @@
from StringIO import StringIO
import llnl.util.tty as tty
from llnl.util.tty.log import log_output
from llnl.util.link_tree import LinkTree
from llnl.util.filesystem import *
from llnl.util.lang import *
@ -55,12 +56,12 @@
import spack.mirror
import spack.hooks
import spack.directives
import spack.build_environment as build_env
import spack.url as url
import spack.build_environment
import spack.url
import spack.util.web
import spack.fetch_strategy as fs
from spack.version import *
from spack.stage import Stage
from spack.util.web import get_pages
from spack.util.compression import allowed_archive, extension
from spack.util.executable import ProcessError
@ -427,8 +428,8 @@ def url_for_version(self, version):
return version_urls[version]
# If we have no idea, try to substitute the version.
return url.substitute_version(self.nearest_url(version),
self.url_version(version))
return spack.url.substitute_version(self.nearest_url(version),
self.url_version(version))
@property
@ -711,20 +712,28 @@ def do_fake_install(self):
mkdirp(self.prefix.man1)
def do_install(self, **kwargs):
"""This class should call this version of the install method.
Package implementations should override install().
def _build_logger(self, log_path):
"""Create a context manager to log build output."""
def do_install(self,
keep_prefix=False, keep_stage=False, ignore_deps=False,
skip_patch=False, verbose=False, make_jobs=None, fake=False):
"""Called by commands to install a package and its dependencies.
Package implementations should override install() to describe
their build process.
Args:
keep_prefix -- Keep install prefix on failure. By default, destroys it.
keep_stage -- Keep stage on successful build. By default, destroys it.
ignore_deps -- Do not install dependencies before installing this package.
fake -- Don't really build -- install fake stub files instead.
skip_patch -- Skip patch stage of build if True.
verbose -- Display verbose build output (by default, suppresses it)
make_jobs -- Number of make jobs to use for install. Default is ncpus.
"""
# whether to keep the prefix on failure. Default is to destroy it.
keep_prefix = kwargs.get('keep_prefix', False)
keep_stage = kwargs.get('keep_stage', False)
ignore_deps = kwargs.get('ignore_deps', False)
fake_install = kwargs.get('fake', False)
skip_patch = kwargs.get('skip_patch', False)
# Override builtin number of make jobs.
self.make_jobs = kwargs.get('make_jobs', None)
if not self.spec.concrete:
raise ValueError("Can only install concrete packages.")
@ -735,10 +744,13 @@ def do_install(self, **kwargs):
tty.msg("Installing %s" % self.name)
if not ignore_deps:
self.do_install_dependencies(**kwargs)
self.do_install_dependencies(
keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps,
fake=fake, skip_patch=skip_patch, verbose=verbose,
make_jobs=make_jobs)
start_time = time.time()
if not fake_install:
if not fake:
if not skip_patch:
self.do_patch()
else:
@ -768,16 +780,26 @@ def real_work():
spack.hooks.pre_install(self)
# Set up process's build environment before running install.
if fake_install:
if fake:
self.do_fake_install()
else:
# Subclasses implement install() to do the real work.
# Do the real install in the source directory.
self.stage.chdir_to_source()
self.install(self.spec, self.prefix)
# This redirects I/O to a build log (and optionally to the terminal)
log_path = join_path(os.getcwd(), 'spack-build.out')
log_file = open(log_path, 'w')
with log_output(log_file, verbose, sys.stdout.isatty(), True):
self.install(self.spec, self.prefix)
# Ensure that something was actually installed.
self._sanity_check_install()
# Move build log into install directory on success
if not fake:
log_install_path = spack.install_layout.build_log_path(self.spec)
install(log_path, log_install_path)
# On successful install, remove the stage.
if not keep_stage:
self.stage.destroy()
@ -792,6 +814,9 @@ def real_work():
print_pkg(self.prefix)
except ProcessError, e:
# Annotate with location of build log.
e.build_log = log_path
# One of the processes returned an error code.
# Suppress detailed stack trace here unless in debug mode
if spack.debug:
@ -801,13 +826,14 @@ def real_work():
# Still need to clean up b/c there was an error.
cleanup()
os._exit(1)
except:
# other exceptions just clean up and raise.
cleanup()
raise
build_env.fork(self, real_work)
spack.build_environment.fork(self, real_work)
# Once everything else is done, run post install hooks
spack.hooks.post_install(self)
@ -867,9 +893,7 @@ def install(self, spec, prefix):
raise InstallError("Package %s provides no install method!" % self.name)
def do_uninstall(self, **kwargs):
force = kwargs.get('force', False)
def do_uninstall(self, force=False):
if not self.installed:
raise InstallError(str(self.spec) + " is not installed.")
@ -912,14 +936,13 @@ def _sanity_check_extension(self):
raise ActivationError("%s does not extend %s!" % (self.name, self.extendee.name))
def do_activate(self, **kwargs):
def do_activate(self, force=False):
"""Called on an etension to invoke the extendee's activate method.
Commands should call this routine, and should not call
activate() directly.
"""
self._sanity_check_extension()
force = kwargs.get('force', False)
spack.install_layout.check_extension_conflict(
self.extendee_spec, self.spec)
@ -929,7 +952,7 @@ def do_activate(self, **kwargs):
for spec in self.spec.traverse(root=False):
if spec.package.extends(self.extendee_spec):
if not spec.package.activated:
spec.package.do_activate(**kwargs)
spec.package.do_activate(force=force)
self.extendee_spec.package.activate(self, **self.extendee_args)
@ -1083,12 +1106,13 @@ def find_versions_of_archive(*archive_urls, **kwargs):
if list_url:
list_urls.add(list_url)
for aurl in archive_urls:
list_urls.add(url.find_list_url(aurl))
list_urls.add(spack.url.find_list_url(aurl))
# Grab some web pages to scrape.
page_map = {}
for lurl in list_urls:
page_map.update(get_pages(lurl, depth=list_depth))
pages = spack.util.web.get_pages(lurl, depth=list_depth)
page_map.update(pages)
# Scrape them for archive URLs
regexes = []
@ -1097,7 +1121,7 @@ def find_versions_of_archive(*archive_urls, **kwargs):
# the version part of the URL. The capture group is converted
# to a generic wildcard, so we can use this to extract things
# on a page that look like archive URLs.
url_regex = url.wildcard_version(aurl)
url_regex = spack.url.wildcard_version(aurl)
# We'll be a bit more liberal and just look for the archive
# part, not the full path.

3
lib/spack/spack/test/__init__.py
View File

@ -54,7 +54,8 @@
'cc',
'link_tree',
'spec_yaml',
'optional_deps']
'optional_deps',
'make_executable']
def list_tests():

125
lib/spack/spack/test/make_executable.py Normal file
View File

@ -0,0 +1,125 @@
##############################################################################
# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""
Tests for Spack's built-in parallel make support.
This just tests whether the right args are getting passed to make.
"""
import os
import unittest
import tempfile
import shutil
from llnl.util.filesystem import *
from spack.util.environment import path_put_first
from spack.build_environment import MakeExecutable
class MakeExecutableTest(unittest.TestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
make_exe = join_path(self.tmpdir, 'make')
with open(make_exe, 'w') as f:
f.write('#!/bin/sh\n')
f.write('echo "$@"')
os.chmod(make_exe, 0700)
path_put_first('PATH', [self.tmpdir])
def tearDown(self):
shutil.rmtree(self.tmpdir)
def test_make_normal(self):
make = MakeExecutable('make', 8)
self.assertEqual(make(return_output=True).strip(), '-j8')
self.assertEqual(make('install', return_output=True).strip(), '-j8 install')
def test_make_explicit(self):
make = MakeExecutable('make', 8)
self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8')
self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install')
def test_make_one_job(self):
make = MakeExecutable('make', 1)
self.assertEqual(make(return_output=True).strip(), '')
self.assertEqual(make('install', return_output=True).strip(), 'install')
def test_make_parallel_false(self):
make = MakeExecutable('make', 8)
self.assertEqual(make(parallel=False, return_output=True).strip(), '')
self.assertEqual(make('install', parallel=False, return_output=True).strip(), 'install')
def test_make_parallel_disabled(self):
make = MakeExecutable('make', 8)
os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true'
self.assertEqual(make(return_output=True).strip(), '')
self.assertEqual(make('install', return_output=True).strip(), 'install')
os.environ['SPACK_NO_PARALLEL_MAKE'] = '1'
self.assertEqual(make(return_output=True).strip(), '')
self.assertEqual(make('install', return_output=True).strip(), 'install')
# These don't disable (false and random string)
os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false'
self.assertEqual(make(return_output=True).strip(), '-j8')
self.assertEqual(make('install', return_output=True).strip(), '-j8 install')
os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar'
self.assertEqual(make(return_output=True).strip(), '-j8')
self.assertEqual(make('install', return_output=True).strip(), '-j8 install')
del os.environ['SPACK_NO_PARALLEL_MAKE']
def test_make_parallel_precedence(self):
make = MakeExecutable('make', 8)
# These should work
os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true'
self.assertEqual(make(parallel=True, return_output=True).strip(), '')
self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install')
os.environ['SPACK_NO_PARALLEL_MAKE'] = '1'
self.assertEqual(make(parallel=True, return_output=True).strip(), '')
self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install')
# These don't disable (false and random string)
os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false'
self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8')
self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install')
os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar'
self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8')
self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install')
del os.environ['SPACK_NO_PARALLEL_MAKE']

3
lib/spack/spack/util/environment.py
View File

@ -35,7 +35,8 @@ def get_path(name):
def env_flag(name):
if name in os.environ:
return os.environ[name].lower() == "true"
value = os.environ[name].lower()
return value == "true" or value == "1"
return False

89
lib/spack/spack/util/executable.py
View File

@ -143,27 +143,72 @@ def which(name, **kwargs):
class ProcessError(spack.error.SpackError):
def __init__(self, msg, long_msg=None):
# Friendlier exception trace info for failed executables
long_msg = long_msg + "\n" if long_msg else ""
for f in inspect.stack():
frame = f[0]
loc = frame.f_locals
if 'self' in loc:
obj = loc['self']
if isinstance(obj, spack.Package):
long_msg += "---\n"
long_msg += "Context:\n"
long_msg += " %s:%d, in %s:\n" % (
inspect.getfile(frame.f_code),
frame.f_lineno,
frame.f_code.co_name)
def __init__(self, msg, long_message=None):
# These are used for detailed debugging information for
# package builds. They're built up gradually as the exception
# propagates.
self.package_context = _get_package_context()
self.build_log = None
lines, start = inspect.getsourcelines(frame)
for i, line in enumerate(lines):
mark = ">> " if start + i == frame.f_lineno else " "
long_msg += " %s%-5d%s\n" % (
mark, start + i, line.rstrip())
break
super(ProcessError, self).__init__(msg, long_message)
super(ProcessError, self).__init__(msg, long_msg)
@property
def long_message(self):
msg = self._long_message
if msg: msg += "\n\n"
if self.build_log:
msg += "See build log for details:\n"
msg += " %s" % self.build_log
if self.package_context:
if msg: msg += "\n\n"
msg += '\n'.join(self.package_context)
return msg
def _get_package_context():
"""Return some context for an error message when the build fails.
This should be called within a ProcessError when the exception is
thrown.
Args:
process_error -- A ProcessError raised during install()
This function inspects the stack to find where we failed in the
package file, and it adds detailed context to the long_message
from there.
"""
lines = []
# Walk up the stack
for f in inspect.stack():
frame = f[0]
# Find a frame with 'self' in the local variables.
if not 'self' in frame.f_locals:
continue
# Look only at a frame in a subclass of spack.Package
obj = frame.f_locals['self']
if type(obj) != spack.Package and isinstance(obj, spack.Package):
break
else:
# Didn't find anything
return lines
# Build a message showing where in install we failed.
lines.append("%s:%d, in %s:" % (
inspect.getfile(frame.f_code),
frame.f_lineno,
frame.f_code.co_name))
sourcelines, start = inspect.getsourcelines(frame)
for i, line in enumerate(sourcelines):
mark = ">> " if start + i == frame.f_lineno else " "
lines.append(" %s%-5d%s" % (mark, start + i, line.rstrip()))
return lines

3
var/spack/packages/py-cython/package.py
View File

@ -3,9 +3,10 @@
class PyCython(Package):
"""The Cython compiler for writing C extensions for the Python language."""
homepage = "https://pypi.python.org/pypi/cython"
url = "https://pypi.python.org/packages/source/C/Cython/Cython-0.21.2.tar.gz"
url = "https://pypi.python.org/packages/source/C/Cython/cython-0.22.tar.gz"
version('0.21.2', 'd21adb870c75680dc857cd05d41046a4')
version('0.22', '1ae25add4ef7b63ee9b4af697300d6b6')
extends('python')

1
var/spack/packages/py-dateutil/package.py
View File

@ -6,6 +6,7 @@ class PyDateutil(Package):
url = "https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.4.0.tar.gz"
version('2.4.0', '75714163bb96bedd07685cdb2071b8bc')
version('2.4.2', '4ef68e1c485b09e9f034e10473e5add2')
extends('python')
depends_on('py-setuptools')

15
var/spack/packages/py-genders/package.py Normal file
View File

@ -0,0 +1,15 @@
from spack import *
class PyGenders(Package):
"""Genders is a static cluster configuration database used for cluster configuration management. It is used by a variety of tools and scripts for management of large clusters."""
homepage = "https://github.com/chaos/genders"
url = "https://github.com/chaos/genders/releases/download/genders-1-22-1/genders-1.22.tar.gz"
version('1.22', '9ea59a024dcbddb85b0ed25ddca9bc8e', url='https://github.com/chaos/genders/releases/download/genders-1-22-1/genders-1.22.tar.gz')
extends('python')
def install(self, spec, prefix):
configure("--prefix=%s" %prefix)
make(parallel=False)
make("install")

3
var/spack/packages/py-h5py/package.py
View File

@ -7,8 +7,9 @@ class PyH5py(Package):
url = "https://pypi.python.org/packages/source/h/h5py/h5py-2.4.0.tar.gz"
version('2.4.0', '80c9a94ae31f84885cc2ebe1323d6758')
version('2.5.0', '6e4301b5ad5da0d51b0a1e5ac19e3b74')
extends('python', ignore=lambda f: re.match(r'cy*', f))
extends('python', ignore=lambda f: re.match(r'bin/cy*', f))
depends_on('hdf5')
depends_on('py-numpy')
depends_on('py-cython')

1
var/spack/packages/py-ipython/package.py
View File

@ -6,6 +6,7 @@ class PyIpython(Package):
url = "https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz"
version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf')
version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f')
extends('python')
depends_on('py-pygments')

3
var/spack/packages/py-matplotlib/package.py
View File

@ -7,6 +7,7 @@ class PyMatplotlib(Package):
url = "https://pypi.python.org/packages/source/m/matplotlib/matplotlib-1.4.2.tar.gz"
version('1.4.2', '7d22efb6cce475025733c50487bd8898')
version('1.4.3', '86af2e3e3c61849ac7576a6f5ca44267')
extends('python', ignore=r'bin/nosetests.*$')
@ -28,7 +29,7 @@ class PyMatplotlib(Package):
def install(self, spec, prefix):
python('setup.py', 'install', '--prefix=%s' % prefix)
if str(self.version) == '1.4.2':
if str(self.version) in ['1.4.2', '1.4.3']:
# hack to fix configuration file
config_file = None
for p,d,f in os.walk(prefix.lib):

1
var/spack/packages/py-nose/package.py
View File

@ -8,6 +8,7 @@ class PyNose(Package):
url = "https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz"
version('1.3.4', '6ed7169887580ddc9a8e16048d38274d')
version('1.3.6', '0ca546d81ca8309080fc80cb389e7a16')
extends('python', ignore=r'bin/nosetests.*$')
depends_on('py-setuptools')

1
var/spack/packages/py-numpy/package.py
View File

@ -6,6 +6,7 @@ class PyNumpy(Package):
url = "https://pypi.python.org/packages/source/n/numpy/numpy-1.9.1.tar.gz"
version('1.9.1', '78842b73560ec378142665e712ae4ad9')
version('1.9.2', 'a1ed53432dbcd256398898d35bc8e645')
extends('python')
depends_on('py-nose')

1
var/spack/packages/py-pandas/package.py
View File

@ -7,6 +7,7 @@ class PyPandas(Package):
url = "https://pypi.python.org/packages/source/p/pandas/pandas-0.16.0.tar.gz#md5=bfe311f05dc0c351f8955fbd1e296e73"
version('0.16.0', 'bfe311f05dc0c351f8955fbd1e296e73')
version('0.16.1', 'fac4f25748f9610a3e00e765474bdea8')
extends('python')
depends_on('py-dateutil')

1
var/spack/packages/py-pygments/package.py
View File

@ -6,6 +6,7 @@ class PyPygments(Package):
url = "https://pypi.python.org/packages/source/P/Pygments/Pygments-2.0.1.tar.gz"
version('2.0.1', 'e0daf4c14a4fe5b630da765904de4d6c')
version('2.0.2', '238587a1370d62405edabd0794b3ec4a')
extends('python')
depends_on('py-setuptools')

1
var/spack/packages/py-pylint/package.py
View File

@ -7,6 +7,7 @@ class PyPylint(Package):
url = "https://pypi.python.org/packages/source/p/pylint/pylint-1.4.1.tar.gz"
version('1.4.1', 'df7c679bdcce5019389038847e4de622')
version('1.4.3', '5924c1c7ca5ca23647812f5971d0ea44')
extends('python')
depends_on('py-nose')

14
var/spack/packages/py-pypar/package.py Normal file
View File

@ -0,0 +1,14 @@
from spack import *
class PyPypar(Package):
"""Pypar is an efficient but easy-to-use module that allows programs written in Python to run in parallel on multiple processors and communicate using MPI."""
homepage = "http://code.google.com/p/pypar/"
url = "https://pypar.googlecode.com/files/pypar-2.1.5_108.tgz"
version('2.1.5_108', '7a1f28327d2a3b679f9455c843d850b8', url='https://pypar.googlecode.com/files/pypar-2.1.5_108.tgz')
extends('python')
depends_on('mpi')
def install(self, spec, prefix):
with working_dir('source'):
python('setup.py', 'install', '--prefix=%s' % prefix)

1
var/spack/packages/py-pytz/package.py
View File

@ -6,6 +6,7 @@ class PyPytz(Package):
url = "https://pypi.python.org/packages/source/p/pytz/pytz-2014.10.tar.gz"
version('2014.10', 'eb1cb941a20c5b751352c52486aa1dd7')
version('2015.4', '417a47b1c432d90333e42084a605d3d8')
extends('python')

1
var/spack/packages/py-rpy2/package.py
View File

@ -6,6 +6,7 @@ class PyRpy2(Package):
url = "https://pypi.python.org/packages/source/r/rpy2/rpy2-2.5.4.tar.gz"
version('2.5.4', '115a20ac30883f096da2bdfcab55196d')
version('2.5.6', 'a36e758b633ce6aec6a5f450bfee980f')
extends('python')
depends_on('py-setuptools')

3
var/spack/packages/py-scientificpython/package.py
View File

@ -7,8 +7,7 @@ class PyScientificpython(Package):
visualization, and parallelization."""
homepage = "https://sourcesup.renater.fr/projects/scientific-py/"
url = "https://sourcesup.renater.fr/frs/download.php/4411/ScientificPython-2.8.1.tar.gz"
url = "https://sourcesup.renater.fr/frs/download.php/file/4411/ScientificPython-2.8.1.tar.gz"
version('2.8.1', '73ee0df19c7b58cdf2954261f0763c77')
extends('python')

1
var/spack/packages/py-scikit-learn/package.py
View File

@ -6,6 +6,7 @@ class PyScikitLearn(Package):
url = "https://pypi.python.org/packages/source/s/scikit-learn/scikit-learn-0.15.2.tar.gz"
version('0.15.2', 'd9822ad0238e17b382a3c756ea94fe0d')
version('0.16.1', '363ddda501e3b6b61726aa40b8dbdb7e')
extends('python')

1
var/spack/packages/py-scipy/package.py
View File

@ -6,6 +6,7 @@ class PyScipy(Package):
url = "https://pypi.python.org/packages/source/s/scipy/scipy-0.15.0.tar.gz"
version('0.15.0', '639112f077f0aeb6d80718dc5019dc7a')
version('0.15.1', 'be56cd8e60591d6332aac792a5880110')
extends('python')
depends_on('py-nose')

1
var/spack/packages/py-setuptools/package.py
View File

@ -6,6 +6,7 @@ class PySetuptools(Package):
url = "https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz"
version('11.3.1', '01f69212e019a2420c1693fb43593930')
version('16.0', '0ace0b96233516fc5f7c857d086aa3ad')
extends('python')

1
var/spack/packages/py-sip/package.py
View File

@ -7,6 +7,7 @@ class PySip(Package):
url = "http://sourceforge.net/projects/pyqt/files/sip/sip-4.16.5/sip-4.16.5.tar.gz"
version('4.16.5', '6d01ea966a53e4c7ae5c5e48c40e49e5')
version('4.16.7', '32abc003980599d33ffd789734de4c36')
extends('python')

1
var/spack/packages/py-virtualenv/package.py
View File

@ -7,6 +7,7 @@ class PyVirtualenv(Package):
url = "https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.11.6.tar.gz"
version('1.11.6', 'f61cdd983d2c4e6aeabb70b1060d6f49')
version('13.0.1', '1ffc011bde6667f0e37ecd976f4934db')
extends('python')
depends_on('py-setuptools')