From bf4231dd4b906f20d522b9df5f2cae2ba6291174 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 2 Jan 2016 22:48:14 +0100 Subject: [PATCH 01/82] depends_on and when multimethod support boolean --- lib/spack/spack/directives.py | 6 +++- lib/spack/spack/multimethod.py | 9 ++++-- lib/spack/spack/util/architecture.py | 46 +++++++++++++++++++++++++++ var/spack/packages/openssl/package.py | 15 +++++++++ 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 lib/spack/spack/util/architecture.py diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index aa9fbd8d339..02477bb904a 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -174,7 +174,11 @@ def version(pkg, ver, checksum=None, **kwargs): def _depends_on(pkg, spec, when=None): - if when is None: + # If when is False do nothing + if when is False: + return + # If when is None or True make sure the condition is always satisfied + if when is None or when is True: when = pkg.name when_spec = parse_anonymous_spec(when, pkg.name) diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index df9b9b2ab14..51c6a8e89d9 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -193,10 +193,11 @@ def install(self, prefix): platform-specific versions. There's not much we can do to get around this because of the way decorators work. """ -class when(object): def __init__(self, spec): pkg = get_calling_module_name() - self.spec = parse_anonymous_spec(spec, pkg) + if spec is True: + spec = pkg + self.spec = parse_anonymous_spec(spec, pkg) if spec is not False else None def __call__(self, method): # Get the first definition of the method in the calling scope @@ -207,7 +208,9 @@ def __call__(self, method): if not type(original_method) == SpecMultiMethod: original_method = SpecMultiMethod(original_method) - original_method.register(self.spec, method) + if self.spec is not None: + original_method.register(self.spec, method) + return original_method diff --git a/lib/spack/spack/util/architecture.py b/lib/spack/spack/util/architecture.py new file mode 100644 index 00000000000..a020c74a7ac --- /dev/null +++ b/lib/spack/spack/util/architecture.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import sys + + +def os_is_in(*args): + """ + Return True if any element in the list is equal to sys.platform, False otherwise + + :param args: list of names to be checked + :return: True or False + """ + return any(map(lambda x: x == sys.platform, args)) + + +def os_is_not_in(*args): + """ + Return True if none of the elements in the list is equal to sys.platform, False otherwise + + :param args: list of names to be checked + :return: True or False + """ + return not os_is_in(*args) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index 40648fca492..1fa288a36dc 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -1,5 +1,7 @@ from spack import * +import spack.util.architecture as arch + class Openssl(Package): """The OpenSSL Project is a collaborative effort to develop a robust, commercial-grade, full-featured, and Open Source @@ -26,3 +28,16 @@ def install(self, spec, prefix): make() make("install") + + @when(arch.os_is_in('darwin')) + def install(self, spec, prefix): + perl = which('perl') + perl("./Configure", + "--prefix=%s" % prefix, + "--openssldir=%s/etc/openssl" % prefix, + "zlib", + "no-krb5", + "shared", + "darwin64-x86_64-cc") + filter_file(r'-arch x86_64', '', 'Makefile') + From 180c673c782d10864dce6cd7df43465dd05579a1 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 12 Jan 2016 13:21:53 -0600 Subject: [PATCH 02/82] Add different URL for older source code --- var/spack/packages/mpc/package.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/mpc/package.py b/var/spack/packages/mpc/package.py index 50477a0ccba..65c0ba7ea13 100644 --- a/var/spack/packages/mpc/package.py +++ b/var/spack/packages/mpc/package.py @@ -34,8 +34,14 @@ class Mpc(Package): version('1.0.3', 'd6a1d5f8ddea3abd2cc3e98f58352d26') version('1.0.2', '68fadff3358fb3e7976c7a398a0af4c3') - depends_on("gmp") - depends_on("mpfr") + depends_on("gmp@4.3.2:") + depends_on("mpfr@2.4.2:") + + def url_for_version(self, version): + if version < Version("1.0.1"): + return "http://www.multiprecision.org/mpc/download/mpc-%s.tar.gz" % version + else: + return "ftp://ftp.gnu.org/gnu/mpc/mpc-%s.tar.gz" % version def install(self, spec, prefix): configure("--prefix=%s" % prefix) From ea9d0c0263126c370119e5fc435652c7214e658a Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 09:58:47 -0600 Subject: [PATCH 03/82] Remove constraints on dependency versions --- var/spack/repos/builtin/packages/mpc/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/mpc/package.py b/var/spack/repos/builtin/packages/mpc/package.py index 65c0ba7ea13..108fec678f1 100644 --- a/var/spack/repos/builtin/packages/mpc/package.py +++ b/var/spack/repos/builtin/packages/mpc/package.py @@ -34,8 +34,8 @@ class Mpc(Package): version('1.0.3', 'd6a1d5f8ddea3abd2cc3e98f58352d26') version('1.0.2', '68fadff3358fb3e7976c7a398a0af4c3') - depends_on("gmp@4.3.2:") - depends_on("mpfr@2.4.2:") + depends_on("gmp") + depends_on("mpfr") def url_for_version(self, version): if version < Version("1.0.1"): From 603f5e69ee43e1249e413c966cde3e02ba10a1a3 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 28 Jan 2016 11:30:28 +0100 Subject: [PATCH 04/82] removed outdated code : os detection (will be handled by platforms) and openssl modifications --- lib/spack/spack/architecture.py | 89 ------------------- .../repos/builtin/packages/openssl/package.py | 15 ---- 2 files changed, 104 deletions(-) delete mode 100644 lib/spack/spack/architecture.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py deleted file mode 100644 index 2701fab90c9..00000000000 --- a/lib/spack/spack/architecture.py +++ /dev/null @@ -1,89 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import os -import re -import platform - -from llnl.util.lang import memoized - -import spack -import spack.error as serr - - -class InvalidSysTypeError(serr.SpackError): - def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) - - -class NoSysTypeError(serr.SpackError): - def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") - - -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" - if not hasattr(spack, "sys_type"): - return None - elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() - else: - return spack.sys_type - - -def get_sys_type_from_environment(): - """Return $SYS_TYPE or None if it's not defined.""" - return os.environ.get('SYS_TYPE') - - -def get_sys_type_from_platform(): - """Return the architecture from Python's platform module.""" - sys_type = platform.system() + '-' + platform.machine() - sys_type = re.sub(r'[^\w-]', '_', sys_type) - return sys_type.lower() - - -@memoized -def sys_type(): - """Returns a SysType for the current machine.""" - methods = [get_sys_type_from_spack_globals, - get_sys_type_from_environment, - get_sys_type_from_platform] - - # search for a method that doesn't return None - sys_type = None - for method in methods: - sys_type = method() - if sys_type: break - - # Couldn't determine the sys_type for this machine. - if sys_type is None: - return "unknown_arch" - - if not isinstance(sys_type, basestring): - raise InvalidSysTypeError(sys_type) - - return sys_type diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 8f0427796be..bbb169ec6bc 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,7 +1,5 @@ from spack import * -import spack.util.architecture as arch - class Openssl(Package): """The OpenSSL Project is a collaborative effort to develop a robust, commercial-grade, full-featured, and Open Source @@ -40,16 +38,3 @@ def install(self, spec, prefix): make() make("install") - - @when(arch.os_is_in('darwin')) - def install(self, spec, prefix): - perl = which('perl') - perl("./Configure", - "--prefix=%s" % prefix, - "--openssldir=%s/etc/openssl" % prefix, - "zlib", - "no-krb5", - "shared", - "darwin64-x86_64-cc") - filter_file(r'-arch x86_64', '', 'Makefile') - From 0cf03518f3edf8b70331b1b5f2eae171d0a0dccf Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 28 Jan 2016 12:06:30 +0100 Subject: [PATCH 05/82] reverted deletion of the wrong `architecture.py` --- lib/spack/spack/architecture.py | 89 ++++++++++++++++++++++++++++ lib/spack/spack/util/architecture.py | 46 -------------- 2 files changed, 89 insertions(+), 46 deletions(-) create mode 100644 lib/spack/spack/architecture.py delete mode 100644 lib/spack/spack/util/architecture.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py new file mode 100644 index 00000000000..2701fab90c9 --- /dev/null +++ b/lib/spack/spack/architecture.py @@ -0,0 +1,89 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import re +import platform + +from llnl.util.lang import memoized + +import spack +import spack.error as serr + + +class InvalidSysTypeError(serr.SpackError): + def __init__(self, sys_type): + super(InvalidSysTypeError, self).__init__( + "Invalid sys_type value for Spack: " + sys_type) + + +class NoSysTypeError(serr.SpackError): + def __init__(self): + super(NoSysTypeError, self).__init__( + "Could not determine sys_type for this machine.") + + +def get_sys_type_from_spack_globals(): + """Return the SYS_TYPE from spack globals, or None if it isn't set.""" + if not hasattr(spack, "sys_type"): + return None + elif hasattr(spack.sys_type, "__call__"): + return spack.sys_type() + else: + return spack.sys_type + + +def get_sys_type_from_environment(): + """Return $SYS_TYPE or None if it's not defined.""" + return os.environ.get('SYS_TYPE') + + +def get_sys_type_from_platform(): + """Return the architecture from Python's platform module.""" + sys_type = platform.system() + '-' + platform.machine() + sys_type = re.sub(r'[^\w-]', '_', sys_type) + return sys_type.lower() + + +@memoized +def sys_type(): + """Returns a SysType for the current machine.""" + methods = [get_sys_type_from_spack_globals, + get_sys_type_from_environment, + get_sys_type_from_platform] + + # search for a method that doesn't return None + sys_type = None + for method in methods: + sys_type = method() + if sys_type: break + + # Couldn't determine the sys_type for this machine. + if sys_type is None: + return "unknown_arch" + + if not isinstance(sys_type, basestring): + raise InvalidSysTypeError(sys_type) + + return sys_type diff --git a/lib/spack/spack/util/architecture.py b/lib/spack/spack/util/architecture.py deleted file mode 100644 index a020c74a7ac..00000000000 --- a/lib/spack/spack/util/architecture.py +++ /dev/null @@ -1,46 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## - -import sys - - -def os_is_in(*args): - """ - Return True if any element in the list is equal to sys.platform, False otherwise - - :param args: list of names to be checked - :return: True or False - """ - return any(map(lambda x: x == sys.platform, args)) - - -def os_is_not_in(*args): - """ - Return True if none of the elements in the list is equal to sys.platform, False otherwise - - :param args: list of names to be checked - :return: True or False - """ - return not os_is_in(*args) From 2cd9ad8ce63ea29093fcc2d8e1bd749d5cbccf0b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 17 Jan 2016 10:46:21 -0500 Subject: [PATCH 06/82] Use "-Wl,-rpath," instead of "-Wl,-rpath=" The former translates to a linker argument "-rpath DIR", whereas the latter translates to "-rpath=DIR". The latter is not support on OS X. --- lib/spack/docs/packaging_guide.rst | 6 +++--- lib/spack/spack/package.py | 4 ++-- lib/spack/spack/test/cc.py | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 59ba63fa355..bb8a26ad02f 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1711,15 +1711,15 @@ Compile-time library search paths * ``-L$dep_prefix/lib`` * ``-L$dep_prefix/lib64`` Runtime library search paths (RPATHs) - * ``-Wl,-rpath=$dep_prefix/lib`` - * ``-Wl,-rpath=$dep_prefix/lib64`` + * ``-Wl,-rpath,$dep_prefix/lib`` + * ``-Wl,-rpath,$dep_prefix/lib64`` Include search paths * ``-I$dep_prefix/include`` An example of this would be the ``libdwarf`` build, which has one dependency: ``libelf``. Every call to ``cc`` in the ``libdwarf`` build will have ``-I$LIBELF_PREFIX/include``, -``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath=$LIBELF_PREFIX/lib`` +``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath,$LIBELF_PREFIX/lib`` inserted on the command line. This is done transparently to the project's build system, which will just think it's using a system where ``libelf`` is readily available. Because of this, you **do diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8cb947c276c..49d4fb6b237 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1220,8 +1220,8 @@ def rpath(self): @property def rpath_args(self): - """Get the rpath args as a string, with -Wl,-rpath= for each element.""" - return " ".join("-Wl,-rpath=%s" % p for p in self.rpath) + """Get the rpath args as a string, with -Wl,-rpath, for each element.""" + return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) def validate_package_url(url_string): diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 905af28a06d..54d56383941 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -39,11 +39,11 @@ 'arg1', '-Wl,--start-group', 'arg2', - '-Wl,-rpath=/first/rpath', 'arg3', '-Wl,-rpath', '-Wl,/second/rpath', + '-Wl,-rpath,/first/rpath', 'arg3', '-Wl,-rpath', '-Wl,/second/rpath', '-llib1', '-llib2', 'arg4', '-Wl,--end-group', - '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath=/fourth/rpath', + '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath,/fourth/rpath', '-llib3', '-llib4', 'arg5', 'arg6'] @@ -95,13 +95,13 @@ def test_cpp_mode(self): def test_ccld_mode(self): self.check_cc('dump-mode', [], "ccld") self.check_cc('dump-mode', ['foo.c', '-o', 'foo'], "ccld") - self.check_cc('dump-mode', ['foo.c', '-o', 'foo', '-Wl,-rpath=foo'], "ccld") - self.check_cc('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath=foo'], "ccld") + self.check_cc('dump-mode', ['foo.c', '-o', 'foo', '-Wl,-rpath,foo'], "ccld") + self.check_cc('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ccld") def test_ld_mode(self): self.check_ld('dump-mode', [], "ld") - self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath=foo'], "ld") + self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ld") def test_includes(self): From 5038a38e296940f463c13d6fdda728d8673bc95a Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 18 Jan 2016 18:13:18 -0500 Subject: [PATCH 07/82] Correct -Xlinker arguments --- lib/spack/spack/test/cc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 54d56383941..11420ec44a1 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -43,7 +43,7 @@ '-llib1', '-llib2', 'arg4', '-Wl,--end-group', - '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath,/fourth/rpath', + '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath', '-Xlinker,/fourth/rpath', '-llib3', '-llib4', 'arg5', 'arg6'] From dc6a33b716dd6712da2e65a78a6a3ed98ca72d4d Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 12 Feb 2016 12:09:29 -0500 Subject: [PATCH 08/82] Handle multiple -Wl,-rpath,... paths --- lib/spack/env/cc | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index aacba996b39..c156b7b6073 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -176,14 +176,21 @@ while [ -n "$1" ]; do -Wl,*) arg="${1#-Wl,}" if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ "$arg" = -rpath=* ]]; then - rpaths+=("${arg#-rpath=}") - elif [[ "$arg" = -rpath ]]; then + if [[ $arg = -rpath=* ]]; then + arg="${arg#-rpath=}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done + elif [[ $arg = -rpath ]]; then shift; arg="$1" - if [[ "$arg" != -Wl,* ]]; then + if [[ $arg != -Wl,* ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi - rpaths+=("${arg#-Wl,}") + # TODO: Handle multiple -Wl, continuations of -Wl,-rpath + arg="${arg#-Wl,}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done else other_args+=("-Wl,$arg") fi @@ -191,11 +198,11 @@ while [ -n "$1" ]; do -Xlinker,*) arg="${1#-Xlinker,}" if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ "$arg" = -rpath=* ]]; then + if [[ $arg = -rpath=* ]]; then rpaths+=("${arg#-rpath=}") - elif [[ "$arg" = -rpath ]]; then + elif [[ $arg = -rpath ]]; then shift; arg="$1" - if [[ "$arg" != -Xlinker,* ]]; then + if [[ $arg != -Xlinker,* ]]; then die "-Xlinker,-rpath was not followed by -Xlinker,*" fi rpaths+=("${arg#-Xlinker,}") From 9a2c1090a6f40db762ac155d7d25063c2965c841 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 12 Feb 2016 12:37:03 -0500 Subject: [PATCH 09/82] Handle -Wl,-rpath,... syntax --- lib/spack/env/cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index c156b7b6073..41933f5e1fe 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -181,6 +181,11 @@ while [ -n "$1" ]; do for rpath in ${arg//,/ }; do rpaths+=("$rpath") done + elif [[ $arg = -rpath,* ]]; then + arg="${arg#-rpath,}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done elif [[ $arg = -rpath ]]; then shift; arg="$1" if [[ $arg != -Wl,* ]]; then From 69064395eb61db3b03d1ed14f16bef7ec2c94ee3 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 13 Feb 2016 15:12:09 -0500 Subject: [PATCH 10/82] Add debug output --- lib/spack/env/cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 41933f5e1fe..a431cffacf4 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -174,19 +174,26 @@ while [ -n "$1" ]; do libs+=("$arg") ;; -Wl,*) + echo "FOUND arg=[$arg]" >&2 arg="${1#-Wl,}" if [ -z "$arg" ]; then shift; arg="$1"; fi + echo "SHIFTED, arg=[$arg]" >&2 if [[ $arg = -rpath=* ]]; then + echo "CASE 1" >&2 arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do + echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done elif [[ $arg = -rpath,* ]]; then + echo "CASE 2" >&2 arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do + echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done elif [[ $arg = -rpath ]]; then + echo "CASE 3" >&2 shift; arg="$1" if [[ $arg != -Wl,* ]]; then die "-Wl,-rpath was not followed by -Wl,*" @@ -194,9 +201,11 @@ while [ -n "$1" ]; do # TODO: Handle multiple -Wl, continuations of -Wl,-rpath arg="${arg#-Wl,}" for rpath in ${arg//,/ }; do + echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done else + echo "OTHER" >&2 other_args+=("-Wl,$arg") fi ;; From 52647b9a5d8ab4b2fff7387f65347164576be088 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 13 Feb 2016 17:55:14 -0500 Subject: [PATCH 11/82] Using regexes instead of globbing to match path names --- lib/spack/env/cc | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index a431cffacf4..c46986e19cd 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -174,26 +174,19 @@ while [ -n "$1" ]; do libs+=("$arg") ;; -Wl,*) - echo "FOUND arg=[$arg]" >&2 arg="${1#-Wl,}" if [ -z "$arg" ]; then shift; arg="$1"; fi - echo "SHIFTED, arg=[$arg]" >&2 - if [[ $arg = -rpath=* ]]; then - echo "CASE 1" >&2 + if [[ $arg =~ -rpath=.* ]]; then arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do - echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done - elif [[ $arg = -rpath,* ]]; then - echo "CASE 2" >&2 + elif [[ $arg =~ -rpath,.* ]]; then arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do - echo " RPATH=[$rpath]" >&2 - rpaths+=("$rpath") + rpaths+=("$rpath") done elif [[ $arg = -rpath ]]; then - echo "CASE 3" >&2 shift; arg="$1" if [[ $arg != -Wl,* ]]; then die "-Wl,-rpath was not followed by -Wl,*" @@ -201,11 +194,9 @@ while [ -n "$1" ]; do # TODO: Handle multiple -Wl, continuations of -Wl,-rpath arg="${arg#-Wl,}" for rpath in ${arg//,/ }; do - echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done else - echo "OTHER" >&2 other_args+=("-Wl,$arg") fi ;; From 43670cbbd01562f6748f3ee6e2505be52e24bccb Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 13 Feb 2016 22:01:36 -0500 Subject: [PATCH 12/82] More games with quoting --- lib/spack/env/cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index c46986e19cd..37483e00733 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -175,20 +175,19 @@ while [ -n "$1" ]; do ;; -Wl,*) arg="${1#-Wl,}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ $arg =~ -rpath=.* ]]; then + if [[ $arg = "-rpath=*" ]]; then arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg =~ -rpath,.* ]]; then + elif [[ $arg = "-rpath,*" ]]; then arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do - rpaths+=("$rpath") + rpaths+=("$rpath") done - elif [[ $arg = -rpath ]]; then + elif [[ $arg = "-rpath" ]]; then shift; arg="$1" - if [[ $arg != -Wl,* ]]; then + if [[ $arg != "-Wl,*" ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi # TODO: Handle multiple -Wl, continuations of -Wl,-rpath From a06e29fecbaf100f089c1eab7c6447cd469b95f6 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 14 Feb 2016 10:43:55 -0500 Subject: [PATCH 13/82] More quoting experiments --- lib/spack/env/cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 37483e00733..853a19dfddd 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -175,22 +175,22 @@ while [ -n "$1" ]; do ;; -Wl,*) arg="${1#-Wl,}" - if [[ $arg = "-rpath=*" ]]; then + # TODO: Handle multiple -Wl, continuations of -Wl,-rpath + if [[ $arg == '-rpath='* ]]; then arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg = "-rpath,*" ]]; then + elif [[ $arg == '-rpath,'* ]]; then arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg = "-rpath" ]]; then + elif [[ $arg == '-rpath' ]]; then shift; arg="$1" if [[ $arg != "-Wl,*" ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi - # TODO: Handle multiple -Wl, continuations of -Wl,-rpath arg="${arg#-Wl,}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") From 9868333e8e2586ffc55ee996a48ef62601a9c874 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 14 Feb 2016 11:34:08 -0500 Subject: [PATCH 14/82] Shell quoting is difficult --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 853a19dfddd..c3d11357224 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -188,7 +188,7 @@ while [ -n "$1" ]; do done elif [[ $arg == '-rpath' ]]; then shift; arg="$1" - if [[ $arg != "-Wl,*" ]]; then + if [[ $arg != '-Wl,'* ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi arg="${arg#-Wl,}" From 19d10291bfeae45315fcef852baddec63b69247a Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Thu, 18 Feb 2016 15:45:29 -0800 Subject: [PATCH 15/82] modify compiler commands in python config files, fix for #431 --- var/spack/repos/builtin/packages/python/package.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index a1ce06feb03..58d401244e3 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -55,6 +55,20 @@ def install(self, spec, prefix): make() make("install") + # Modify compiler paths in configuration files. This is necessary for + # building site packages outside of spack + filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', + (r'\4\5'), + join_path(prefix.lib, 'python%d.%d' % self.version[:2], '_sysconfigdata.py')) + + python3_version = '' + if spec.satisfies('@3:'): + python3_version = '-%d.%dm' % self.version[:2] + makefile_filepath = join_path(prefix.lib, 'python%d.%d' % self.version[:2], 'config%s' % python3_version, 'Makefile') + filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', + (r'\4\5'), + makefile_filepath) + # ======================================================================== # Set up environment to make install easy for python extensions. From 30c304748213bee3669c75d4384bad146f68dbd7 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 19 Feb 2016 12:39:38 +0100 Subject: [PATCH 16/82] gromacs : added package --- .../repos/builtin/packages/fftw/package.py | 2 + .../repos/builtin/packages/gromacs/package.py | 56 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 var/spack/repos/builtin/packages/gromacs/package.py diff --git a/var/spack/repos/builtin/packages/fftw/package.py b/var/spack/repos/builtin/packages/fftw/package.py index 4d2b9642426..bc129aaf1a1 100644 --- a/var/spack/repos/builtin/packages/fftw/package.py +++ b/var/spack/repos/builtin/packages/fftw/package.py @@ -47,6 +47,8 @@ class Fftw(Package): depends_on('mpi', when='+mpi') + # TODO : add support for architecture specific optimizations as soon as targets are supported + def install(self, spec, prefix): options = ['--prefix=%s' % prefix, '--enable-shared', diff --git a/var/spack/repos/builtin/packages/gromacs/package.py b/var/spack/repos/builtin/packages/gromacs/package.py new file mode 100644 index 00000000000..5fe8399308b --- /dev/null +++ b/var/spack/repos/builtin/packages/gromacs/package.py @@ -0,0 +1,56 @@ +from spack import * + + +class Gromacs(Package): + """ + GROMACS (GROningen MAchine for Chemical Simulations) is a molecular dynamics package primarily designed for + simulations of proteins, lipids and nucleic acids. It was originally developed in the Biophysical Chemistry + department of University of Groningen, and is now maintained by contributors in universities and research centers + across the world. + + GROMACS is one of the fastest and most popular software packages available and can run on CPUs as well as GPUs. + It is free, open source released under the GNU General Public License. Starting from version 4.6, GROMACS is + released under the GNU Lesser General Public License. + """ + + homepage = 'http://www.gromacs.org' + url = 'ftp://ftp.gromacs.org/pub/gromacs/gromacs-5.1.2.tar.gz' + + version('5.1.2', '614d0be372f1a6f1f36382b7a6fcab98') + + variant('mpi', default=True, description='Activate MPI support') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Enables debug mode') + variant('double', default=False, description='Produces a double precision version of the executables') + + depends_on('mpi', when='+mpi') + + depends_on('fftw') + + # TODO : add GPU support + + def install(self, spec, prefix): + + options = [] + + if '+mpi' in spec: + options.append('-DGMX_MPI:BOOL=ON') + + if '+double' in spec: + options.append('-DGMX_DOUBLE:BOOL=ON') + + if '~shared' in spec: + options.append('-DBUILD_SHARED_LIBS:BOOL=OFF') + + if '+debug' in spec: + options.append('-DCMAKE_BUILD_TYPE:STRING=Debug') + else: + options.append('-DCMAKE_BUILD_TYPE:STRING=Release') + + options.extend(std_cmake_args) + + with working_dir('spack-build', create=True): + + cmake('..', *options) + make() + make('install') From 5c8dd6c3c86400ac82b061ce45f58b543526bfff Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Sat, 20 Feb 2016 17:18:49 -0800 Subject: [PATCH 17/82] llvm trunk version Adding a trunk version to the llvm package. This has all the features and requirements of the others, with the additional caveat that the llvm project makes no guarantee that trunk on all repositories together will necessarily make a working compiler. It has been tested, and worked with a version today, but not yesterday, so if you test keep that in mind. --- .../repos/builtin/packages/llvm/package.py | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index 1805d3ded80..a8f19f90711 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -117,6 +117,21 @@ class Llvm(Package): }, } releases = [ + { + 'version' : 'trunk', + 'repo' : 'http://llvm.org/svn/llvm-project/llvm/trunk', + 'resources' : { + 'compiler-rt' : 'http://llvm.org/svn/llvm-project/compiler-rt/trunk', + 'openmp' : 'http://llvm.org/svn/llvm-project/openmp/trunk', + 'polly' : 'http://llvm.org/svn/llvm-project/polly/trunk', + 'libcxx' : 'http://llvm.org/svn/llvm-project/libcxx/trunk', + 'libcxxabi' : 'http://llvm.org/svn/llvm-project/libcxxabi/trunk', + 'clang' : 'http://llvm.org/svn/llvm-project/cfe/trunk', + 'clang-tools-extra' : 'http://llvm.org/svn/llvm-project/clang-tools-extra/trunk', + 'lldb' : 'http://llvm.org/svn/llvm-project/lldb/trunk', + 'llvm-libunwind' : 'http://llvm.org/svn/llvm-project/libunwind/trunk', + } + }, { 'version' : '3.7.0', 'md5':'b98b9495e5655a672d6cb83e1a180f8e', @@ -161,15 +176,25 @@ class Llvm(Package): ] for release in releases: - version(release['version'], release['md5'], url=llvm_url % release) + if release['version'] == 'trunk' : + version(release['version'], svn=release['repo']) - for name, md5 in release['resources'].items(): - resource(name=name, - url=resources[name]['url'] % release, - md5=md5, - destination=resources[name]['destination'], - when='@%(version)s' % release, - placement=resources[name].get('placement', None)) + for name, repo in release['resources'].items(): + resource(name=name, + svn=repo, + destination=resources[name]['destination'], + when='@%(version)s' % release, + placement=resources[name].get('placement', None)) + else: + version(release['version'], release['md5'], url=llvm_url % release) + + for name, md5 in release['resources'].items(): + resource(name=name, + url=resources[name]['url'] % release, + md5=md5, + destination=resources[name]['destination'], + when='@%(version)s' % release, + placement=resources[name].get('placement', None)) # SVN - current develop version('develop', svn='http://llvm.org/svn/llvm-project/llvm/trunk') From 976ae91dccd36d1feb6d0d20cd1ef5b3470c4c39 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 24 Feb 2016 14:11:57 +0100 Subject: [PATCH 18/82] llvm : removed duplicate version --- .../repos/builtin/packages/llvm/package.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index a8f19f90711..934d994bd3e 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -196,25 +196,6 @@ class Llvm(Package): when='@%(version)s' % release, placement=resources[name].get('placement', None)) - # SVN - current develop - version('develop', svn='http://llvm.org/svn/llvm-project/llvm/trunk') - resource(name='clang', svn='http://llvm.org/svn/llvm-project/cfe/trunk', - destination='tools', when='@develop', placement='clang') - resource(name='compiler-rt', svn='http://llvm.org/svn/llvm-project/compiler-rt/trunk', - destination='projects', when='@develop', placement='compiler-rt') - resource(name='openmp', svn='http://llvm.org/svn/llvm-project/openmp/trunk', - destination='projects', when='@develop', placement='openmp') - resource(name='libcxx', svn='http://llvm.org/svn/llvm-project/libcxx/trunk', - destination='projects', when='@develop', placement='libcxx') - resource(name='libcxxabi', svn='http://llvm.org/svn/llvm-project/libcxxabi/trunk', - destination='projects', when='@develop', placement='libcxxabi') - resource(name='polly', svn='http://llvm.org/svn/llvm-project/polly/trunk', - destination='tools', when='@develop', placement='polly') - resource(name='lldb', svn='http://llvm.org/svn/llvm-project/lldb/trunk', - destination='tools', when='@develop', placement='lldb') - - - def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ] From 6f42dd556d1ead8e7cad9788004dc33c11240564 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 24 Feb 2016 17:37:58 +0100 Subject: [PATCH 19/82] stage : on-going refactoring --- lib/spack/spack/package.py | 149 +++++++++++++++++----------------- lib/spack/spack/stage.py | 68 ++++++++++------ lib/spack/spack/test/stage.py | 122 +++++++++++----------------- 3 files changed, 167 insertions(+), 172 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8019b29cba5..4fdc582479d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -847,93 +847,94 @@ def do_install(self, make_jobs=make_jobs) start_time = time.time() - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() - - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) - - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) - - - def real_work(): - try: - tty.msg("Building %s." % self.name) - - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) - - # Set up process's build environment before running install. - if fake: - self.do_fake_install() + with self.stage: + if not fake: + if not skip_patch: + self.do_patch() else: - # Do the real install in the source directory. - self.stage.chdir_to_source() + self.do_stage() - # This redirects I/O to a build log (and optionally to the terminal) - log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): - self.install(self.spec, self.prefix) + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) - # Ensure that something was actually installed. - self._sanity_check_install() + def cleanup(): + if not keep_prefix: + # If anything goes wrong, remove the install prefix + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed." + + "Manually remove this directory to fix:", + self.prefix, wrap=True) - # Move build log into install directory on success - if not fake: - log_install_path = spack.install_layout.build_log_path(self.spec) - install(log_path, log_install_path) - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + def real_work(): + try: + tty.msg("Building %s." % self.name) - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) - tty.msg("Successfully installed %s." % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + # Set up process's build environment before running install. + if fake: + self.do_fake_install() + else: + # Do the real install in the source directory. + self.stage.chdir_to_source() - except ProcessError, e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e + # This redirects I/O to a build log (and optionally to the terminal) + log_path = join_path(os.getcwd(), 'spack-build.out') + log_file = open(log_path, 'w') + with log_output(log_file, verbose, sys.stdout.isatty(), True): + self.install(self.spec, self.prefix) - except: - # other exceptions just clean up and raise. - cleanup() - raise + # Ensure that something was actually installed. + self._sanity_check_install() - # Set parallelism before starting build. - self.make_jobs = make_jobs + # Move build log into install directory on success + if not fake: + log_install_path = spack.install_layout.build_log_path(self.spec) + install(log_path, log_install_path) - # Do the build. - spack.build_environment.fork(self, real_work) + # On successful install, remove the stage. + if not keep_stage: + self.stage.destroy() - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) + tty.msg("Successfully installed %s." % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) + + except ProcessError, e: + # Annotate with location of build log. + e.build_log = log_path + cleanup() + raise e + + except: + # other exceptions just clean up and raise. + cleanup() + raise + + # Set parallelism before starting build. + self.make_jobs = make_jobs + + # Do the build. + spack.build_environment.fork(self, real_work) + + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) + + # Once everything else is done, run post install hooks + spack.hooks.post_install(self) def _sanity_check_install(self): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index f217450d42d..96b1eaf3f2c 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -96,12 +96,44 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. - self.name = kwargs.get('name') + # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name + # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root + self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names()) self.mirror_path = kwargs.get('mirror_path') self.tmp_root = find_tmp_root() - self.path = None + # Try to construct here a temporary name for the stage directory + # If this is a named stage, then construct a named path. + self.path = join_path(spack.stage_path, self.name) + + self.delete_on_exit = True + + def __enter__(self): + """ + Entering a stage context will create the stage directory + """ + # FIXME : if _setup is used only here, then it makes no sense to retain the function self._setup() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Exiting from a stage context will delete the stage directory unless: + - it was explicitly requested not to do so + - an exception has been raised + + Args: + exc_type: exception type + exc_val: exception value + exc_tb: exception traceback + + Returns: + Boolean + """ + self.delete_on_exit = False if exc_type is not None else self.delete_on_exit + + if self.delete_on_exit: + self.destroy() def _cleanup_dead_links(self): """Remove any dead links in the stage directory.""" @@ -163,35 +195,17 @@ def _setup(self): mkdirp(spack.stage_path) self._cleanup_dead_links() - # If this is a named stage, then construct a named path. - if self.name is not None: - self.path = join_path(spack.stage_path, self.name) - # If this is a temporary stage, them make the temp directory - tmp_dir = None if self.tmp_root: - if self.name is None: - # Unnamed tmp root. Link the path in + if self._need_to_create_path(): tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - self.name = os.path.basename(tmp_dir) - self.path = join_path(spack.stage_path, self.name) - if self._need_to_create_path(): - os.symlink(tmp_dir, self.path) - - else: - if self._need_to_create_path(): - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) + os.symlink(tmp_dir, self.path) # if we're not using a tmp dir, create the stage directly in the # stage dir, rather than linking to it. else: - if self.name is None: - self.path = tempfile.mkdtemp('', STAGE_PREFIX, spack.stage_path) - self.name = os.path.basename(self.path) - else: - if self._need_to_create_path(): - mkdirp(self.path) + if self._need_to_create_path(): + mkdirp(self.path) # Make sure we can actually do something with the stage we made. ensure_access(self.path) @@ -389,6 +403,12 @@ def source_path(self): def path(self): return self[0].path + def __enter__(self): + return self[0].__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + return self[0].__exit__(exc_type, exc_val, exc_tb) + def chdir_to_source(self): return self[0].chdir_to_source() diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index c1b2a2a5730..a7314eba4c5 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -192,116 +192,90 @@ def check_destroy(self, stage, stage_name): def test_setup_and_destroy_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_no_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_setup_and_destroy_no_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_chdir(self): - stage = Stage(archive_url, name=stage_name) - - stage.chdir() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.chdir() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) self.check_destroy(stage, stage_name) def test_fetch(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - self.check_expand_archive(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + self.check_expand_archive(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) self.check_destroy(stage, stage_name) def test_restage(self): - stage = Stage(archive_url, name=stage_name) + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) - stage.fetch() - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + # Try to make a file in the old archive dir + with open('foobar', 'w') as file: + file.write("this file is to be destroyed.") - # Try to make a file in the old archive dir - with open('foobar', 'w') as file: - file.write("this file is to be destroyed.") + self.assertTrue('foobar' in os.listdir(stage.source_path)) - self.assertTrue('foobar' in os.listdir(stage.source_path)) - - # Make sure the file is not there after restage. - stage.restage() - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.chdir_to_source() - self.check_chdir_to_source(stage, stage_name) - self.assertFalse('foobar' in os.listdir(stage.source_path)) - - stage.destroy() + # Make sure the file is not there after restage. + stage.restage() + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.chdir_to_source() + self.check_chdir_to_source(stage, stage_name) + self.assertFalse('foobar' in os.listdir(stage.source_path)) self.check_destroy(stage, stage_name) From 20845a739f15190cf9610da4375d3e3c9fc61b6b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 24 Feb 2016 22:18:51 -0500 Subject: [PATCH 20/82] Libevent depends on OpenSSL --- var/spack/repos/builtin/packages/libevent/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/libevent/package.py b/var/spack/repos/builtin/packages/libevent/package.py index 11b1083d672..2a44c49325d 100644 --- a/var/spack/repos/builtin/packages/libevent/package.py +++ b/var/spack/repos/builtin/packages/libevent/package.py @@ -23,6 +23,9 @@ class Libevent(Package): version('2.0.12', '42986228baf95e325778ed328a93e070') + depends_on('openssl') + + def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 30d9ca2bde033da28b6f6105f93ee05f4fd0acfa Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Fri, 26 Feb 2016 16:06:17 -0800 Subject: [PATCH 21/82] Updated the silo package and added the '+fortran' variant. --- .../repos/builtin/packages/silo/package.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py index 9eda11df15f..d1aed78e0e8 100644 --- a/var/spack/repos/builtin/packages/silo/package.py +++ b/var/spack/repos/builtin/packages/silo/package.py @@ -1,19 +1,28 @@ from spack import * class Silo(Package): - """Silo is a library for reading and writing a wide variety of scientific data to binary, disk files.""" + """Silo is a library for reading and writing a wide variety of scientific + data to binary, disk files.""" homepage = "http://wci.llnl.gov/simulation/computer-codes/silo" url = "https://wci.llnl.gov/content/assets/docs/simulation/computer-codes/silo/silo-4.8/silo-4.8.tar.gz" - #version('4.9', 'a83eda4f06761a86726e918fc55e782a') version('4.8', 'b1cbc0e7ec435eb656dc4b53a23663c9') - depends_on("hdf5@:1.8.12") + variant('fortran', default=True, description='Enable Fortran support') + + depends_on("hdf5") def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-hdf5=%s" %spec['hdf5'].prefix) + config_args = [ + '--enable-fortran' if '+fortran' in spec else '--disable-fortran', + ] + + configure( + "--prefix=%s" % prefix, + "--with-hdf5=%s,%s" % (spec['hdf5'].prefix.include, spec['hdf5'].prefix.lib), + "--with-zlib=%s,%s" % (spec['zlib'].prefix.include, spec['zlib'].prefix.lib), + *config_args) make() make("install") From 8f3ac9ac8b7b1d16672e7a52f691966de1f8483f Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Sat, 27 Feb 2016 01:40:32 +0100 Subject: [PATCH 22/82] adding new version to praview, compiled with spack -d install -j 8 --keep-stage paraview@5.0.0+qt+python+tcl+opengl2%gcc@4.8.2 ^netcdf -mpi --- .../repos/builtin/packages/paraview/package.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index e43bdd44930..ccf2d14c06e 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -2,9 +2,11 @@ class Paraview(Package): homepage = 'http://www.paraview.org' - url = 'http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz' + url = 'http://www.paraview.org/files/v5.0/ParaView-v' + _url_str = 'http://www.paraview.org/files/v%s/ParaView-v%s-source.tar.gz' - version('4.4.0', 'fa1569857dd680ebb4d7ff89c2227378', url='http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz') + version('4.4.0', 'fa1569857dd680ebb4d7ff89c2227378') + version('5.0.0', '4598f0b421460c8bbc635c9a1c3bdbee') variant('python', default=False, description='Enable Python support') @@ -25,8 +27,8 @@ class Paraview(Package): depends_on('bzip2') depends_on('freetype') - depends_on('hdf5') depends_on('hdf5+mpi', when='+mpi') + depends_on('hdf5~mpi', when='~mpi') depends_on('jpeg') depends_on('libpng') depends_on('libtiff') @@ -35,6 +37,11 @@ class Paraview(Package): #depends_on('protobuf') # version mismatches? #depends_on('sqlite') # external version not supported depends_on('zlib') + + def url_for_version(self, version): + """Handle ParaView version-based custom URLs.""" + return self._url_str % (version.up_to(2), version) + def install(self, spec, prefix): with working_dir('spack-build', create=True): From 67f327f805868d369eec0392631392281b68c39d Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Sun, 28 Feb 2016 19:47:19 -0800 Subject: [PATCH 23/82] Updated links to use new llnl.gov address --- README.md | 4 ++-- lib/spack/docs/getting_started.rst | 2 +- lib/spack/spack/cmd/repo.py | 2 +- lib/spack/spack/repository.py | 2 +- lib/spack/spack/resource.py | 2 +- lib/spack/spack/test/namespace_trie.py | 2 +- lib/spack/spack/test/tally_plugin.py | 8 ++++---- var/spack/repos/builtin/packages/gdb/package.py | 2 +- var/spack/repos/builtin/packages/texinfo/package.py | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index bdce345764f..8664953c0cc 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ written in pure Python, and specs allow package authors to write a single build script for many different builds of the same package. See the -[Feature Overview](http://llnl.github.io/spack/features.html) +[Feature Overview](http://software.llnl.gov/spack/features.html) for examples and highlights. To install spack and install your first package: @@ -31,7 +31,7 @@ To install spack and install your first package: Documentation ---------------- -[**Full documentation**](http://llnl.github.io/spack) for Spack is +[**Full documentation**](http://software.llnl.gov/spack) for Spack is the first place to look. See also: diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 67ca18e71ae..2c5b68ea650 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -22,7 +22,7 @@ go: $ spack install libelf For a richer experience, use Spack's `shell support -`_: +`_: .. code-block:: sh diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 34c755fb67f..908f5400ab2 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index f58cd52125b..e8d0cc09ec2 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py index 2bf92947fd3..ddfaaf4cb09 100644 --- a/lib/spack/spack/resource.py +++ b/lib/spack/spack/resource.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index d0d809004d5..647976df210 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index 9ca898c47ce..e0b9618e0cb 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://scalability-software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -34,7 +34,7 @@ def __init__(self): self.successCount = 0 self.failCount = 0 self.errorCount = 0 - + @property def numberOfTestsRun(self): """Excludes skipped tests""" @@ -48,10 +48,10 @@ def configure(self, options, conf): def addSuccess(self, test): self.successCount += 1 - + def addError(self, test, err): self.errorCount += 1 - + def addFailure(self, test, err): self.failCount += 1 diff --git a/var/spack/repos/builtin/packages/gdb/package.py b/var/spack/repos/builtin/packages/gdb/package.py index dd02b426b94..b346fe80c28 100644 --- a/var/spack/repos/builtin/packages/gdb/package.py +++ b/var/spack/repos/builtin/packages/gdb/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/repos/builtin/packages/texinfo/package.py b/var/spack/repos/builtin/packages/texinfo/package.py index a83c10c0c19..6cf8d790722 100644 --- a/var/spack/repos/builtin/packages/texinfo/package.py +++ b/var/spack/repos/builtin/packages/texinfo/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify From a339ac0a727458ab55b2e0d27ff17dcc139c0a32 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Mon, 29 Feb 2016 22:29:30 -0500 Subject: [PATCH 24/82] Bug Fix: When Spack create roots around for other versions, it sometimes finds files it thinks are tarballs, but are not. Previously, it would crash if any such files are found. This change allows it to simply skip them and move on, processing the rest of the files it finds correctly. --- lib/spack/spack/cmd/checksum.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index b1ad89dbb8a..c4519932335 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -58,24 +58,29 @@ def get_checksums(versions, urls, **kwargs): tty.msg("Downloading...") hashes = [] - for i, (url, version) in enumerate(zip(urls, versions)): + i = 0 + for url, version in zip(urls, versions): stage = Stage(url) try: stage.fetch() if i == 0 and first_stage_function: first_stage_function(stage) - hashes.append( - spack.util.crypto.checksum(hashlib.md5, stage.archive_file)) + hashes.append((version, + spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) except FailedDownloadError, e: tty.msg("Failed to fetch %s" % url) continue + except Exception, e: + tty.msg('Something failed on %s, skipping.\n (%s)' % (url, e)) + continue finally: if not keep_stage: stage.destroy() + i += 1 - return zip(versions, hashes) + return hashes From e414c5fdfbc91e5934ee59233475f8f862e2e0ce Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Mon, 29 Feb 2016 22:31:18 -0500 Subject: [PATCH 25/82] Added missing cmake dependencies --- var/spack/repos/builtin/packages/cgal/package.py | 1 + var/spack/repos/builtin/packages/curl/package.py | 6 +++--- var/spack/repos/builtin/packages/expat/package.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/cgal/package.py b/var/spack/repos/builtin/packages/cgal/package.py index 97356433be1..ef4a2736dbb 100644 --- a/var/spack/repos/builtin/packages/cgal/package.py +++ b/var/spack/repos/builtin/packages/cgal/package.py @@ -46,6 +46,7 @@ class Cgal(Package): depends_on('mpfr') depends_on('gmp') depends_on('zlib') + depends_on('cmake') # FIXME : Qt5 dependency missing (needs Qt5 and OpenGL) # FIXME : Optional third party libraries missing diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py index 9e684445c71..6c302281a77 100644 --- a/var/spack/repos/builtin/packages/curl/package.py +++ b/var/spack/repos/builtin/packages/curl/package.py @@ -13,13 +13,13 @@ class Curl(Package): version('7.43.0', '11bddbb452a8b766b932f859aaeeed39') version('7.42.1', '296945012ce647b94083ed427c1877a8') - depends_on("openssl") +# depends_on("openssl") depends_on("zlib") def install(self, spec, prefix): configure('--prefix=%s' % prefix, - '--with-zlib=%s' % spec['zlib'].prefix, - '--with-ssl=%s' % spec['openssl'].prefix) + '--with-zlib=%s' % spec['zlib'].prefix) +# '--with-ssl=%s' % spec['openssl'].prefix) make() make("install") diff --git a/var/spack/repos/builtin/packages/expat/package.py b/var/spack/repos/builtin/packages/expat/package.py index 082da5bf0be..3f925c65461 100644 --- a/var/spack/repos/builtin/packages/expat/package.py +++ b/var/spack/repos/builtin/packages/expat/package.py @@ -7,6 +7,7 @@ class Expat(Package): version('2.1.0', 'dd7dab7a5fea97d2a6a43f511449b7cd') + depends_on('cmake') def install(self, spec, prefix): From 15ae92aae9c8ac521bb5d7ee5f210d265494d87a Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Mon, 29 Feb 2016 22:59:28 -0500 Subject: [PATCH 26/82] New packages added: blitz, netcdf-cxx4, netcdf-fortran, proj, udunits2 --- .../repos/builtin/packages/blitz/package.py | 40 ++++++++++++++++++ .../builtin/packages/netcdf-cxx4/package.py | 41 +++++++++++++++++++ .../packages/netcdf-fortran/package.py | 21 ++++++++++ .../repos/builtin/packages/proj/package.py | 37 +++++++++++++++++ .../builtin/packages/udunits2/package.py | 16 ++++++++ 5 files changed, 155 insertions(+) create mode 100644 var/spack/repos/builtin/packages/blitz/package.py create mode 100644 var/spack/repos/builtin/packages/netcdf-cxx4/package.py create mode 100644 var/spack/repos/builtin/packages/netcdf-fortran/package.py create mode 100644 var/spack/repos/builtin/packages/proj/package.py create mode 100644 var/spack/repos/builtin/packages/udunits2/package.py diff --git a/var/spack/repos/builtin/packages/blitz/package.py b/var/spack/repos/builtin/packages/blitz/package.py new file mode 100644 index 00000000000..82ff6349250 --- /dev/null +++ b/var/spack/repos/builtin/packages/blitz/package.py @@ -0,0 +1,40 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install blitz +# +# You can always get back here to change things with: +# +# spack edit blitz +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class Blitz(Package): + """N-dimensional arrays for C++""" + homepage = "http://github.com/blitzpp/blitz" + +# This version doesn't have the configure script generated yet. + url = "https://github.com/blitzpp/blitz/tarball/1.0.0" +#http://prdownloads.sourceforge.net/%(namelower)s + + version('1.0.0', '9f040b9827fe22228a892603671a77af') + + # FIXME: Add dependencies if this package requires them. + # depends_on("foo") + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + # FIXME: Spack couldn't guess one, so here are some options: + configure('--prefix=%s' % prefix) + # cmake('.', *std_cmake_args) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py new file mode 100644 index 00000000000..9d70eab05f5 --- /dev/null +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -0,0 +1,41 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install netcdf-cxx4 +# +# You can always get back here to change things with: +# +# spack edit netcdf-cxx4 +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class NetcdfCxx4(Package): + """C++ interface for NetCDF4""" + homepage = "http://www.unidata.ucar.edu/downloads/netcdf/netcdf-cxx/index.jsp" + url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz" + + version('4.2', 'd019853802092cf686254aaba165fc81') + + + variant('mpi', default=True, description='Enables MPI parallelism') +# variant('hdf4', default=False, description="Enable HDF4 support") + + # NetCDF-CXX4 doesn't really depend (directly) on MPI. However... this + # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) + depends_on('mpi', when='+mpi') + depends_on('netcdf') + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix=%s' % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py new file mode 100644 index 00000000000..8e5c8ecc3a0 --- /dev/null +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -0,0 +1,21 @@ +from spack import * + +class NetcdfFortran(Package): + """Fortran interface for NetCDF4""" + + homepage = "http://www.unidata.ucar.edu/downloads/netcdf/netcdf-cxx/index.jsp" + url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-fortran-4.4.3.tar.gz" + + version('4.4.3', 'bfd4ae23a34635b273d3eb0d91cbde9e') + + variant('mpi', default=True, description='Enables MPI parallelism') + + # NetCDF-CXX4 doesn't really depend (directly) on MPI. However... this + # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) + depends_on('mpi', when='+mpi') + depends_on('netcdf') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/proj/package.py b/var/spack/repos/builtin/packages/proj/package.py new file mode 100644 index 00000000000..4a0d3feac75 --- /dev/null +++ b/var/spack/repos/builtin/packages/proj/package.py @@ -0,0 +1,37 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install proj +# +# You can always get back here to change things with: +# +# spack edit proj +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class Proj(Package): + """Cartographic Projections""" + homepage = "https://github.com/OSGeo/proj.4/wiki" + url = "http://download.osgeo.org/proj/proj-4.9.2.tar.gz" + + version('4.9.2', '9843131676e31bbd903d60ae7dc76cf9') + version('4.9.1', '3cbb2a964fd19a496f5f4265a717d31c') + version('4.8.0', 'd815838c92a29179298c126effbb1537') + version('4.7.0', '927d34623b52e0209ba2bfcca18fe8cd') + version('4.6.1', '7dbaab8431ad50c25669fd3fb28dc493') + + # FIXME: Add dependencies if this package requires them. + # depends_on("foo") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/udunits2/package.py b/var/spack/repos/builtin/packages/udunits2/package.py new file mode 100644 index 00000000000..9954a733bb5 --- /dev/null +++ b/var/spack/repos/builtin/packages/udunits2/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Udunits2(Package): + """Automated units conversion""" + + homepage = "http://www.unidata.ucar.edu/software/udunits" + url = "ftp://ftp.unidata.ucar.edu/pub/udunits/udunits-2.2.20.tar.gz" + + version('2.2.20', '1586b70a49dfe05da5fcc29ef239dce0') + + depends_on('expat') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") From eb1d90a8cfadeb556754b57e5d21e5605b9a0e9d Mon Sep 17 00:00:00 2001 From: citibeth Date: Mon, 29 Feb 2016 23:31:54 -0500 Subject: [PATCH 27/82] Undid accidental change on this branch. --- var/spack/repos/builtin/packages/curl/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py index 6c302281a77..9e684445c71 100644 --- a/var/spack/repos/builtin/packages/curl/package.py +++ b/var/spack/repos/builtin/packages/curl/package.py @@ -13,13 +13,13 @@ class Curl(Package): version('7.43.0', '11bddbb452a8b766b932f859aaeeed39') version('7.42.1', '296945012ce647b94083ed427c1877a8') -# depends_on("openssl") + depends_on("openssl") depends_on("zlib") def install(self, spec, prefix): configure('--prefix=%s' % prefix, - '--with-zlib=%s' % spec['zlib'].prefix) -# '--with-ssl=%s' % spec['openssl'].prefix) + '--with-zlib=%s' % spec['zlib'].prefix, + '--with-ssl=%s' % spec['openssl'].prefix) make() make("install") From a9f0b98d4d30e44032f5b9c86c5b6e27425098f5 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 1 Mar 2016 15:53:53 -0600 Subject: [PATCH 28/82] Typo fix --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index aacba996b39..a323c481243 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -130,7 +130,7 @@ if [ -z "$mode" ]; then done fi -# Dump the version and exist if we're in testing mode. +# Dump the version and exit if we're in testing mode. if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then echo "$mode" exit From c488f7c4d8e2ff240d561820df11a51518199a53 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 1 Mar 2016 23:57:34 -0800 Subject: [PATCH 29/82] Fix bug in install permission setting. --- lib/spack/llnl/util/filesystem.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 24cfbfde719..da3cf96050f 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -152,15 +152,20 @@ def set_install_permissions(path): def copy_mode(src, dest): src_mode = os.stat(src).st_mode dest_mode = os.stat(dest).st_mode - if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR - if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP - if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH + if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR + if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP + if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH os.chmod(dest, dest_mode) def install(src, dest): """Manually install a file to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) + + # Expand dsst to its eventual full path if it is a directory. + if os.path.isdir(dest): + dest = join_path(dest, os.path.basename(src)) + shutil.copy(src, dest) set_install_permissions(dest) copy_mode(src, dest) From be306d09e99a69732afc3f44724222ab6c6d71cc Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 00:04:46 -0800 Subject: [PATCH 30/82] Move repo creation code into repository.py --- lib/spack/spack/cmd/repo.py | 46 +-------------------------- lib/spack/spack/repository.py | 58 ++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 46 deletions(-) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 908f5400ab2..c2e352786d1 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -74,51 +74,7 @@ def setup_parser(subparser): def repo_create(args): """Create a new package repository.""" - root = canonicalize_path(args.directory) - namespace = args.namespace - - if not args.namespace: - namespace = os.path.basename(root) - - if not re.match(r'\w[\.\w-]*', namespace): - tty.die("'%s' is not a valid namespace." % namespace) - - existed = False - if os.path.exists(root): - if os.path.isfile(root): - tty.die('File %s already exists and is not a directory' % root) - elif os.path.isdir(root): - if not os.access(root, os.R_OK | os.W_OK): - tty.die('Cannot create new repo in %s: cannot access directory.' % root) - if os.listdir(root): - tty.die('Cannot create new repo in %s: directory is not empty.' % root) - existed = True - - full_path = os.path.realpath(root) - parent = os.path.dirname(full_path) - if not os.access(parent, os.R_OK | os.W_OK): - tty.die("Cannot create repository in %s: can't access parent!" % root) - - try: - config_path = os.path.join(root, repo_config_name) - packages_path = os.path.join(root, packages_dir_name) - - mkdirp(packages_path) - with open(config_path, 'w') as config: - config.write("repo:\n") - config.write(" namespace: '%s'\n" % namespace) - - except (IOError, OSError) as e: - tty.die('Failed to create new repository in %s.' % root, - "Caused by %s: %s" % (type(e), e)) - - # try to clean up. - if existed: - shutil.rmtree(config_path, ignore_errors=True) - shutil.rmtree(packages_path, ignore_errors=True) - else: - shutil.rmtree(root, ignore_errors=True) - + full_path, namespace = create_repo(args.directory, args.namespace) tty.msg("Created repo with namespace '%s'." % namespace) tty.msg("To register it with spack, run this command:", 'spack repo add %s' % full_path) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index e8d0cc09ec2..6aa75cb43e5 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -33,7 +33,7 @@ from external import yaml import llnl.util.tty as tty -from llnl.util.filesystem import join_path +from llnl.util.filesystem import * import spack.error import spack.config @@ -705,6 +705,58 @@ def __contains__(self, pkg_name): return self.exists(pkg_name) +def create_repo(root, namespace=None): + """Create a new repository in root with the specified namespace. + + If the namespace is not provided, use basename of root. + Return the canonicalized path and the namespace of the created repository. + """ + root = canonicalize_path(root) + if not namespace: + namespace = os.path.basename(root) + + if not re.match(r'\w[\.\w-]*', namespace): + raise InvalidNamespaceError("'%s' is not a valid namespace." % namespace) + + existed = False + if os.path.exists(root): + if os.path.isfile(root): + raise BadRepoError('File %s already exists and is not a directory' % root) + elif os.path.isdir(root): + if not os.access(root, os.R_OK | os.W_OK): + raise BadRepoError('Cannot create new repo in %s: cannot access directory.' % root) + if os.listdir(root): + raise BadRepoError('Cannot create new repo in %s: directory is not empty.' % root) + existed = True + + full_path = os.path.realpath(root) + parent = os.path.dirname(full_path) + if not os.access(parent, os.R_OK | os.W_OK): + raise BadRepoError("Cannot create repository in %s: can't access parent!" % root) + + try: + config_path = os.path.join(root, repo_config_name) + packages_path = os.path.join(root, packages_dir_name) + + mkdirp(packages_path) + with open(config_path, 'w') as config: + config.write("repo:\n") + config.write(" namespace: '%s'\n" % namespace) + + except (IOError, OSError) as e: + raise BadRepoError('Failed to create new repository in %s.' % root, + "Caused by %s: %s" % (type(e), e)) + + # try to clean up. + if existed: + shutil.rmtree(config_path, ignore_errors=True) + shutil.rmtree(packages_path, ignore_errors=True) + else: + shutil.rmtree(root, ignore_errors=True) + + return full_path, namespace + + class RepoError(spack.error.SpackError): """Superclass for repository-related errors.""" @@ -713,6 +765,10 @@ class NoRepoConfiguredError(RepoError): """Raised when there are no repositories configured.""" +class InvalidNamespaceError(RepoError): + """Raised when an invalid namespace is encountered.""" + + class BadRepoError(RepoError): """Raised when repo layout is invalid.""" From 21d125c9147f5d3cfc0eb8a9516c8ce24df7279b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 00:08:36 -0800 Subject: [PATCH 31/82] Fixes #175: Dump environment provenance as well as build log. --- lib/spack/spack/directory_layout.py | 14 +++++++++++++- lib/spack/spack/package.py | 7 +++++++ lib/spack/spack/util/environment.py | 7 +++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 3e416a6a1fc..29d87b65b33 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -173,7 +173,9 @@ def __init__(self, root, **kwargs): self.spec_file_name = 'spec.yaml' self.extension_file_name = 'extensions.yaml' - self.build_log_name = 'build.out' # TODO: use config file. + self.build_log_name = 'build.out' # build log. + self.build_env_name = 'build.env' # build environment + self.packages_dir = 'repos' # archive of package.py files # Cache of already written/read extension maps. self._extension_maps = {} @@ -231,6 +233,16 @@ def build_log_path(self, spec): self.build_log_name) + def build_env_path(self, spec): + return join_path(self.path_for_spec(spec), self.metadata_dir, + self.build_env_name) + + + def build_packages_path(self, spec): + return join_path(self.path_for_spec(spec), self.metadata_dir, + self.packages_dir) + + def create_install_directory(self, spec): _check_concrete(spec) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8019b29cba5..5b1927fe8f4 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -66,6 +66,7 @@ from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive, extension from spack.util.executable import ProcessError +from spack.util.environment import dump_environment """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] @@ -884,10 +885,14 @@ def real_work(): # Do the real install in the source directory. self.stage.chdir_to_source() + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') + # This redirects I/O to a build log (and optionally to the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) self.install(self.spec, self.prefix) # Ensure that something was actually installed. @@ -896,7 +901,9 @@ def real_work(): # Move build log into install directory on success if not fake: log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) install(log_path, log_install_path) + install(env_path, env_install_path) # On successful install, remove the stage. if not keep_stage: diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index cd413dcfbc8..ae8e5708bef 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -63,3 +63,10 @@ def pop_keys(dictionary, *keys): for key in keys: if key in dictionary: dictionary.pop(key) + + +def dump_environment(path): + """Dump the current environment out to a file.""" + with open(path, 'w') as env_file: + for key,val in sorted(os.environ.items()): + env_file.write("%s=%s\n" % (key, val)) From 52081c46d641fcf679e70a04e9937db67e8787e4 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 00:09:24 -0800 Subject: [PATCH 32/82] Fixes #469: Store package.py files in the .spack directory. - Adds packages in spack repos inside the .spack directory, so that packages can be rebuilt automatically later. --- lib/spack/spack/package.py | 51 +++++++++++++++++++++++++++++++++++ lib/spack/spack/repository.py | 39 +++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 5b1927fe8f4..9f1825ca211 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -58,6 +58,7 @@ import spack.mirror import spack.hooks import spack.directives +import spack.repository import spack.build_environment import spack.url import spack.util.web @@ -502,6 +503,7 @@ def fetcher(self): self._fetcher = self._make_fetcher() return self._fetcher + @fetcher.setter def fetcher(self, f): self._fetcher = f @@ -905,6 +907,9 @@ def real_work(): install(log_path, log_install_path) install(env_path, env_install_path) + packages_dir = spack.install_layout.build_packages_path(self.spec) + dump_packages(self.spec, packages_dir) + # On successful install, remove the stage. if not keep_stage: self.stage.destroy() @@ -1219,6 +1224,52 @@ def validate_package_url(url_string): tty.die("Invalid file type in URL: '%s'" % url_string) +def dump_packages(spec, path): + """Dump all package information for a spec and its dependencies. + + This creates a package repository within path for every + namespace in the spec DAG, and fills the repos wtih package + files and patch files for every node in the DAG. + """ + mkdirp(path) + + # Copy in package.py files from any dependencies. + # Note that we copy them in as they are in the *install* directory + # NOT as they are in the repository, because we want a snapshot of + # how *this* particular build was done. + for node in spec.traverse(): + if node is not spec: + # Locate the dependency package in the install tree and find + # its provenance information. + source = spack.install_layout.build_packages_path(node) + source_repo_root = join_path(source, node.namespace) + + # There's no provenance installed for the source package. Skip it. + # User can always get something current from the builtin repo. + if not os.path.isdir(source_repo_root): + continue + + # Create a source repo and get the pkg directory out of it. + try: + source_repo = spack.repository.Repo(source_repo_root) + source_pkg_dir = source_repo.dirname_for_package_name(node.name) + except RepoError as e: + tty.warn("Warning: Couldn't copy in provenance for %s" % node.name) + + # Create a destination repository + dest_repo_root = join_path(path, node.namespace) + if not os.path.exists(dest_repo_root): + spack.repository.create_repo(dest_repo_root) + repo = spack.repository.Repo(dest_repo_root) + + # Get the location of the package in the dest repo. + dest_pkg_dir = repo.dirname_for_package_name(node.name) + if node is not spec: + install_tree(source_pkg_dir, dest_pkg_dir) + else: + spack.repo.dump_provenance(node, dest_pkg_dir) + + def print_pkg(message): """Outputs a message with a package icon.""" from llnl.util.tty.color import cwrite diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 6aa75cb43e5..8d06fefe7f0 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -316,6 +316,16 @@ def get(self, spec, new=False): return self.repo_for_pkg(spec).get(spec) + @_autospec + def dump_provenance(self, spec, path): + """Dump provenance information for a spec to a particular path. + + This dumps the package file and any associated patch files. + Raises UnknownPackageError if not found. + """ + return self.repo_for_pkg(spec).dump_provenance(spec, path) + + def dirname_for_package_name(self, pkg_name): return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) @@ -552,6 +562,35 @@ def get(self, spec, new=False): return self._instances[key] + @_autospec + def dump_provenance(self, spec, path): + """Dump provenance information for a spec to a particular path. + + This dumps the package file and any associated patch files. + Raises UnknownPackageError if not found. + """ + # Some preliminary checks. + if spec.virtual: + raise UnknownPackageError(spec.name) + + if spec.namespace and spec.namespace != self.namespace: + raise UnknownPackageError("Repository %s does not contain package %s." + % (self.namespace, spec.fullname)) + + # Install any patch files needed by packages. + mkdirp(path) + for spec, patches in spec.package.patches.items(): + for patch in patches: + if patch.path: + if os.path.exists(patch.path): + install(patch.path, path) + else: + tty.warn("Patch file did not exist: %s" % patch.path) + + # Install the package.py file itself. + install(self.filename_for_package_name(spec), path) + + def purge(self): """Clear entire package instance cache.""" self._instances.clear() From a0c6519de9d03744e33b5187d42ec0d966dbd3e0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 01:41:43 -0800 Subject: [PATCH 33/82] Fixes #476: create was broken for FTP URLs. --- lib/spack/spack/util/web.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index e26daef2964..73f4858b023 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -86,12 +86,12 @@ def _spider(args): if not "Content-type" in resp.headers: tty.debug("ignoring page " + url) - return pages + return pages, links if not resp.headers["Content-type"].startswith('text/html'): tty.debug("ignoring page " + url + " with content type " + resp.headers["Content-type"]) - return pages + return pages, links # Do the real GET request when we know it's just HTML. req.get_method = lambda: "GET" @@ -173,7 +173,7 @@ def spider(root_url, **kwargs): performance over a sequential fetch. """ max_depth = kwargs.setdefault('depth', 1) - pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) + pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) return pages, links From 726b350689bf6da7b82eec170001adc8100beb71 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 12:52:38 +0100 Subject: [PATCH 34/82] test : fixed failing unit tests --- lib/spack/spack/mirror.py | 49 +++++++++---------- lib/spack/spack/stage.py | 7 ++- lib/spack/spack/test/configure_guess.py | 12 ++--- lib/spack/spack/test/git_fetch.py | 31 +++++++------ lib/spack/spack/test/hg_fetch.py | 31 +++++++------ lib/spack/spack/test/link_tree.py | 6 +-- lib/spack/spack/test/mirror.py | 62 ++++++++++++------------- lib/spack/spack/test/svn_fetch.py | 31 +++++++------ 8 files changed, 116 insertions(+), 113 deletions(-) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index fa29e208036..bc8870926f9 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -168,32 +168,33 @@ def create(path, specs, **kwargs): pkg = spec.package tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) try: - for ii, stage in enumerate(pkg.stage): - fetcher = stage.fetcher - if ii == 0: - # create a subdirectory for the current package@version - archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) - name = spec.format("$_$@") - else: - resource = stage.resource - archive_path = join_path(subdir, suggest_archive_basename(resource)) - name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) - subdir = os.path.dirname(archive_path) - mkdirp(subdir) + with pkg.stage: + for ii, stage in enumerate(pkg.stage): + fetcher = stage.fetcher + if ii == 0: + # create a subdirectory for the current package@version + archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) + name = spec.format("$_$@") + else: + resource = stage.resource + archive_path = join_path(subdir, suggest_archive_basename(resource)) + name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) + subdir = os.path.dirname(archive_path) + mkdirp(subdir) - if os.path.exists(archive_path): - tty.msg("{name} : already added".format(name=name)) - else: - everything_already_exists = False - fetcher.fetch() - if not kwargs.get('no_checksum', False): - fetcher.check() - tty.msg("{name} : checksum passed".format(name=name)) + if os.path.exists(archive_path): + tty.msg("{name} : already added".format(name=name)) + else: + everything_already_exists = False + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("{name} : checksum passed".format(name=name)) - # Fetchers have to know how to archive their files. Use - # that to move/copy/create an archive in the mirror. - fetcher.archive(archive_path) - tty.msg("{name} : added".format(name=name)) + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("{name} : added".format(name=name)) if everything_already_exists: present.append(spec) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 96b1eaf3f2c..956d1c87069 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -404,10 +404,13 @@ def path(self): return self[0].path def __enter__(self): - return self[0].__enter__() + for item in self: + item.__enter__() + return self def __exit__(self, exc_type, exc_val, exc_tb): - return self[0].__exit__(exc_type, exc_val, exc_tb) + for item in reversed(self): + item.__exit__(exc_type, exc_val, exc_tb) def chdir_to_source(self): return self[0].chdir_to_source() diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index a4e8565b62b..bc2332acc23 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -52,8 +52,6 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) - if self.stage: - self.stage.destroy() os.chdir(self.orig_dir) @@ -64,12 +62,12 @@ def check_archive(self, filename, system): url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz') print url - self.stage = Stage(url) - self.stage.fetch() + with Stage(url) as stage: + stage.fetch() - guesser = ConfigureGuesser() - guesser(self.stage) - self.assertEqual(system, guesser.build_system) + guesser = ConfigureGuesser() + guesser(stage) + self.assertEqual(system, guesser.build_system) def test_python(self): diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index d84433176a8..caa076823e4 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -76,26 +76,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('git')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked_file = 'foobarbaz' - touch(untracked_file) - self.assertTrue(os.path.isfile(untracked_file)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked_file)) + untracked_file = 'foobarbaz' + touch(untracked_file) + self.assertTrue(os.path.isfile(untracked_file)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked_file)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_master(self): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index bbcb64e4c1a..75aa7ab17e7 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -68,26 +68,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('hg')] = args - self.pkg.do_stage() - self.assertEqual(self.repo.get_rev(), rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assertEqual(self.repo.get_rev(), rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assertEqual(self.repo.get_rev(), rev) + self.assertEqual(self.repo.get_rev(), rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 886b7ef4c5d..7b67e873dde 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -38,6 +38,8 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') + # FIXME : possibly this test needs to be refactored to avoid the explicit call to __enter__ and __exit__ + self.stage.__enter__() with working_dir(self.stage.path): touchp('source/1') @@ -51,10 +53,8 @@ def setUp(self): source_path = os.path.join(self.stage.path, 'source') self.link_tree = LinkTree(source_path) - def tearDown(self): - if self.stage: - self.stage.destroy() + self.stage.__exit__(None, None, None) def check_file_link(self, filename): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index f83cc8090c9..9e2c6319782 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -74,14 +74,14 @@ def set_up_package(self, name, MockRepoClass, url_attr): def check_mirror(self): - stage = Stage('spack-mirror-test') - mirror_root = join_path(stage.path, 'test-mirror') + with Stage('spack-mirror-test') as stage: + mirror_root = join_path(stage.path, 'test-mirror') + + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) - # register mirror with spack config - mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } - spack.config.update_config('mirrors', mirrors) - try: os.chdir(stage.path) spack.mirror.create( mirror_root, self.repos, no_checksum=True) @@ -97,38 +97,36 @@ def check_mirror(self): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Now try to fetch each package. - for name, mock_repo in self.repos.items(): - spec = Spec(name).concretized() - pkg = spec.package + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package - pkg._stage = None - saved_checksum_setting = spack.do_checksum - try: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) + saved_checksum_setting = spack.do_checksum + with pkg.stage: + try: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path(mock_repo.path, 'checked_out') - svn('checkout', mock_repo.url, original_path) + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) - dcmp = dircmp(original_path, pkg.stage.source_path) + dcmp = dircmp(original_path, pkg.stage.source_path) - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) - # and that all original files are present. - self.assertTrue(all(l in exclude for l in dcmp.left_only)) + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) - finally: - spack.do_checksum = saved_checksum_setting - pkg.do_clean() - finally: - stage.destroy() + finally: + spack.do_checksum = saved_checksum_setting + pkg.do_clean() def test_git_mirror(self): diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 454a7f1d1ff..6ac9e2f3431 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -82,26 +82,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('svn')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_default(self): From 65b2a24f7c12380d0815e5705d1c1e66e5fd22a9 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 13:03:40 +0100 Subject: [PATCH 35/82] stage : removed _setup method --- lib/spack/spack/stage.py | 57 ++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 956d1c87069..48770fb4079 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -111,9 +111,32 @@ def __init__(self, url_or_fetch_strategy, **kwargs): def __enter__(self): """ Entering a stage context will create the stage directory + + If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise + this will attempt to create a stage in a temporary directory and link it into spack.stage_path. + + Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location + in tmp_dirs, fall back to making the stage inside spack.stage_path. """ - # FIXME : if _setup is used only here, then it makes no sense to retain the function - self._setup() + # Create the top-level stage directory + mkdirp(spack.stage_path) + self._cleanup_dead_links() + + # If this is a temporary stage, them make the temp directory + if self.tmp_root: + if self._need_to_create_path(): + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + + # if we're not using a tmp dir, create the stage directly in the + # stage dir, rather than linking to it. + else: + if self._need_to_create_path(): + mkdirp(self.path) + + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) + return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -180,36 +203,6 @@ def _need_to_create_path(self): return False - def _setup(self): - """Creates the stage directory. - If spack.use_tmp_stage is False, the stage directory is created - directly under spack.stage_path. - - If spack.use_tmp_stage is True, this will attempt to create a - stage in a temporary directory and link it into spack.stage_path. - Spack will use the first writable location in spack.tmp_dirs to - create a stage. If there is no valid location in tmp_dirs, fall - back to making the stage inside spack.stage_path. - """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - self._cleanup_dead_links() - - # If this is a temporary stage, them make the temp directory - if self.tmp_root: - if self._need_to_create_path(): - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - - # if we're not using a tmp dir, create the stage directly in the - # stage dir, rather than linking to it. - else: - if self._need_to_create_path(): - mkdirp(self.path) - - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - @property def archive_file(self): """Path to the source archive within this stage directory.""" From d649b715edae0871254695e2deacf078554a1475 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 13:16:04 +0100 Subject: [PATCH 36/82] stage : updated functions doc _cleanup_dead_links : fixed minor bug --- lib/spack/spack/stage.py | 41 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 48770fb4079..900acd664da 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -42,33 +42,26 @@ class Stage(object): - """A Stage object manages a directory where some source code is - downloaded and built before being installed. It handles - fetching the source code, either as an archive to be expanded - or by checking it out of a repository. A stage's lifecycle - looks like this: + """ + A Stage object is a context manager that handles a directory where some source code is downloaded and built + before being installed. It handles fetching the source code, either as an archive to be expanded or by checking + it out of a repository. A stage's lifecycle looks like this: - Stage() - Constructor creates the stage directory. - fetch() - Fetch a source archive into the stage. - expand_archive() - Expand the source archive. - - Build and install the archive. This is handled by the Package class. - destroy() - Remove the stage once the package has been installed. + ``` + with Stage() as stage: # Context manager creates and destroys the stage directory + fetch() # Fetch a source archive into the stage. + expand_archive() # Expand the source archive. + # Build and install the archive. This is handled by the Package class. + ``` - If spack.use_tmp_stage is True, spack will attempt to create stages - in a tmp directory. Otherwise, stages are created directly in - spack.stage_path. + If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. + Otherwise, stages are created directly in spack.stage_path. - There are two kinds of stages: named and unnamed. Named stages can - persist between runs of spack, e.g. if you fetched a tarball but - didn't finish building it, you won't have to fetch it again. + There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you + fetched a tarball but didn't finish building it, you won't have to fetch it again. - Unnamed stages are created using standard mkdtemp mechanisms or - similar, and are intended to persist for only one run of spack. + Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for + only one run of spack. """ def __init__(self, url_or_fetch_strategy, **kwargs): @@ -164,7 +157,7 @@ def _cleanup_dead_links(self): path = join_path(spack.stage_path, file) if os.path.islink(path): real_path = os.path.realpath(path) - if not os.path.exists(path): + if not os.path.exists(real_path): os.unlink(path) def _need_to_create_path(self): From 9001b9ed3c01f8ccaceaca60d6a34c3551f77240 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 15:56:09 +0100 Subject: [PATCH 37/82] package : minor syntax fixes mirror : extracted add_single_spec from create --- lib/spack/spack/mirror.py | 98 ++++++++++++++++++++------------------ lib/spack/spack/package.py | 4 +- 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index bc8870926f9..5ed7aff176b 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -110,7 +110,6 @@ def suggest_archive_basename(resource): return basename - def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -158,58 +157,65 @@ def create(path, specs, **kwargs): "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. - present = [] - mirrored = [] - error = [] + categories = { + 'present': [], + 'mirrored': [], + 'error': [] + } # Iterate through packages and download all the safe tarballs for each of them - everything_already_exists = True for spec in version_specs: - pkg = spec.package - tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) - try: - with pkg.stage: - for ii, stage in enumerate(pkg.stage): - fetcher = stage.fetcher - if ii == 0: - # create a subdirectory for the current package@version - archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) - name = spec.format("$_$@") - else: - resource = stage.resource - archive_path = join_path(subdir, suggest_archive_basename(resource)) - name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) - subdir = os.path.dirname(archive_path) - mkdirp(subdir) + add_single_spec(spec, mirror_root, categories, **kwargs) - if os.path.exists(archive_path): - tty.msg("{name} : already added".format(name=name)) - else: - everything_already_exists = False - fetcher.fetch() - if not kwargs.get('no_checksum', False): - fetcher.check() - tty.msg("{name} : checksum passed".format(name=name)) + return categories['present'], categories['mirrored'], categories['error'] - # Fetchers have to know how to archive their files. Use - # that to move/copy/create an archive in the mirror. - fetcher.archive(archive_path) - tty.msg("{name} : added".format(name=name)) - if everything_already_exists: - present.append(spec) - else: - mirrored.append(spec) - except Exception, e: - if spack.debug: - sys.excepthook(*sys.exc_info()) - else: - tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) - error.append(spec) - finally: - pkg.stage.destroy() +def add_single_spec(spec, mirror_root, categories, **kwargs): + tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) + spec_exists_in_mirror = True + try: + with spec.package.stage: + # fetcher = stage.fetcher + # fetcher.fetch() + # ... + # fetcher.archive(archive_path) + for ii, stage in enumerate(spec.package.stage): + fetcher = stage.fetcher + if ii == 0: + # create a subdirectory for the current package@version + archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) + name = spec.format("$_$@") + else: + resource = stage.resource + archive_path = join_path(subdir, suggest_archive_basename(resource)) + name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) + subdir = os.path.dirname(archive_path) + mkdirp(subdir) - return (present, mirrored, error) + if os.path.exists(archive_path): + tty.msg("{name} : already added".format(name=name)) + else: + spec_exists_in_mirror = False + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("{name} : checksum passed".format(name=name)) + + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("{name} : added".format(name=name)) + + if spec_exists_in_mirror: + categories['present'].append(spec) + else: + categories['mirrored'].append(spec) + except Exception as e: + if spack.debug: + sys.excepthook(*sys.exc_info()) + else: + tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) + categories['error'].append(spec) class MirrorError(spack.error.SpackError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 4fdc582479d..29b781a7499 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -732,7 +732,7 @@ def do_patch(self): # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. if os.path.isfile(bad_file): - tty.msg("Patching failed last time. Restaging.") + tty.msg("Patching failed last time. Restaging.") self.stage.restage() self.stage.chdir_to_source() @@ -912,7 +912,7 @@ def real_work(): % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) print_pkg(self.prefix) - except ProcessError, e: + except ProcessError as e: # Annotate with location of build log. e.build_log = log_path cleanup() From 901e4851b993f73c4bdd92b85b2469d6673b7617 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 16:19:32 +0100 Subject: [PATCH 38/82] _cleanup_dead_links : factored method into a function and put it in llnl.filesystem --- lib/spack/llnl/util/filesystem.py | 19 +++++++++++++++++-- lib/spack/spack/stage.py | 20 ++++---------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 24cfbfde719..4a708b639a3 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links'] import os import sys @@ -235,7 +235,7 @@ def touchp(path): def force_symlink(src, dest): try: os.symlink(src, dest) - except OSError, e: + except OSError as e: os.remove(dest) os.symlink(src, dest) @@ -339,3 +339,18 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) + +def remove_dead_links(root): + """ + Removes any dead link that is present in root + + Args: + root: path where to search for dead links + + """ + for file in os.listdir(root): + path = join_path(root, file) + if os.path.islink(path): + real_path = os.path.realpath(path) + if not os.path.exists(real_path): + os.unlink(path) \ No newline at end of file diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 900acd664da..e9106431920 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -98,7 +98,7 @@ def __init__(self, url_or_fetch_strategy, **kwargs): # Try to construct here a temporary name for the stage directory # If this is a named stage, then construct a named path. self.path = join_path(spack.stage_path, self.name) - + # Flag to decide whether to delete the stage folder on exit or not self.delete_on_exit = True def __enter__(self): @@ -113,20 +113,17 @@ def __enter__(self): """ # Create the top-level stage directory mkdirp(spack.stage_path) - self._cleanup_dead_links() + remove_dead_links(spack.stage_path) - # If this is a temporary stage, them make the temp directory + # If a tmp_root exists then create a directory there and then link it in the stage area, + # otherwise create the stage directory in self.path if self.tmp_root: if self._need_to_create_path(): tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) os.symlink(tmp_dir, self.path) - - # if we're not using a tmp dir, create the stage directly in the - # stage dir, rather than linking to it. else: if self._need_to_create_path(): mkdirp(self.path) - # Make sure we can actually do something with the stage we made. ensure_access(self.path) @@ -151,15 +148,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): if self.delete_on_exit: self.destroy() - def _cleanup_dead_links(self): - """Remove any dead links in the stage directory.""" - for file in os.listdir(spack.stage_path): - path = join_path(spack.stage_path, file) - if os.path.islink(path): - real_path = os.path.realpath(path) - if not os.path.exists(real_path): - os.unlink(path) - def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. From 4d63544fe918393de9265f8879bb0199a542cc3b Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 16:55:57 +0100 Subject: [PATCH 39/82] remove_link_tree : moved to llnl.util.filesystem --- lib/spack/llnl/util/filesystem.py | 20 ++++++++++++++++++-- lib/spack/spack/stage.py | 21 +++------------------ 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 4a708b639a3..015eeb9aa1c 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree'] import os import sys @@ -353,4 +353,20 @@ def remove_dead_links(root): if os.path.islink(path): real_path = os.path.realpath(path) if not os.path.exists(real_path): - os.unlink(path) \ No newline at end of file + os.unlink(path) + +def remove_linked_tree(path): + """ + Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real + directory before removing the link. + + Args: + path: directory to be removed + + """ + if os.path.exists(path): + if os.path.islink(path): + shutil.rmtree(os.path.realpath(path), True) + os.unlink(path) + else: + shutil.rmtree(path, True) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index e9106431920..e87b822a8f4 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -114,15 +114,13 @@ def __enter__(self): # Create the top-level stage directory mkdirp(spack.stage_path) remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, # otherwise create the stage directory in self.path - if self.tmp_root: - if self._need_to_create_path(): + if self._need_to_create_path(): + if self.tmp_root: tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) os.symlink(tmp_dir, self.path) - else: - if self._need_to_create_path(): + else: mkdirp(self.path) # Make sure we can actually do something with the stage we made. ensure_access(self.path) @@ -436,19 +434,6 @@ def ensure_access(file=spack.stage_path): tty.die("Insufficient permissions for %s" % file) -def remove_linked_tree(path): - """Removes a directory and its contents. If the directory is a symlink, - follows the link and reamoves the real directory before removing the - link. - """ - if os.path.exists(path): - if os.path.islink(path): - shutil.rmtree(os.path.realpath(path), True) - os.unlink(path) - else: - shutil.rmtree(path, True) - - def purge(): """Remove all build directories in the top-level stage path.""" if os.path.isdir(spack.stage_path): From ca41909ec52ab2b8f82ca91421d3882300531ac2 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 17:19:27 +0100 Subject: [PATCH 40/82] package : removed do_clean() --- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/package.py | 9 +------- lib/spack/spack/test/git_fetch.py | 4 ---- lib/spack/spack/test/hg_fetch.py | 3 --- lib/spack/spack/test/mirror.py | 38 ++++++++++++------------------- lib/spack/spack/test/svn_fetch.py | 4 ---- 6 files changed, 17 insertions(+), 43 deletions(-) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 6e7179122c1..0c8bd1d5281 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -43,4 +43,4 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.do_clean() + package.stage.destroy() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 29b781a7499..fa64b2c0472 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1138,13 +1138,6 @@ def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() - - def do_clean(self): - """Removes the package's build stage and source tarball.""" - if os.path.exists(self.stage.path): - self.stage.destroy() - - def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) @@ -1181,7 +1174,7 @@ def fetch_remote_versions(self): try: return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) - except spack.error.NoNetworkConnectionError, e: + except spack.error.NoNetworkConnectionError as e: tty.die("Package.fetch_versions couldn't connect to:", e.url, e.message) diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index caa076823e4..6d6a67a1d38 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -52,19 +52,15 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def assert_rev(self, rev): """Check that the current git revision is equal to the supplied rev.""" self.assertEqual(self.repo.rev_hash('HEAD'), self.repo.rev_hash(rev)) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 75aa7ab17e7..d884ed78a0f 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -49,13 +49,10 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def try_fetch(self, rev, test_file, args): """Tries to: diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 9e2c6319782..f117e042424 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -104,29 +104,21 @@ def check_mirror(self): saved_checksum_setting = spack.do_checksum with pkg.stage: - try: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) - - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path(mock_repo.path, 'checked_out') - svn('checkout', mock_repo.url, original_path) - - dcmp = dircmp(original_path, pkg.stage.source_path) - - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) - - # and that all original files are present. - self.assertTrue(all(l in exclude for l in dcmp.left_only)) - - finally: - spack.do_checksum = saved_checksum_setting - pkg.do_clean() + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) + dcmp = dircmp(original_path, pkg.stage.source_path) + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) + spack.do_checksum = saved_checksum_setting def test_git_mirror(self): diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 6ac9e2f3431..13a00f5df79 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -51,13 +51,10 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" @@ -70,7 +67,6 @@ def get_rev(): return match.group(1) self.assertEqual(get_rev(), rev) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with From 21cd05aad5e8f45a7ebac7b213855a16685d229f Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 17:28:21 +0100 Subject: [PATCH 41/82] package : updated doc --- lib/spack/spack/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fa64b2c0472..0214dcd7716 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -291,7 +291,6 @@ class SomePackage(Package): .. code-block:: python - p.do_clean() # removes the stage directory entirely p.do_restage() # removes the build directory and # re-expands the archive. From 6bbadbfaac406f9abe81e64f174d8a2b4c9c255e Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 2 Mar 2016 13:18:43 -0600 Subject: [PATCH 42/82] Add CPATH pointing to include directories. --- lib/spack/docs/basic_usage.rst | 2 +- lib/spack/spack/modules.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 0578f0c8db6..f94ac3d2ba8 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -896,7 +896,7 @@ Or, similarly with modules, you could type: $ spack load mpich %gcc@4.4.7 These commands will add appropriate directories to your ``PATH``, -``MANPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use +``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use a package, you can type unload or unuse similarly: .. code-block:: sh diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index c8347635646..c27043db8cd 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -33,6 +33,7 @@ * /bin directories to be appended to PATH * /lib* directories for LD_LIBRARY_PATH + * /include directories for CPATH * /man* and /share/man* directories for MANPATH * the package prefix for CMAKE_PREFIX_PATH @@ -121,6 +122,7 @@ def add_path(path_name, directory): ('LIBRARY_PATH', self.spec.prefix.lib64), ('LD_LIBRARY_PATH', self.spec.prefix.lib), ('LD_LIBRARY_PATH', self.spec.prefix.lib64), + ('CPATH', self.spec.prefix.include), ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')), ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]: From 976d0240c467b9ff74333fafcd2ac1e896819a80 Mon Sep 17 00:00:00 2001 From: citibeth Date: Wed, 2 Mar 2016 18:58:47 -0500 Subject: [PATCH 43/82] Removed FIXMEs. --- .../repos/builtin/packages/blitz/package.py | 27 +------------------ .../builtin/packages/netcdf-cxx4/package.py | 21 +-------------- .../packages/netcdf-fortran/package.py | 2 +- .../repos/builtin/packages/proj/package.py | 19 +------------ 4 files changed, 4 insertions(+), 65 deletions(-) diff --git a/var/spack/repos/builtin/packages/blitz/package.py b/var/spack/repos/builtin/packages/blitz/package.py index 82ff6349250..9413b276fe9 100644 --- a/var/spack/repos/builtin/packages/blitz/package.py +++ b/var/spack/repos/builtin/packages/blitz/package.py @@ -1,40 +1,15 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install blitz -# -# You can always get back here to change things with: -# -# spack edit blitz -# -# See the spack documentation for more information on building -# packages. -# from spack import * class Blitz(Package): """N-dimensional arrays for C++""" homepage = "http://github.com/blitzpp/blitz" - -# This version doesn't have the configure script generated yet. url = "https://github.com/blitzpp/blitz/tarball/1.0.0" -#http://prdownloads.sourceforge.net/%(namelower)s version('1.0.0', '9f040b9827fe22228a892603671a77af') - # FIXME: Add dependencies if this package requires them. - # depends_on("foo") + # No dependencies def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. - # FIXME: Spack couldn't guess one, so here are some options: configure('--prefix=%s' % prefix) - # cmake('.', *std_cmake_args) - - # FIXME: Add logic to build and install here make() make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index 9d70eab05f5..8d51a106794 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -1,19 +1,3 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install netcdf-cxx4 -# -# You can always get back here to change things with: -# -# spack edit netcdf-cxx4 -# -# See the spack documentation for more information on building -# packages. -# from spack import * class NetcdfCxx4(Package): @@ -25,17 +9,14 @@ class NetcdfCxx4(Package): variant('mpi', default=True, description='Enables MPI parallelism') -# variant('hdf4', default=False, description="Enable HDF4 support") - # NetCDF-CXX4 doesn't really depend (directly) on MPI. However... this + # netcdf-cxx4 doesn't really depend (directly) on MPI. However... this # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) depends_on('mpi', when='+mpi') depends_on('netcdf') def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. configure('--prefix=%s' % prefix) - # FIXME: Add logic to build and install here make() make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 8e5c8ecc3a0..9e4aee95fb8 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -10,7 +10,7 @@ class NetcdfFortran(Package): variant('mpi', default=True, description='Enables MPI parallelism') - # NetCDF-CXX4 doesn't really depend (directly) on MPI. However... this + # netcdf-fortran doesn't really depend (directly) on MPI. However... this # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) depends_on('mpi', when='+mpi') depends_on('netcdf') diff --git a/var/spack/repos/builtin/packages/proj/package.py b/var/spack/repos/builtin/packages/proj/package.py index 4a0d3feac75..797772f4f6f 100644 --- a/var/spack/repos/builtin/packages/proj/package.py +++ b/var/spack/repos/builtin/packages/proj/package.py @@ -1,19 +1,3 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install proj -# -# You can always get back here to change things with: -# -# spack edit proj -# -# See the spack documentation for more information on building -# packages. -# from spack import * class Proj(Package): @@ -27,8 +11,7 @@ class Proj(Package): version('4.7.0', '927d34623b52e0209ba2bfcca18fe8cd') version('4.6.1', '7dbaab8431ad50c25669fd3fb28dc493') - # FIXME: Add dependencies if this package requires them. - # depends_on("foo") + # No dependencies def install(self, spec, prefix): configure('--prefix=%s' % prefix) From 717bcb862428a1b1a3b9c70f925c3fe6c325f622 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 09:17:49 +0100 Subject: [PATCH 44/82] test : optimized import statements --- lib/spack/spack/test/concretize.py | 2 -- lib/spack/spack/test/config.py | 6 ++--- lib/spack/spack/test/configure_guess.py | 9 ++------ lib/spack/spack/test/database.py | 11 +++++---- lib/spack/spack/test/directory_layout.py | 13 +++++------ lib/spack/spack/test/git_fetch.py | 11 ++------- lib/spack/spack/test/hg_fetch.py | 10 +++------ lib/spack/spack/test/install.py | 9 ++------ lib/spack/spack/test/link_tree.py | 7 ++---- lib/spack/spack/test/lock.py | 8 +++---- lib/spack/spack/test/make_executable.py | 6 ++--- lib/spack/spack/test/mirror.py | 5 ++--- lib/spack/spack/test/mock_packages_test.py | 8 +++---- lib/spack/spack/test/mock_repo.py | 4 ---- lib/spack/spack/test/multimethod.py | 5 +---- lib/spack/spack/test/namespace_trie.py | 1 + lib/spack/spack/test/optional_deps.py | 4 +--- lib/spack/spack/test/packages.py | 26 ++++++---------------- lib/spack/spack/test/python_version.py | 3 +-- lib/spack/spack/test/spec_dag.py | 2 -- lib/spack/spack/test/spec_semantics.py | 1 - lib/spack/spack/test/spec_syntax.py | 3 ++- lib/spack/spack/test/stage.py | 8 +++---- lib/spack/spack/test/svn_fetch.py | 14 ++++-------- lib/spack/spack/test/tally_plugin.py | 4 ++-- lib/spack/spack/test/unit_install.py | 3 ++- lib/spack/spack/test/url_extrapolate.py | 3 --- lib/spack/spack/test/url_parse.py | 2 +- lib/spack/spack/test/url_substitution.py | 1 - lib/spack/spack/test/versions.py | 1 + lib/spack/spack/test/yaml.py | 1 + 31 files changed, 62 insertions(+), 129 deletions(-) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 7f2938aec5a..794344fb6ae 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -22,8 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - import spack from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index d8be5a855b8..0562d2d6209 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -22,13 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import shutil import os +import shutil from tempfile import mkdtemp -from ordereddict_backport import OrderedDict + import spack import spack.config +from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * # Some sample compiler config data diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index bc2332acc23..2440d120e5d 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -23,20 +23,15 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest import shutil import tempfile +import unittest from llnl.util.filesystem import * - from spack.cmd.create import ConfigureGuesser from spack.stage import Stage - -from spack.fetch_strategy import URLFetchStrategy -from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which from spack.test.mock_packages_test import * -from spack.test.mock_repo import MockArchive +from spack.util.executable import which class InstallTest(unittest.TestCase): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 0205f4b8ce2..9a57e1f03ea 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -26,19 +26,18 @@ These tests check the database is functioning properly, both in memory and in its file """ -import tempfile -import shutil import multiprocessing - -from llnl.util.lock import * -from llnl.util.filesystem import join_path +import shutil +import tempfile import spack +from llnl.util.filesystem import join_path +from llnl.util.lock import * +from llnl.util.tty.colify import colify from spack.database import Database from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * -from llnl.util.tty.colify import colify def _print_ref_counts(): """Print out all ref counts for the graph used here, for debugging""" diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 925cb648edd..d814572d4a0 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -25,20 +25,17 @@ """\ This test verifies that the Spack directory layout works properly. """ -import unittest -import tempfile -import shutil import os - -from llnl.util.filesystem import * +import shutil +import tempfile import spack -from spack.spec import Spec -from spack.repository import RepoPath +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout +from spack.repository import RepoPath +from spack.spec import Spec from spack.test.mock_packages_test import * - # number of packages to test (to reduce test time) max_packages = 10 diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 6d6a67a1d38..35780441166 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -23,19 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which - +from llnl.util.filesystem import * from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo +from spack.version import ver class GitFetchTest(MockPackagesTest): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index d884ed78a0f..b8a0c1ec460 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -23,16 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest - -from llnl.util.filesystem import * - import spack + from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * from spack.test.mock_repo import MockHgRepo +from llnl.util.filesystem import * +from spack.test.mock_packages_test import * class HgFetchTest(MockPackagesTest): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 8863d13c426..8297893f012 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -22,18 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import unittest import shutil import tempfile -from llnl.util.filesystem import * - import spack -from spack.stage import Stage -from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which +from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite from spack.test.mock_packages_test import * from spack.test.mock_repo import MockArchive diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 7b67e873dde..ee37e765c7b 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -24,8 +24,6 @@ ############################################################################## import os import unittest -import shutil -import tempfile from llnl.util.filesystem import * from llnl.util.link_tree import LinkTree @@ -38,8 +36,7 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') - # FIXME : possibly this test needs to be refactored to avoid the explicit call to __enter__ and __exit__ - self.stage.__enter__() + self.stage.create() with working_dir(self.stage.path): touchp('source/1') @@ -54,7 +51,7 @@ def setUp(self): self.link_tree = LinkTree(source_path) def tearDown(self): - self.stage.__exit__(None, None, None) + self.stage.destroy() def check_file_link(self, filename): diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index bc68df01dbe..3b11d18da49 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -25,15 +25,13 @@ """ These tests ensure that our lock works correctly. """ -import unittest -import os -import tempfile import shutil +import tempfile +import unittest from multiprocessing import Process -from llnl.util.lock import * from llnl.util.filesystem import join_path, touch - +from llnl.util.lock import * from spack.util.multiproc import Barrier # This is the longest a failed test will take, as the barriers will diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index d568a28d443..a2606acf19b 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -28,13 +28,13 @@ This just tests whether the right args are getting passed to make. """ import os -import unittest -import tempfile import shutil +import tempfile +import unittest from llnl.util.filesystem import * -from spack.util.environment import path_put_first from spack.build_environment import MakeExecutable +from spack.util.environment import path_put_first class MakeExecutableTest(unittest.TestCase): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index f117e042424..e707adfe9d5 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -23,11 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from filecmp import dircmp - import spack import spack.mirror -from spack.util.compression import decompressor_for + +from filecmp import dircmp from spack.test.mock_packages_test import * from spack.test.mock_repo import * diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index e9f1f95df55..0b8867b61e2 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,17 +22,15 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys import os import shutil -import unittest import tempfile -from ordereddict_backport import OrderedDict - -from llnl.util.filesystem import mkdirp +import unittest import spack import spack.config +from llnl.util.filesystem import mkdirp +from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index ed94023b0eb..a8bdfb55710 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -26,13 +26,9 @@ import shutil from llnl.util.filesystem import * - -import spack -from spack.version import ver from spack.stage import Stage from spack.util.executable import which - # # VCS Systems used by mock repo code. # diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 7bf4ff0a0a5..2d4b8cd584a 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -25,14 +25,11 @@ """ Test for multi_method dispatch. """ -import unittest import spack from spack.multimethod import * -from spack.version import * -from spack.spec import Spec -from spack.multimethod import when from spack.test.mock_packages_test import * +from spack.version import * class MultiMethodTest(MockPackagesTest): diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index d0d809004d5..83fb34bf765 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + from spack.util.naming import NamespaceTrie diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index ebd72819992..55f35ea4c96 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -22,10 +22,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import spack -from spack.spec import Spec, CompilerSpec +from spack.spec import Spec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 83984dc5f65..07ff0b21af0 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -22,14 +22,12 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - -from llnl.util.filesystem import join_path import spack +from llnl.util.filesystem import join_path from spack.repository import Repo -from spack.util.naming import mod_to_class from spack.test.mock_packages_test import * +from spack.util.naming import mod_to_class class PackagesTest(MockPackagesTest): @@ -77,30 +75,20 @@ def test_package_class_names(self): # def test_import_package(self): - import spack.pkg.builtin.mock.mpich + pass def test_import_package_as(self): - import spack.pkg.builtin.mock.mpich as mp + pass def test_import_class_from_package(self): - from spack.pkg.builtin.mock.mpich import Mpich + pass def test_import_module_from_package(self): - from spack.pkg.builtin.mock import mpich + pass def test_import_namespace_container_modules(self): - import spack.pkg - import spack.pkg as p - from spack import pkg - - import spack.pkg.builtin - import spack.pkg.builtin as b - from spack.pkg import builtin - - import spack.pkg.builtin.mock - import spack.pkg.builtin.mock as m - from spack.pkg.builtin import mock + pass diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index d74d3b9b7d7..42949753044 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -28,12 +28,11 @@ Spack was originally 2.7, but enough systems in 2014 are still using 2.6 on their frontend nodes that we need 2.6 to get adopted. """ -import unittest import os import re +import unittest import llnl.util.tty as tty - import pyqver2 import spack diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 632f777cdeb..5e6162b6e66 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -31,8 +31,6 @@ import spack import spack.package -from llnl.util.lang import list_modules - from spack.spec import Spec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 44a09cbd7fb..8c33d1ff6e7 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -22,7 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest from spack.spec import * from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 1daaa4be8fc..6e08e30e135 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -23,9 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + import spack.spec -from spack.spec import * from spack.parse import Token +from spack.spec import * # Sample output for a complex lexing. complex_lex = [Token(ID, 'mvapich_foo'), diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index a7314eba4c5..dbcf89d8645 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -25,15 +25,13 @@ """\ Test that the Stage class works correctly. """ -import unittest -import shutil import os -import getpass +import shutil +import unittest from contextlib import * -from llnl.util.filesystem import * - import spack +from llnl.util.filesystem import * from spack.stage import Stage from spack.util.executable import which diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 13a00f5df79..1ee4ee700e7 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -24,18 +24,12 @@ ############################################################################## import os import re -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * - import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * + from spack.test.mock_repo import svn, MockSvnRepo +from spack.version import ver +from spack.test.mock_packages_test import * +from llnl.util.filesystem import * class SvnFetchTest(MockPackagesTest): diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index 9ca898c47ce..eb1e4a3240e 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -22,10 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from nose.plugins import Plugin - import os +from nose.plugins import Plugin + class Tally(Plugin): name = 'tally' diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index ccc409dd602..18615b7efe8 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -22,10 +22,11 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest import itertools +import unittest import spack + test_install = __import__("spack.cmd.test-install", fromlist=["BuildId", "create_test_output", "TestResult"]) diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 87adf894017..068a335b491 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -25,10 +25,7 @@ """\ Tests ability of spack to extrapolate URL versions from existing versions. """ -import spack import spack.url as url -from spack.spec import Spec -from spack.version import ver from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index efde7c0c73b..561d4658a10 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -27,8 +27,8 @@ detection in Homebrew. """ import unittest + import spack.url as url -from pprint import pprint class UrlParseTest(unittest.TestCase): diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index aec8baf4eaa..2be38af0d34 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -27,7 +27,6 @@ """ import unittest -import spack import spack.url as url diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 108450e098e..2732006eb3d 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -28,6 +28,7 @@ where it makes sense. """ import unittest + from spack.version import * diff --git a/lib/spack/spack/test/yaml.py b/lib/spack/spack/test/yaml.py index 5a357b8e696..b930c022f26 100644 --- a/lib/spack/spack/test/yaml.py +++ b/lib/spack/spack/test/yaml.py @@ -26,6 +26,7 @@ Test Spack's custom YAML format. """ import unittest + import spack.util.spack_yaml as syaml test_file = """\ From 353d12d2e6e24c0f62795dd9e649a801b566c4d8 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 09:18:47 +0100 Subject: [PATCH 45/82] stage : added create function as decided in #477 comments --- lib/spack/spack/stage.py | 50 ++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index e87b822a8f4..5d8b09fa9e6 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -105,26 +105,10 @@ def __enter__(self): """ Entering a stage context will create the stage directory - If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise - this will attempt to create a stage in a temporary directory and link it into spack.stage_path. - - Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location - in tmp_dirs, fall back to making the stage inside spack.stage_path. + Returns: + self """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, - # otherwise create the stage directory in self.path - if self._need_to_create_path(): - if self.tmp_root: - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - else: - mkdirp(self.path) - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - + self.create() return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -314,8 +298,34 @@ def restage(self): """ self.fetcher.reset() + def create(self): + """ + Creates the stage directory + + If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise + this will attempt to create a stage in a temporary directory and link it into spack.stage_path. + + Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location + in tmp_dirs, fall back to making the stage inside spack.stage_path. + """ + # Create the top-level stage directory + mkdirp(spack.stage_path) + remove_dead_links(spack.stage_path) + # If a tmp_root exists then create a directory there and then link it in the stage area, + # otherwise create the stage directory in self.path + if self._need_to_create_path(): + if self.tmp_root: + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + else: + mkdirp(self.path) + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) + def destroy(self): - """Remove this stage directory.""" + """ + Removes this stage directory + """ remove_linked_tree(self.path) # Make sure we don't end up in a removed directory From 5ef36215569e1151164ecb630011de425387732a Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 10:10:30 +0100 Subject: [PATCH 46/82] stage : minor syntax fix --- lib/spack/spack/stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 5d8b09fa9e6..985043c1035 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -246,7 +246,7 @@ def fetch(self, mirror_only=False): self.fetcher = fetcher self.fetcher.fetch() break - except spack.error.SpackError, e: + except spack.error.SpackError as e: tty.msg("Fetching from %s failed." % fetcher) tty.debug(e) continue From db92699c02f9aaf435d6bc073c89cd2c75d8824c Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 11:05:36 +0100 Subject: [PATCH 47/82] test : fixed tests that were broken by import optimization --- lib/spack/spack/test/packages.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 07ff0b21af0..f0b5e05f3b5 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -75,20 +75,30 @@ def test_package_class_names(self): # def test_import_package(self): - pass + import spack.pkg.builtin.mock.mpich def test_import_package_as(self): - pass + import spack.pkg.builtin.mock.mpich as mp def test_import_class_from_package(self): - pass + from spack.pkg.builtin.mock.mpich import Mpich def test_import_module_from_package(self): - pass + from spack.pkg.builtin.mock import mpich def test_import_namespace_container_modules(self): - pass + import spack.pkg + import spack.pkg as p + from spack import pkg + + import spack.pkg.builtin + import spack.pkg.builtin as b + from spack.pkg import builtin + + import spack.pkg.builtin.mock + import spack.pkg.builtin.mock as m + from spack.pkg.builtin import mock From a531a4f5e306388fcb6aa3c2f9db7bb93b530242 Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Thu, 3 Mar 2016 18:00:50 +0100 Subject: [PATCH 48/82] help spack versions paraview to get versions without v --- lib/spack/spack/url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 02c0b83e260..ad551a6dedb 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -225,7 +225,7 @@ def parse_version_offset(path): (r'_((\d+\.)+\d+[a-z]?)[.]orig$', stem), # e.g. http://www.openssl.org/source/openssl-0.9.8s.tar.gz - (r'-([^-]+(-alpha|-beta)?)', stem), + (r'-v?([^-]+(-alpha|-beta)?)', stem), # e.g. astyle_1.23_macosx.tar.gz (r'_([^_]+(_alpha|_beta)?)', stem), From 7183db1b7df3bce7e5f174ce38a50476c2e3b218 Mon Sep 17 00:00:00 2001 From: Scott Pakin Date: Thu, 3 Mar 2016 11:30:57 -0700 Subject: [PATCH 49/82] Added LLVM 3.7.1 support --- .../repos/builtin/packages/llvm/package.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index 934d994bd3e..280e400f696 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -34,7 +34,7 @@ class Llvm(Package): it is the full name of the project. """ homepage = 'http://llvm.org/' - url = 'http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz' + url = 'http://llvm.org/releases/3.7.1/llvm-3.7.1.src.tar.xz' version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package @@ -132,6 +132,21 @@ class Llvm(Package): 'llvm-libunwind' : 'http://llvm.org/svn/llvm-project/libunwind/trunk', } }, + { + 'version' : '3.7.1', + 'md5':'bf8b3a2c79e61212c5409041dfdbd319', + 'resources' : { + 'compiler-rt' : '1c6975daf30bb3b0473b53c3a1a6ff01', + 'openmp' : 'b4ad08cda4e5c22e42b66062b140438e', + 'polly' : '3a2a7367002740881637f4d47bca4dc3', + 'libcxx' : 'f9c43fa552a10e14ff53b94d04bea140', + 'libcxxabi' : '52d925afac9f97e9dcac90745255c169', + 'clang' : '0acd026b5529164197563d135a8fd83e', + 'clang-tools-extra' : '5d49ff745037f061a7c86aeb6a24c3d2', + 'lldb' : 'a106d8a0d21fc84d76953822fbaf3398', + 'llvm-libunwind' : '814bd52c9247c5d04629658fbcb3ab8c', + } + }, { 'version' : '3.7.0', 'md5':'b98b9495e5655a672d6cb83e1a180f8e', From c5d9ee89246b3d2aeddb756a04588424051d3295 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 3 Mar 2016 14:41:20 -0800 Subject: [PATCH 50/82] Revert "refactoring proposal : turn Stage into a context manager" --- lib/spack/llnl/util/filesystem.py | 35 +--- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/mirror.py | 55 +++--- lib/spack/spack/package.py | 176 +++++++++---------- lib/spack/spack/stage.py | 186 +++++++++++---------- lib/spack/spack/test/concretize.py | 2 + lib/spack/spack/test/config.py | 6 +- lib/spack/spack/test/configure_guess.py | 21 ++- lib/spack/spack/test/database.py | 11 +- lib/spack/spack/test/directory_layout.py | 13 +- lib/spack/spack/test/git_fetch.py | 46 +++-- lib/spack/spack/test/hg_fetch.py | 44 ++--- lib/spack/spack/test/install.py | 9 +- lib/spack/spack/test/link_tree.py | 7 +- lib/spack/spack/test/lock.py | 8 +- lib/spack/spack/test/make_executable.py | 6 +- lib/spack/spack/test/mirror.py | 69 ++++---- lib/spack/spack/test/mock_packages_test.py | 8 +- lib/spack/spack/test/mock_repo.py | 4 + lib/spack/spack/test/multimethod.py | 5 +- lib/spack/spack/test/namespace_trie.py | 1 - lib/spack/spack/test/optional_deps.py | 4 +- lib/spack/spack/test/packages.py | 6 +- lib/spack/spack/test/python_version.py | 3 +- lib/spack/spack/test/spec_dag.py | 2 + lib/spack/spack/test/spec_semantics.py | 1 + lib/spack/spack/test/spec_syntax.py | 3 +- lib/spack/spack/test/stage.py | 130 ++++++++------ lib/spack/spack/test/svn_fetch.py | 49 +++--- lib/spack/spack/test/tally_plugin.py | 4 +- lib/spack/spack/test/unit_install.py | 3 +- lib/spack/spack/test/url_extrapolate.py | 3 + lib/spack/spack/test/url_parse.py | 2 +- lib/spack/spack/test/url_substitution.py | 1 + lib/spack/spack/test/versions.py | 1 - lib/spack/spack/test/yaml.py | 1 - 36 files changed, 504 insertions(+), 423 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 366237ef8f3..da3cf96050f 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] import os import sys @@ -240,7 +240,7 @@ def touchp(path): def force_symlink(src, dest): try: os.symlink(src, dest) - except OSError as e: + except OSError, e: os.remove(dest) os.symlink(src, dest) @@ -344,34 +344,3 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) - -def remove_dead_links(root): - """ - Removes any dead link that is present in root - - Args: - root: path where to search for dead links - - """ - for file in os.listdir(root): - path = join_path(root, file) - if os.path.islink(path): - real_path = os.path.realpath(path) - if not os.path.exists(real_path): - os.unlink(path) - -def remove_linked_tree(path): - """ - Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real - directory before removing the link. - - Args: - path: directory to be removed - - """ - if os.path.exists(path): - if os.path.islink(path): - shutil.rmtree(os.path.realpath(path), True) - os.unlink(path) - else: - shutil.rmtree(path, True) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 0c8bd1d5281..6e7179122c1 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -43,4 +43,4 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.stage.destroy() + package.do_clean() diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 5ed7aff176b..fa29e208036 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -110,6 +110,7 @@ def suggest_archive_basename(resource): return basename + def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -157,29 +158,17 @@ def create(path, specs, **kwargs): "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. - categories = { - 'present': [], - 'mirrored': [], - 'error': [] - } + present = [] + mirrored = [] + error = [] # Iterate through packages and download all the safe tarballs for each of them + everything_already_exists = True for spec in version_specs: - add_single_spec(spec, mirror_root, categories, **kwargs) - - return categories['present'], categories['mirrored'], categories['error'] - - -def add_single_spec(spec, mirror_root, categories, **kwargs): - tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) - spec_exists_in_mirror = True - try: - with spec.package.stage: - # fetcher = stage.fetcher - # fetcher.fetch() - # ... - # fetcher.archive(archive_path) - for ii, stage in enumerate(spec.package.stage): + pkg = spec.package + tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) + try: + for ii, stage in enumerate(pkg.stage): fetcher = stage.fetcher if ii == 0: # create a subdirectory for the current package@version @@ -195,7 +184,7 @@ def add_single_spec(spec, mirror_root, categories, **kwargs): if os.path.exists(archive_path): tty.msg("{name} : already added".format(name=name)) else: - spec_exists_in_mirror = False + everything_already_exists = False fetcher.fetch() if not kwargs.get('no_checksum', False): fetcher.check() @@ -206,16 +195,20 @@ def add_single_spec(spec, mirror_root, categories, **kwargs): fetcher.archive(archive_path) tty.msg("{name} : added".format(name=name)) - if spec_exists_in_mirror: - categories['present'].append(spec) - else: - categories['mirrored'].append(spec) - except Exception as e: - if spack.debug: - sys.excepthook(*sys.exc_info()) - else: - tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) - categories['error'].append(spec) + if everything_already_exists: + present.append(spec) + else: + mirrored.append(spec) + except Exception, e: + if spack.debug: + sys.excepthook(*sys.exc_info()) + else: + tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) + error.append(spec) + finally: + pkg.stage.destroy() + + return (present, mirrored, error) class MirrorError(spack.error.SpackError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 106c546d5ce..9f1825ca211 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -293,6 +293,7 @@ class SomePackage(Package): .. code-block:: python + p.do_clean() # removes the stage directory entirely p.do_restage() # removes the build directory and # re-expands the archive. @@ -502,6 +503,7 @@ def fetcher(self): self._fetcher = self._make_fetcher() return self._fetcher + @fetcher.setter def fetcher(self, f): self._fetcher = f @@ -733,7 +735,7 @@ def do_patch(self): # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. if os.path.isfile(bad_file): - tty.msg("Patching failed last time. Restaging.") + tty.msg("Patching failed last time. Restaging.") self.stage.restage() self.stage.chdir_to_source() @@ -848,103 +850,102 @@ def do_install(self, make_jobs=make_jobs) start_time = time.time() - with self.stage: - if not fake: - if not skip_patch: - self.do_patch() + if not fake: + if not skip_patch: + self.do_patch() + else: + self.do_stage() + + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) + + def cleanup(): + if not keep_prefix: + # If anything goes wrong, remove the install prefix + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed." + + "Manually remove this directory to fix:", + self.prefix, wrap=True) + + + def real_work(): + try: + tty.msg("Building %s." % self.name) + + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) + + # Set up process's build environment before running install. + if fake: + self.do_fake_install() else: - self.do_stage() + # Do the real install in the source directory. + self.stage.chdir_to_source() - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) + # This redirects I/O to a build log (and optionally to the terminal) + log_path = join_path(os.getcwd(), 'spack-build.out') + log_file = open(log_path, 'w') + with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) + self.install(self.spec, self.prefix) + # Ensure that something was actually installed. + self._sanity_check_install() - def real_work(): - try: - tty.msg("Building %s." % self.name) + # Move build log into install directory on success + if not fake: + log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) + install(log_path, log_install_path) + install(env_path, env_install_path) - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) + packages_dir = spack.install_layout.build_packages_path(self.spec) + dump_packages(self.spec, packages_dir) - # Set up process's build environment before running install. - if fake: - self.do_fake_install() - else: - # Do the real install in the source directory. - self.stage.chdir_to_source() + # On successful install, remove the stage. + if not keep_stage: + self.stage.destroy() - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # This redirects I/O to a build log (and optionally to the terminal) - log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): - dump_environment(env_path) - self.install(self.spec, self.prefix) + tty.msg("Successfully installed %s." % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - # Ensure that something was actually installed. - self._sanity_check_install() + except ProcessError, e: + # Annotate with location of build log. + e.build_log = log_path + cleanup() + raise e - # Move build log into install directory on success - if not fake: - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - install(log_path, log_install_path) - install(env_path, env_install_path) + except: + # other exceptions just clean up and raise. + cleanup() + raise - packages_dir = spack.install_layout.build_packages_path(self.spec) - dump_packages(self.spec, packages_dir) + # Set parallelism before starting build. + self.make_jobs = make_jobs - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + # Do the build. + spack.build_environment.fork(self, real_work) - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) - tty.msg("Successfully installed %s." % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) - - except ProcessError as e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e - - except: - # other exceptions just clean up and raise. - cleanup() - raise - - # Set parallelism before starting build. - self.make_jobs = make_jobs - - # Do the build. - spack.build_environment.fork(self, real_work) - - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) - - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) + # Once everything else is done, run post install hooks + spack.hooks.post_install(self) def _sanity_check_install(self): @@ -1148,6 +1149,13 @@ def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() + + def do_clean(self): + """Removes the package's build stage and source tarball.""" + if os.path.exists(self.stage.path): + self.stage.destroy() + + def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) @@ -1184,7 +1192,7 @@ def fetch_remote_versions(self): try: return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) - except spack.error.NoNetworkConnectionError as e: + except spack.error.NoNetworkConnectionError, e: tty.die("Package.fetch_versions couldn't connect to:", e.url, e.message) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 985043c1035..f217450d42d 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -42,26 +42,33 @@ class Stage(object): - """ - A Stage object is a context manager that handles a directory where some source code is downloaded and built - before being installed. It handles fetching the source code, either as an archive to be expanded or by checking - it out of a repository. A stage's lifecycle looks like this: + """A Stage object manages a directory where some source code is + downloaded and built before being installed. It handles + fetching the source code, either as an archive to be expanded + or by checking it out of a repository. A stage's lifecycle + looks like this: - ``` - with Stage() as stage: # Context manager creates and destroys the stage directory - fetch() # Fetch a source archive into the stage. - expand_archive() # Expand the source archive. - # Build and install the archive. This is handled by the Package class. - ``` + Stage() + Constructor creates the stage directory. + fetch() + Fetch a source archive into the stage. + expand_archive() + Expand the source archive. + + Build and install the archive. This is handled by the Package class. + destroy() + Remove the stage once the package has been installed. - If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. - Otherwise, stages are created directly in spack.stage_path. + If spack.use_tmp_stage is True, spack will attempt to create stages + in a tmp directory. Otherwise, stages are created directly in + spack.stage_path. - There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you - fetched a tarball but didn't finish building it, you won't have to fetch it again. + There are two kinds of stages: named and unnamed. Named stages can + persist between runs of spack, e.g. if you fetched a tarball but + didn't finish building it, you won't have to fetch it again. - Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for - only one run of spack. + Unnamed stages are created using standard mkdtemp mechanisms or + similar, and are intended to persist for only one run of spack. """ def __init__(self, url_or_fetch_strategy, **kwargs): @@ -89,46 +96,21 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. - # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name - # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root - self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names()) + self.name = kwargs.get('name') self.mirror_path = kwargs.get('mirror_path') self.tmp_root = find_tmp_root() - # Try to construct here a temporary name for the stage directory - # If this is a named stage, then construct a named path. - self.path = join_path(spack.stage_path, self.name) - # Flag to decide whether to delete the stage folder on exit or not - self.delete_on_exit = True + self.path = None + self._setup() - def __enter__(self): - """ - Entering a stage context will create the stage directory - - Returns: - self - """ - self.create() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """ - Exiting from a stage context will delete the stage directory unless: - - it was explicitly requested not to do so - - an exception has been raised - - Args: - exc_type: exception type - exc_val: exception value - exc_tb: exception traceback - - Returns: - Boolean - """ - self.delete_on_exit = False if exc_type is not None else self.delete_on_exit - - if self.delete_on_exit: - self.destroy() + def _cleanup_dead_links(self): + """Remove any dead links in the stage directory.""" + for file in os.listdir(spack.stage_path): + path = join_path(spack.stage_path, file) + if os.path.islink(path): + real_path = os.path.realpath(path) + if not os.path.exists(path): + os.unlink(path) def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we @@ -166,6 +148,54 @@ def _need_to_create_path(self): return False + def _setup(self): + """Creates the stage directory. + If spack.use_tmp_stage is False, the stage directory is created + directly under spack.stage_path. + + If spack.use_tmp_stage is True, this will attempt to create a + stage in a temporary directory and link it into spack.stage_path. + Spack will use the first writable location in spack.tmp_dirs to + create a stage. If there is no valid location in tmp_dirs, fall + back to making the stage inside spack.stage_path. + """ + # Create the top-level stage directory + mkdirp(spack.stage_path) + self._cleanup_dead_links() + + # If this is a named stage, then construct a named path. + if self.name is not None: + self.path = join_path(spack.stage_path, self.name) + + # If this is a temporary stage, them make the temp directory + tmp_dir = None + if self.tmp_root: + if self.name is None: + # Unnamed tmp root. Link the path in + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + self.name = os.path.basename(tmp_dir) + self.path = join_path(spack.stage_path, self.name) + if self._need_to_create_path(): + os.symlink(tmp_dir, self.path) + + else: + if self._need_to_create_path(): + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + + # if we're not using a tmp dir, create the stage directly in the + # stage dir, rather than linking to it. + else: + if self.name is None: + self.path = tempfile.mkdtemp('', STAGE_PREFIX, spack.stage_path) + self.name = os.path.basename(self.path) + else: + if self._need_to_create_path(): + mkdirp(self.path) + + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) + @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -246,7 +276,7 @@ def fetch(self, mirror_only=False): self.fetcher = fetcher self.fetcher.fetch() break - except spack.error.SpackError as e: + except spack.error.SpackError, e: tty.msg("Fetching from %s failed." % fetcher) tty.debug(e) continue @@ -298,34 +328,8 @@ def restage(self): """ self.fetcher.reset() - def create(self): - """ - Creates the stage directory - - If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise - this will attempt to create a stage in a temporary directory and link it into spack.stage_path. - - Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location - in tmp_dirs, fall back to making the stage inside spack.stage_path. - """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, - # otherwise create the stage directory in self.path - if self._need_to_create_path(): - if self.tmp_root: - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - else: - mkdirp(self.path) - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - def destroy(self): - """ - Removes this stage directory - """ + """Remove this stage directory.""" remove_linked_tree(self.path) # Make sure we don't end up in a removed directory @@ -385,15 +389,6 @@ def source_path(self): def path(self): return self[0].path - def __enter__(self): - for item in self: - item.__enter__() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - for item in reversed(self): - item.__exit__(exc_type, exc_val, exc_tb) - def chdir_to_source(self): return self[0].chdir_to_source() @@ -444,6 +439,19 @@ def ensure_access(file=spack.stage_path): tty.die("Insufficient permissions for %s" % file) +def remove_linked_tree(path): + """Removes a directory and its contents. If the directory is a symlink, + follows the link and reamoves the real directory before removing the + link. + """ + if os.path.exists(path): + if os.path.islink(path): + shutil.rmtree(os.path.realpath(path), True) + os.unlink(path) + else: + shutil.rmtree(path, True) + + def purge(): """Remove all build directories in the top-level stage path.""" if os.path.isdir(spack.stage_path): diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 794344fb6ae..7f2938aec5a 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -22,6 +22,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest + import spack from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 0562d2d6209..d8be5a855b8 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -22,13 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os +import unittest import shutil +import os from tempfile import mkdtemp - +from ordereddict_backport import OrderedDict import spack import spack.config -from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * # Some sample compiler config data diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index 2440d120e5d..a4e8565b62b 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -23,15 +23,20 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import unittest import shutil import tempfile -import unittest from llnl.util.filesystem import * + from spack.cmd.create import ConfigureGuesser from spack.stage import Stage -from spack.test.mock_packages_test import * + +from spack.fetch_strategy import URLFetchStrategy +from spack.directory_layout import YamlDirectoryLayout from spack.util.executable import which +from spack.test.mock_packages_test import * +from spack.test.mock_repo import MockArchive class InstallTest(unittest.TestCase): @@ -47,6 +52,8 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) + if self.stage: + self.stage.destroy() os.chdir(self.orig_dir) @@ -57,12 +64,12 @@ def check_archive(self, filename, system): url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz') print url - with Stage(url) as stage: - stage.fetch() + self.stage = Stage(url) + self.stage.fetch() - guesser = ConfigureGuesser() - guesser(stage) - self.assertEqual(system, guesser.build_system) + guesser = ConfigureGuesser() + guesser(self.stage) + self.assertEqual(system, guesser.build_system) def test_python(self): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 9a57e1f03ea..0205f4b8ce2 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -26,18 +26,19 @@ These tests check the database is functioning properly, both in memory and in its file """ -import multiprocessing -import shutil import tempfile +import shutil +import multiprocessing + +from llnl.util.lock import * +from llnl.util.filesystem import join_path import spack -from llnl.util.filesystem import join_path -from llnl.util.lock import * -from llnl.util.tty.colify import colify from spack.database import Database from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * +from llnl.util.tty.colify import colify def _print_ref_counts(): """Print out all ref counts for the graph used here, for debugging""" diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index d814572d4a0..925cb648edd 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -25,17 +25,20 @@ """\ This test verifies that the Spack directory layout works properly. """ -import os -import shutil +import unittest import tempfile +import shutil +import os + +from llnl.util.filesystem import * import spack -from llnl.util.filesystem import * -from spack.directory_layout import YamlDirectoryLayout -from spack.repository import RepoPath from spack.spec import Spec +from spack.repository import RepoPath +from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * + # number of packages to test (to reduce test time) max_packages = 10 diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 35780441166..d84433176a8 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -23,12 +23,19 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import unittest +import shutil +import tempfile + +from llnl.util.filesystem import * import spack -from llnl.util.filesystem import * +from spack.version import ver +from spack.stage import Stage +from spack.util.executable import which + from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo -from spack.version import ver class GitFetchTest(MockPackagesTest): @@ -45,15 +52,19 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) + def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() self.repo.destroy() + self.pkg.do_clean() + def assert_rev(self, rev): """Check that the current git revision is equal to the supplied rev.""" self.assertEqual(self.repo.rev_hash('HEAD'), self.repo.rev_hash(rev)) + def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -65,27 +76,26 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('git')] = args - with self.pkg.stage: - self.pkg.do_stage() - self.assert_rev(rev) + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked_file = 'foobarbaz' - touch(untracked_file) - self.assertTrue(os.path.isfile(untracked_file)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked_file)) + untracked_file = 'foobarbaz' + touch(untracked_file) + self.assertTrue(os.path.isfile(untracked_file)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked_file)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_master(self): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index b8a0c1ec460..bbcb64e4c1a 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -23,12 +23,16 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import spack +import unittest -from spack.version import ver -from spack.test.mock_repo import MockHgRepo from llnl.util.filesystem import * + +import spack +from spack.version import ver +from spack.stage import Stage +from spack.util.executable import which from spack.test.mock_packages_test import * +from spack.test.mock_repo import MockHgRepo class HgFetchTest(MockPackagesTest): @@ -45,10 +49,13 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) + def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() self.repo.destroy() + self.pkg.do_clean() + def try_fetch(self, rev, test_file, args): """Tries to: @@ -61,27 +68,26 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('hg')] = args - with self.pkg.stage: - self.pkg.do_stage() - self.assertEqual(self.repo.get_rev(), rev) + self.pkg.do_stage() + self.assertEqual(self.repo.get_rev(), rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assertEqual(self.repo.get_rev(), rev) + self.assertEqual(self.repo.get_rev(), rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 8297893f012..8863d13c426 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -22,13 +22,18 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os +import unittest import shutil import tempfile -import spack from llnl.util.filesystem import * -from spack.directory_layout import YamlDirectoryLayout + +import spack +from spack.stage import Stage from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite +from spack.directory_layout import YamlDirectoryLayout +from spack.util.executable import which from spack.test.mock_packages_test import * from spack.test.mock_repo import MockArchive diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index ee37e765c7b..886b7ef4c5d 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -24,6 +24,8 @@ ############################################################################## import os import unittest +import shutil +import tempfile from llnl.util.filesystem import * from llnl.util.link_tree import LinkTree @@ -36,7 +38,6 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') - self.stage.create() with working_dir(self.stage.path): touchp('source/1') @@ -50,8 +51,10 @@ def setUp(self): source_path = os.path.join(self.stage.path, 'source') self.link_tree = LinkTree(source_path) + def tearDown(self): - self.stage.destroy() + if self.stage: + self.stage.destroy() def check_file_link(self, filename): diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index 3b11d18da49..bc68df01dbe 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -25,13 +25,15 @@ """ These tests ensure that our lock works correctly. """ -import shutil -import tempfile import unittest +import os +import tempfile +import shutil from multiprocessing import Process -from llnl.util.filesystem import join_path, touch from llnl.util.lock import * +from llnl.util.filesystem import join_path, touch + from spack.util.multiproc import Barrier # This is the longest a failed test will take, as the barriers will diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index a2606acf19b..d568a28d443 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -28,13 +28,13 @@ This just tests whether the right args are getting passed to make. """ import os -import shutil -import tempfile import unittest +import tempfile +import shutil from llnl.util.filesystem import * -from spack.build_environment import MakeExecutable from spack.util.environment import path_put_first +from spack.build_environment import MakeExecutable class MakeExecutableTest(unittest.TestCase): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index e707adfe9d5..f83cc8090c9 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -23,10 +23,11 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +from filecmp import dircmp + import spack import spack.mirror - -from filecmp import dircmp +from spack.util.compression import decompressor_for from spack.test.mock_packages_test import * from spack.test.mock_repo import * @@ -73,14 +74,14 @@ def set_up_package(self, name, MockRepoClass, url_attr): def check_mirror(self): - with Stage('spack-mirror-test') as stage: - mirror_root = join_path(stage.path, 'test-mirror') - - # register mirror with spack config - mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } - spack.config.update_config('mirrors', mirrors) + stage = Stage('spack-mirror-test') + mirror_root = join_path(stage.path, 'test-mirror') + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) + try: os.chdir(stage.path) spack.mirror.create( mirror_root, self.repos, no_checksum=True) @@ -96,28 +97,38 @@ def check_mirror(self): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Now try to fetch each package. - for name, mock_repo in self.repos.items(): - spec = Spec(name).concretized() - pkg = spec.package + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package - saved_checksum_setting = spack.do_checksum - with pkg.stage: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path(mock_repo.path, 'checked_out') - svn('checkout', mock_repo.url, original_path) - dcmp = dircmp(original_path, pkg.stage.source_path) - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) - # and that all original files are present. - self.assertTrue(all(l in exclude for l in dcmp.left_only)) - spack.do_checksum = saved_checksum_setting + pkg._stage = None + saved_checksum_setting = spack.do_checksum + try: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) + + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) + + dcmp = dircmp(original_path, pkg.stage.source_path) + + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) + + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) + + finally: + spack.do_checksum = saved_checksum_setting + pkg.do_clean() + finally: + stage.destroy() def test_git_mirror(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 0b8867b61e2..e9f1f95df55 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,15 +22,17 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import sys import os import shutil -import tempfile import unittest +import tempfile +from ordereddict_backport import OrderedDict + +from llnl.util.filesystem import mkdirp import spack import spack.config -from llnl.util.filesystem import mkdirp -from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index a8bdfb55710..ed94023b0eb 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -26,9 +26,13 @@ import shutil from llnl.util.filesystem import * + +import spack +from spack.version import ver from spack.stage import Stage from spack.util.executable import which + # # VCS Systems used by mock repo code. # diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 2d4b8cd584a..7bf4ff0a0a5 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -25,11 +25,14 @@ """ Test for multi_method dispatch. """ +import unittest import spack from spack.multimethod import * -from spack.test.mock_packages_test import * from spack.version import * +from spack.spec import Spec +from spack.multimethod import when +from spack.test.mock_packages_test import * class MultiMethodTest(MockPackagesTest): diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index 2023ba6d96f..647976df210 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -23,7 +23,6 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest - from spack.util.naming import NamespaceTrie diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index 55f35ea4c96..ebd72819992 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -22,8 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest -from spack.spec import Spec +import spack +from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index f0b5e05f3b5..83984dc5f65 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -22,12 +22,14 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest + +from llnl.util.filesystem import join_path import spack -from llnl.util.filesystem import join_path from spack.repository import Repo -from spack.test.mock_packages_test import * from spack.util.naming import mod_to_class +from spack.test.mock_packages_test import * class PackagesTest(MockPackagesTest): diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index 42949753044..d74d3b9b7d7 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -28,11 +28,12 @@ Spack was originally 2.7, but enough systems in 2014 are still using 2.6 on their frontend nodes that we need 2.6 to get adopted. """ +import unittest import os import re -import unittest import llnl.util.tty as tty + import pyqver2 import spack diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 5e6162b6e66..632f777cdeb 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -31,6 +31,8 @@ import spack import spack.package +from llnl.util.lang import list_modules + from spack.spec import Spec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 8c33d1ff6e7..44a09cbd7fb 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest from spack.spec import * from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 6e08e30e135..1daaa4be8fc 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -23,10 +23,9 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest - import spack.spec -from spack.parse import Token from spack.spec import * +from spack.parse import Token # Sample output for a complex lexing. complex_lex = [Token(ID, 'mvapich_foo'), diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index dbcf89d8645..c1b2a2a5730 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -25,13 +25,15 @@ """\ Test that the Stage class works correctly. """ -import os -import shutil import unittest +import shutil +import os +import getpass from contextlib import * -import spack from llnl.util.filesystem import * + +import spack from spack.stage import Stage from spack.util.executable import which @@ -190,90 +192,116 @@ def check_destroy(self, stage, stage_name): def test_setup_and_destroy_name_with_tmp(self): with use_tmp(True): - with Stage(archive_url, name=stage_name) as stage: - self.check_setup(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + self.check_setup(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_setup_and_destroy_name_without_tmp(self): with use_tmp(False): - with Stage(archive_url, name=stage_name) as stage: - self.check_setup(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + self.check_setup(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_setup_and_destroy_no_name_with_tmp(self): with use_tmp(True): - with Stage(archive_url) as stage: - self.check_setup(stage, None) + stage = Stage(archive_url) + self.check_setup(stage, None) + + stage.destroy() self.check_destroy(stage, None) def test_setup_and_destroy_no_name_without_tmp(self): with use_tmp(False): - with Stage(archive_url) as stage: - self.check_setup(stage, None) + stage = Stage(archive_url) + self.check_setup(stage, None) + + stage.destroy() self.check_destroy(stage, None) def test_chdir(self): - with Stage(archive_url, name=stage_name) as stage: - stage.chdir() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + + stage.chdir() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_fetch(self): - with Stage(archive_url, name=stage_name) as stage: - stage.fetch() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + + stage.fetch() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_expand_archive(self): - with Stage(archive_url, name=stage_name) as stage: - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - stage.expand_archive() - self.check_expand_archive(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + + stage.expand_archive() + self.check_expand_archive(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_expand_archive(self): - with Stage(archive_url, name=stage_name) as stage: - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_restage(self): - with Stage(archive_url, name=stage_name) as stage: - stage.fetch() - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + stage = Stage(archive_url, name=stage_name) - # Try to make a file in the old archive dir - with open('foobar', 'w') as file: - file.write("this file is to be destroyed.") + stage.fetch() + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) - self.assertTrue('foobar' in os.listdir(stage.source_path)) + # Try to make a file in the old archive dir + with open('foobar', 'w') as file: + file.write("this file is to be destroyed.") - # Make sure the file is not there after restage. - stage.restage() - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - stage.chdir_to_source() - self.check_chdir_to_source(stage, stage_name) - self.assertFalse('foobar' in os.listdir(stage.source_path)) + self.assertTrue('foobar' in os.listdir(stage.source_path)) + + # Make sure the file is not there after restage. + stage.restage() + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + + stage.chdir_to_source() + self.check_chdir_to_source(stage, stage_name) + self.assertFalse('foobar' in os.listdir(stage.source_path)) + + stage.destroy() self.check_destroy(stage, stage_name) diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 1ee4ee700e7..454a7f1d1ff 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -24,13 +24,19 @@ ############################################################################## import os import re -import spack +import unittest +import shutil +import tempfile -from spack.test.mock_repo import svn, MockSvnRepo -from spack.version import ver -from spack.test.mock_packages_test import * from llnl.util.filesystem import * +import spack +from spack.version import ver +from spack.stage import Stage +from spack.util.executable import which +from spack.test.mock_packages_test import * +from spack.test.mock_repo import svn, MockSvnRepo + class SvnFetchTest(MockPackagesTest): """Tests fetching from a dummy git repository.""" @@ -45,10 +51,13 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) + def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() self.repo.destroy() + self.pkg.do_clean() + def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" @@ -61,6 +70,7 @@ def get_rev(): return match.group(1) self.assertEqual(get_rev(), rev) + def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -72,27 +82,26 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('svn')] = args - with self.pkg.stage: - self.pkg.do_stage() - self.assert_rev(rev) + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index 4163ab95dd6..e0b9618e0cb 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -22,10 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os - from nose.plugins import Plugin +import os + class Tally(Plugin): name = 'tally' diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index 18615b7efe8..ccc409dd602 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -22,11 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import itertools import unittest +import itertools import spack - test_install = __import__("spack.cmd.test-install", fromlist=["BuildId", "create_test_output", "TestResult"]) diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 068a335b491..87adf894017 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -25,7 +25,10 @@ """\ Tests ability of spack to extrapolate URL versions from existing versions. """ +import spack import spack.url as url +from spack.spec import Spec +from spack.version import ver from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index 561d4658a10..efde7c0c73b 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -27,8 +27,8 @@ detection in Homebrew. """ import unittest - import spack.url as url +from pprint import pprint class UrlParseTest(unittest.TestCase): diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index 2be38af0d34..aec8baf4eaa 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -27,6 +27,7 @@ """ import unittest +import spack import spack.url as url diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 2732006eb3d..108450e098e 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -28,7 +28,6 @@ where it makes sense. """ import unittest - from spack.version import * diff --git a/lib/spack/spack/test/yaml.py b/lib/spack/spack/test/yaml.py index b930c022f26..5a357b8e696 100644 --- a/lib/spack/spack/test/yaml.py +++ b/lib/spack/spack/test/yaml.py @@ -26,7 +26,6 @@ Test Spack's custom YAML format. """ import unittest - import spack.util.spack_yaml as syaml test_file = """\ From 311d71ec86bc7e245520a816cee4053c31a8a086 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 3 Mar 2016 16:50:51 -0600 Subject: [PATCH 51/82] Modifications to get PGI working --- lib/spack/env/cc | 6 +++--- lib/spack/env/pgi/case-insensitive/pgCC | 1 - lib/spack/env/pgi/{pgf77 => pgc++} | 0 lib/spack/env/pgi/{pgf90 => pgfortran} | 0 lib/spack/spack/compilers/pgi.py | 18 +++++++++--------- 5 files changed, 12 insertions(+), 13 deletions(-) delete mode 120000 lib/spack/env/pgi/case-insensitive/pgCC rename lib/spack/env/pgi/{pgf77 => pgc++} (100%) rename lib/spack/env/pgi/{pgf90 => pgfortran} (100%) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index a323c481243..b8b6c86e010 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -90,15 +90,15 @@ case "$command" in command="$SPACK_CC" language="C" ;; - c++|CC|g++|clang++|icpc|pgCC|xlc++) + c++|CC|g++|clang++|icpc|pgc++|xlc++) command="$SPACK_CXX" language="C++" ;; - f90|fc|f95|gfortran|ifort|pgf90|xlf90|nagfor) + f90|fc|f95|gfortran|ifort|pgfortran|xlf90|nagfor) command="$SPACK_FC" language="Fortran 90" ;; - f77|gfortran|ifort|pgf77|xlf|nagfor) + f77|gfortran|ifort|pgfortran|xlf|nagfor) command="$SPACK_F77" language="Fortran 77" ;; diff --git a/lib/spack/env/pgi/case-insensitive/pgCC b/lib/spack/env/pgi/case-insensitive/pgCC deleted file mode 120000 index e2deb67f3b6..00000000000 --- a/lib/spack/env/pgi/case-insensitive/pgCC +++ /dev/null @@ -1 +0,0 @@ -../../cc \ No newline at end of file diff --git a/lib/spack/env/pgi/pgf77 b/lib/spack/env/pgi/pgc++ similarity index 100% rename from lib/spack/env/pgi/pgf77 rename to lib/spack/env/pgi/pgc++ diff --git a/lib/spack/env/pgi/pgf90 b/lib/spack/env/pgi/pgfortran similarity index 100% rename from lib/spack/env/pgi/pgf90 rename to lib/spack/env/pgi/pgfortran diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 9ac74cfbdb5..c6a1078bd9f 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -29,28 +29,28 @@ class Pgi(Compiler): cc_names = ['pgcc'] # Subclasses use possible names of C++ compiler - cxx_names = ['pgCC'] + cxx_names = ['pgc++', 'pgCC'] # Subclasses use possible names of Fortran 77 compiler - f77_names = ['pgf77'] + f77_names = ['pgfortran', 'pgf77'] # Subclasses use possible names of Fortran 90 compiler - fc_names = ['pgf95', 'pgf90'] + fc_names = ['pgfortran', 'pgf95', 'pgf90'] # Named wrapper links within spack.build_env_path link_paths = { 'cc' : 'pgi/pgcc', - 'cxx' : 'pgi/case-insensitive/pgCC', - 'f77' : 'pgi/pgf77', - 'fc' : 'pgi/pgf90' } + 'cxx' : 'pgi/pgc++', + 'f77' : 'pgi/pgfortran', + 'fc' : 'pgi/pgfortran' } @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. Output looks like this:: - pgf95 10.2-0 64-bit target on x86-64 Linux -tp nehalem-64 - Copyright 1989-2000, The Portland Group, Inc. All Rights Reserved. - Copyright 2000-2010, STMicroelectronics, Inc. All Rights Reserved. + pgcc 15.10-0 64-bit target on x86-64 Linux -tp sandybridge + The Portland Group - PGI Compilers and Tools + Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. """ return get_compiler_version( comp, '-V', r'pg[^ ]* ([^ ]+) \d\d\d?-bit target') From 202fbc681265bd26739e0af3c37bb09a80b7e564 Mon Sep 17 00:00:00 2001 From: citibeth Date: Thu, 3 Mar 2016 23:40:29 -0500 Subject: [PATCH 52/82] Rmoved periods. Should help use of cut'n'paste from console windows. --- lib/spack/spack/cmd/checksum.py | 6 +++--- lib/spack/spack/cmd/compiler.py | 6 +++--- lib/spack/spack/cmd/create.py | 10 +++++----- lib/spack/spack/cmd/mirror.py | 4 ++-- lib/spack/spack/cmd/module.py | 6 +++--- lib/spack/spack/cmd/repo.py | 12 ++++++------ lib/spack/spack/config.py | 6 +++--- lib/spack/spack/database.py | 2 +- lib/spack/spack/directives.py | 4 ++-- lib/spack/spack/directory_layout.py | 4 ++-- lib/spack/spack/fetch_strategy.py | 16 ++++++++-------- lib/spack/spack/mirror.py | 4 ++-- lib/spack/spack/package.py | 26 +++++++++++++------------- lib/spack/spack/repository.py | 6 +++--- lib/spack/spack/stage.py | 4 ++-- 15 files changed, 58 insertions(+), 58 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index c4519932335..966ff9a5e96 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -100,11 +100,11 @@ def checksum(parser, args): else: versions = pkg.fetch_remote_versions() if not versions: - tty.die("Could not fetch any versions for %s." % pkg.name) + tty.die("Could not fetch any versions for %s" % pkg.name) sorted_versions = sorted(versions, reverse=True) - tty.msg("Found %s versions of %s." % (len(versions), pkg.name), + tty.msg("Found %s versions of %s" % (len(versions), pkg.name), *spack.cmd.elide_list( ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) print @@ -121,7 +121,7 @@ def checksum(parser, args): keep_stage=args.keep_stage) if not version_hashes: - tty.die("Could not fetch any versions for %s." % pkg.name) + tty.die("Could not fetch any versions for %s" % pkg.name) version_lines = [" version('%s', '%s')" % (v, h) for v, h in version_hashes] tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 75b51f6b495..3e58e821847 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -96,7 +96,7 @@ def compiler_remove(args): compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) if not compilers: - tty.die("No compilers match spec %s." % cspec) + tty.die("No compilers match spec %s" % cspec) elif not args.all and len(compilers) > 1: tty.error("Multiple compilers match spec %s. Choose one:" % cspec) colify(reversed(sorted([c.spec for c in compilers])), indent=4) @@ -105,7 +105,7 @@ def compiler_remove(args): for compiler in compilers: spack.compilers.remove_compiler_from_config(compiler.spec, scope=args.scope) - tty.msg("Removed compiler %s." % compiler.spec) + tty.msg("Removed compiler %s" % compiler.spec) def compiler_info(args): @@ -114,7 +114,7 @@ def compiler_info(args): compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) if not compilers: - tty.error("No compilers match spec %s." % cspec) + tty.error("No compilers match spec %s" % cspec) else: for c in compilers: print str(c.spec) + ":" diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 6809209046e..4564143f834 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -156,7 +156,7 @@ def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL version = spack.url.parse_version(url) if not version: - tty.die("Couldn't guess a version string from %s." % url) + tty.die("Couldn't guess a version string from %s" % url) # Try to guess a name. If it doesn't work, allow the user to override. if args.alternate_name: @@ -189,7 +189,7 @@ def find_repository(spec, args): try: repo = Repo(repo_path) if spec.namespace and spec.namespace != repo.namespace: - tty.die("Can't create package with namespace %s in repo with namespace %s." + tty.die("Can't create package with namespace %s in repo with namespace %s" % (spec.namespace, repo.namespace)) except RepoError as e: tty.die(str(e)) @@ -252,7 +252,7 @@ def create(parser, args): name = spec.name # factors out namespace, if any repo = find_repository(spec, args) - tty.msg("This looks like a URL for %s version %s." % (name, version)) + tty.msg("This looks like a URL for %s version %s" % (name, version)) tty.msg("Creating template for package %s" % name) # Fetch tarballs (prompting user if necessary) @@ -266,7 +266,7 @@ def create(parser, args): keep_stage=args.keep_stage) if not ver_hash_tuples: - tty.die("Could not fetch any tarballs for %s." % name) + tty.die("Could not fetch any tarballs for %s" % name) # Prepend 'py-' to python package names, by convention. if guesser.build_system == 'python': @@ -291,4 +291,4 @@ def create(parser, args): # If everything checks out, go ahead and edit. spack.editor(pkg_path) - tty.msg("Created package %s." % pkg_path) + tty.msg("Created package %s" % pkg_path) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 8e9438c1a37..fcd15a6a904 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -126,7 +126,7 @@ def mirror_remove(args): old_value = mirrors.pop(name) spack.config.update_config('mirrors', mirrors, scope=args.scope) - tty.msg("Removed mirror %s with url %s." % (name, old_value)) + tty.msg("Removed mirror %s with url %s" % (name, old_value)) def mirror_list(args): @@ -203,7 +203,7 @@ def mirror_create(args): verb = "updated" if existed else "created" tty.msg( - "Successfully %s mirror in %s." % (verb, directory), + "Successfully %s mirror in %s" % (verb, directory), "Archive stats:", " %-4d already present" % p, " %-4d added" % m, diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index a5a9570eb52..1d6867c1d96 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -58,7 +58,7 @@ def module_find(mtype, spec_array): should type to use that package's module. """ if mtype not in module_types: - tty.die("Invalid module type: '%s'. Options are %s." % (mtype, comma_or(module_types))) + tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) specs = spack.cmd.parse_specs(spec_array) if len(specs) > 1: @@ -78,7 +78,7 @@ def module_find(mtype, spec_array): mt = module_types[mtype] mod = mt(specs[0]) if not os.path.isfile(mod.file_name): - tty.die("No %s module is installed for %s." % (mtype, spec)) + tty.die("No %s module is installed for %s" % (mtype, spec)) print mod.use_name @@ -94,7 +94,7 @@ def module_refresh(): shutil.rmtree(cls.path, ignore_errors=False) mkdirp(cls.path) for spec in specs: - tty.debug(" Writing file for %s." % spec) + tty.debug(" Writing file for %s" % spec) cls(spec).write() diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index c2e352786d1..87c782000f9 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -89,11 +89,11 @@ def repo_add(args): # check if the path exists if not os.path.exists(canon_path): - tty.die("No such file or directory: '%s'." % path) + tty.die("No such file or directory: %s" % path) # Make sure the path is a directory. if not os.path.isdir(canon_path): - tty.die("Not a Spack repository: '%s'." % path) + tty.die("Not a Spack repository: %s" % path) # Make sure it's actually a spack repository by constructing it. repo = Repo(canon_path) @@ -103,7 +103,7 @@ def repo_add(args): if not repos: repos = [] if repo.root in repos or path in repos: - tty.die("Repository is already registered with Spack: '%s'" % path) + tty.die("Repository is already registered with Spack: %s" % path) repos.insert(0, canon_path) spack.config.update_config('repos', repos, args.scope) @@ -122,7 +122,7 @@ def repo_remove(args): if canon_path == repo_canon_path: repos.remove(repo_path) spack.config.update_config('repos', repos, args.scope) - tty.msg("Removed repository '%s'." % repo_path) + tty.msg("Removed repository %s" % repo_path) return # If it is a namespace, remove corresponding repo @@ -132,13 +132,13 @@ def repo_remove(args): if repo.namespace == path_or_namespace: repos.remove(path) spack.config.update_config('repos', repos, args.scope) - tty.msg("Removed repository '%s' with namespace %s." + tty.msg("Removed repository %s with namespace '%s'." % (repo.root, repo.namespace)) return except RepoError as e: continue - tty.die("No repository with path or namespace: '%s'" + tty.die("No repository with path or namespace: %s" % path_or_namespace) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 6fecde99801..576a5afa2ee 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -205,7 +205,7 @@ def validate_section_name(section): """Raise a ValueError if the section is not a valid section.""" if section not in section_schemas: - raise ValueError("Invalid config section: '%s'. Options are %s." + raise ValueError("Invalid config section: '%s'. Options are %s" % (section, section_schemas)) @@ -335,7 +335,7 @@ def validate_scope(scope): return config_scopes[scope] else: - raise ValueError("Invalid config scope: '%s'. Must be one of %s." + raise ValueError("Invalid config scope: '%s'. Must be one of %s" % (scope, config_scopes.keys())) @@ -350,7 +350,7 @@ def _read_config_file(filename, schema): "Invlaid configuration. %s exists but is not a file." % filename) elif not os.access(filename, os.R_OK): - raise ConfigFileError("Config file is not readable: %s." % filename) + raise ConfigFileError("Config file is not readable: %s" % filename) try: tty.debug("Reading config file %s" % filename) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 9cbe7de44a3..089d29325e6 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -330,7 +330,7 @@ def _check_ref_counts(self): found = rec.ref_count if not expected == found: raise AssertionError( - "Invalid ref_count: %s: %d (expected %d), in DB %s." + "Invalid ref_count: %s: %d (expected %d), in DB %s" % (key, found, expected, self._index_path)) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index c8542f55f02..61cd3030124 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -125,7 +125,7 @@ def __init__(self, dicts=None): dicts = (dicts,) elif type(dicts) not in (list, tuple): raise TypeError( - "dicts arg must be list, tuple, or string. Found %s." + "dicts arg must be list, tuple, or string. Found %s" % type(dicts)) self.dicts = dicts @@ -317,5 +317,5 @@ class CircularReferenceError(DirectiveError): def __init__(self, directive, package): super(CircularReferenceError, self).__init__( directive, - "Package '%s' cannot pass itself to %s." % (package, directive)) + "Package '%s' cannot pass itself to %s" % (package, directive)) self.package = package diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 29d87b65b33..08c23627f46 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -335,7 +335,7 @@ def _extension_map(self, spec): if not dag_hash in by_hash: raise InvalidExtensionSpecError( - "Spec %s not found in %s." % (dag_hash, prefix)) + "Spec %s not found in %s" % (dag_hash, prefix)) ext_spec = by_hash[dag_hash] if not prefix == ext_spec.prefix: @@ -450,7 +450,7 @@ class ExtensionConflictError(DirectoryLayoutError): """Raised when an extension is added to a package that already has it.""" def __init__(self, spec, ext_spec, conflict): super(ExtensionConflictError, self).__init__( - "%s cannot be installed in %s because it conflicts with %s."% ( + "%s cannot be installed in %s because it conflicts with %s"% ( ext_spec.short_spec, spec.short_spec, conflict.short_spec)) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 83a2dbb59c5..ec17cb97f14 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -153,7 +153,7 @@ def fetch(self): self.stage.chdir() if self.archive_file: - tty.msg("Already downloaded %s." % self.archive_file) + tty.msg("Already downloaded %s" % self.archive_file) return tty.msg("Trying to fetch from %s" % self.url) @@ -275,8 +275,8 @@ def check(self): checker = crypto.Checker(self.digest) if not checker.check(self.archive_file): raise ChecksumError( - "%s checksum failed for %s." % (checker.hash_name, self.archive_file), - "Expected %s but got %s." % (self.digest, checker.sum)) + "%s checksum failed for %s" % (checker.hash_name, self.archive_file), + "Expected %s but got %s" % (self.digest, checker.sum)) @_needs_stage def reset(self): @@ -312,7 +312,7 @@ def __init__(self, name, *rev_types, **kwargs): # Ensure that there's only one of the rev_types if sum(k in kwargs for k in rev_types) > 1: raise FetchStrategyError( - "Supply only one of %s to fetch with %s." % ( + "Supply only one of %s to fetch with %s" % ( comma_or(rev_types), name)) # Set attributes for each rev type. @@ -321,7 +321,7 @@ def __init__(self, name, *rev_types, **kwargs): @_needs_stage def check(self): - tty.msg("No checksum needed when fetching with %s." % self.name) + tty.msg("No checksum needed when fetching with %s" % self.name) @_needs_stage def expand(self): @@ -395,7 +395,7 @@ def fetch(self): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.stage.source_path) + tty.msg("Already fetched %s" % self.stage.source_path) return args = [] @@ -505,7 +505,7 @@ def fetch(self): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.stage.source_path) + tty.msg("Already fetched %s" % self.stage.source_path) return tty.msg("Trying to check out svn repository: %s" % self.url) @@ -584,7 +584,7 @@ def fetch(self): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.stage.source_path) + tty.msg("Already fetched %s" % self.stage.source_path) return args = [] diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index fa29e208036..58e31c2c7b5 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -73,7 +73,7 @@ def get_matching_versions(specs, **kwargs): # Skip any package that has no known versions. if not pkg.versions: - tty.msg("No safe (checksummed) versions for package %s." % pkg.name) + tty.msg("No safe (checksummed) versions for package %s" % pkg.name) continue num_versions = kwargs.get('num_versions', 0) @@ -203,7 +203,7 @@ def create(path, specs, **kwargs): if spack.debug: sys.excepthook(*sys.exc_info()) else: - tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) + tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message) error.append(spec) finally: pkg.stage.destroy() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 9f1825ca211..fb96f61de9e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -688,7 +688,7 @@ def do_fetch(self, mirror_only=False): if not ignore_checksum: raise FetchError( - "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg) + "Will not fetch %s" % self.spec.format('$_$@'), checksum_msg) self.stage.fetch(mirror_only) @@ -722,7 +722,7 @@ def do_patch(self): # If there are no patches, note it. if not self.patches and not has_patch_fun: - tty.msg("No patches needed for %s." % self.name) + tty.msg("No patches needed for %s" % self.name) return # Construct paths to special files in the archive dir used to @@ -745,7 +745,7 @@ def do_patch(self): tty.msg("Already patched %s" % self.name) return elif os.path.isfile(no_patches_file): - tty.msg("No patches needed for %s." % self.name) + tty.msg("No patches needed for %s" % self.name) return # Apply all the patches for specs that match this one @@ -766,10 +766,10 @@ def do_patch(self): if has_patch_fun: try: self.patch() - tty.msg("Ran patch() for %s." % self.name) + tty.msg("Ran patch() for %s" % self.name) patched = True except: - tty.msg("patch() function failed for %s." % self.name) + tty.msg("patch() function failed for %s" % self.name) touch(bad_file) raise @@ -838,7 +838,7 @@ def do_install(self, raise ValueError("Can only install concrete packages.") if os.path.exists(self.prefix): - tty.msg("%s is already installed in %s." % (self.name, self.prefix)) + tty.msg("%s is already installed in %s" % (self.name, self.prefix)) return tty.msg("Installing %s" % self.name) @@ -874,7 +874,7 @@ def cleanup(): def real_work(): try: - tty.msg("Building %s." % self.name) + tty.msg("Building %s" % self.name) # Run the pre-install hook in the child process after # the directory is created. @@ -918,8 +918,8 @@ def real_work(): self._total_time = time.time() - start_time build_time = self._total_time - self._fetch_time - tty.msg("Successfully installed %s." % self.name, - "Fetch: %s. Build: %s. Total: %s." + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s" % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) print_pkg(self.prefix) @@ -1025,7 +1025,7 @@ def do_uninstall(self, force=False): # Uninstalling in Spack only requires removing the prefix. self.remove_prefix() spack.installed_db.remove(self.spec) - tty.msg("Successfully uninstalled %s." % self.spec.short_spec) + tty.msg("Successfully uninstalled %s" % self.spec.short_spec) # Once everything else is done, run post install hooks spack.hooks.post_uninstall(self) @@ -1072,7 +1072,7 @@ def do_activate(self, force=False): self.extendee_spec.package.activate(self, **self.extendee_args) spack.install_layout.add_extension(self.extendee_spec, self.spec) - tty.msg("Activated extension %s for %s." + tty.msg("Activated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1124,7 +1124,7 @@ def do_deactivate(self, **kwargs): if self.activated: spack.install_layout.remove_extension(self.extendee_spec, self.spec) - tty.msg("Deactivated extension %s for %s." + tty.msg("Deactivated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1320,7 +1320,7 @@ class PackageVersionError(PackageError): """Raised when a version URL cannot automatically be determined.""" def __init__(self, version): super(PackageVersionError, self).__init__( - "Cannot determine a URL automatically for version %s." % version, + "Cannot determine a URL automatically for version %s" % version, "Please provide a url for this version in the package.py file.") diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 8d06fefe7f0..3c3ba08bcc2 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -156,7 +156,7 @@ def _add(self, repo): if repo.namespace in self.by_namespace: raise DuplicateRepoError( - "Package repos '%s' and '%s' both provide namespace %s." + "Package repos '%s' and '%s' both provide namespace %s" % (repo.root, self.by_namespace[repo.namespace].root, repo.namespace)) # Add repo to the pkg indexes @@ -545,7 +545,7 @@ def get(self, spec, new=False): raise UnknownPackageError(spec.name) if spec.namespace and spec.namespace != self.namespace: - raise UnknownPackageError("Repository %s does not contain package %s." + raise UnknownPackageError("Repository %s does not contain package %s" % (self.namespace, spec.fullname)) key = hash(spec) @@ -825,7 +825,7 @@ class UnknownPackageError(PackageLoadError): def __init__(self, name, repo=None): msg = None if repo: - msg = "Package %s not found in repository %s." % (name, repo) + msg = "Package %s not found in repository %s" % (name, repo) else: msg = "Package %s not found." % name super(UnknownPackageError, self).__init__(msg) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index f217450d42d..5591cb9ba5a 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -306,9 +306,9 @@ def expand_archive(self): archive_dir = self.source_path if not archive_dir: self.fetcher.expand() - tty.msg("Created stage in %s." % self.path) + tty.msg("Created stage in %s" % self.path) else: - tty.msg("Already staged %s in %s." % (self.name, self.path)) + tty.msg("Already staged %s in %s" % (self.name, self.path)) def chdir_to_source(self): """Changes directory to the expanded archive directory. From 0eb6ef2cd031d1668e1891425bdc6234df0594df Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 4 Mar 2016 11:09:40 -0500 Subject: [PATCH 53/82] 1. Removed fake MPI dependency from netcdf-cxx4 and netcdf-fortran. 2. Removed Fortran bootstrap variant from netcdf. Users who need NetCDF Fortran interface should install netcdf-fortran. 3. Added result of ./configure --help on the netcdf-fortran. Verified the package has no additional options that should be exposed through Spack. --- .../builtin/packages/netcdf-cxx4/package.py | 6 - .../packages/netcdf-fortran/package.py | 156 +++++++++++++++++- .../repos/builtin/packages/netcdf/package.py | 12 +- 3 files changed, 154 insertions(+), 20 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index 8d51a106794..fb4c2886cd8 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -7,12 +7,6 @@ class NetcdfCxx4(Package): version('4.2', 'd019853802092cf686254aaba165fc81') - - variant('mpi', default=True, description='Enables MPI parallelism') - - # netcdf-cxx4 doesn't really depend (directly) on MPI. However... this - # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) - depends_on('mpi', when='+mpi') depends_on('netcdf') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 9e4aee95fb8..4e0b14d0121 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -10,12 +10,162 @@ class NetcdfFortran(Package): variant('mpi', default=True, description='Enables MPI parallelism') - # netcdf-fortran doesn't really depend (directly) on MPI. However... this - # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) - depends_on('mpi', when='+mpi') depends_on('netcdf') def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() make("install") + + + + +# netcdf-fortran configure parameters are below +# --------------------------------------------- +# +# `configure' configures netCDF-Fortran 4.4.3 to adapt to many kinds of systems. +# +# Usage: ./configure [OPTION]... [VAR=VALUE]... +# +# To assign environment variables (e.g., CC, CFLAGS...), specify them as +# VAR=VALUE. See below for descriptions of some of the useful variables. +# +# Defaults for the options are specified in brackets. +# +# Configuration: +# -h, --help display this help and exit +# --help=short display options specific to this package +# --help=recursive display the short help of all the included packages +# -V, --version display version information and exit +# -q, --quiet, --silent do not print `checking ...' messages +# --cache-file=FILE cache test results in FILE [disabled] +# -C, --config-cache alias for `--cache-file=config.cache' +# -n, --no-create do not create output files +# --srcdir=DIR find the sources in DIR [configure dir or `..'] +# +# Installation directories: +# --prefix=PREFIX install architecture-independent files in PREFIX +# [/usr/local] +# --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX +# [PREFIX] +# +# By default, `make install' will install all the files in +# `/usr/local/bin', `/usr/local/lib' etc. You can specify +# an installation prefix other than `/usr/local' using `--prefix', +# for instance `--prefix=$HOME'. +# +# For better control, use the options below. +# +# Fine tuning of the installation directories: +# --bindir=DIR user executables [EPREFIX/bin] +# --sbindir=DIR system admin executables [EPREFIX/sbin] +# --libexecdir=DIR program executables [EPREFIX/libexec] +# --sysconfdir=DIR read-only single-machine data [PREFIX/etc] +# --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] +# --localstatedir=DIR modifiable single-machine data [PREFIX/var] +# --libdir=DIR object code libraries [EPREFIX/lib] +# --includedir=DIR C header files [PREFIX/include] +# --oldincludedir=DIR C header files for non-gcc [/usr/include] +# --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] +# --datadir=DIR read-only architecture-independent data [DATAROOTDIR] +# --infodir=DIR info documentation [DATAROOTDIR/info] +# --localedir=DIR locale-dependent data [DATAROOTDIR/locale] +# --mandir=DIR man documentation [DATAROOTDIR/man] +# --docdir=DIR documentation root [DATAROOTDIR/doc/netcdf-fortran] +# --htmldir=DIR html documentation [DOCDIR] +# --dvidir=DIR dvi documentation [DOCDIR] +# --pdfdir=DIR pdf documentation [DOCDIR] +# --psdir=DIR ps documentation [DOCDIR] +# +# Program names: +# --program-prefix=PREFIX prepend PREFIX to installed program names +# --program-suffix=SUFFIX append SUFFIX to installed program names +# --program-transform-name=PROGRAM run sed PROGRAM on installed program names +# +# System types: +# --build=BUILD configure for building on BUILD [guessed] +# --host=HOST cross-compile to build programs to run on HOST [BUILD] +# --target=TARGET configure for building compilers for TARGET [HOST] +# +# Optional Features: +# --disable-option-checking ignore unrecognized --enable/--with options +# --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) +# --enable-FEATURE[=ARG] include FEATURE [ARG=yes] +# --enable-silent-rules less verbose build output (undo: "make V=1") +# --disable-silent-rules verbose build output (undo: "make V=0") +# --enable-maintainer-mode +# enable make rules and dependencies not useful (and +# sometimes confusing) to the casual installer +# --enable-valgrind-tests build with valgrind-tests (valgrind is required, +# static builds only) +# --enable-parallel-tests Run extra parallel IO tests. Ignored if netCDF-4 is +# not enabled, or built on a system without parallel +# I/O support. +# --enable-extra-tests run some extra tests that may not pass because of +# known issues +# --enable-doxygen Enable generation of documentation with doxygen. +# --enable-dot Use dot (provided by graphviz) to generate charts +# and graphs in the doxygen-based documentation. +# --enable-internal-docs Include documentation of library internals. This is +# of interest only to those developing the netCDF +# library. +# --enable-dependency-tracking +# do not reject slow dependency extractors +# --disable-dependency-tracking +# speeds up one-time build +# --disable-f03-compiler-check +# disable check of ISO_C_BINDING support in Fortran +# compiler +# --disable-f03 suppress netCDF Fortran 2003 native code +# --disable-fortran-type-check +# cause the Fortran type sizes checks to be skipped +# --enable-large-file-tests +# Run tests which create very large data files (~13 GB +# disk space required, but it will be recovered when +# tests are complete). See option --with-temp-large to +# specify temporary directory +# --enable-benchmarks Run benchmarks. This is an experimental feature. +# --enable-shared[=PKGS] build shared libraries [default=yes] +# --enable-static[=PKGS] build static libraries [default=yes] +# --enable-fast-install[=PKGS] +# optimize for fast installation [default=yes] +# --disable-libtool-lock avoid locking (might break parallel builds) +# --disable-largefile omit support for large files +# --enable-extra-example-tests +# Run extra example tests; requires GNU sed. Ignored +# if netCDF-4 is not enabled. +# --enable-dll build a win32 DLL (only works on mingw) +# +# Optional Packages: +# --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] +# --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) +# --with-temp-large= +# specify directory where large files (i.e. >2 GB) +# will be written, if large files tests are run with +# --enable-large-file-tests +# --with-pic[=PKGS] try to use only PIC/non-PIC objects [default=use +# both] +# --with-gnu-ld assume the C compiler uses GNU ld [default=no] +# --with-sysroot=DIR Search for dependent libraries within DIR +# (or the compiler's sysroot if not specified). +# +# Some influential environment variables: +# CC C compiler command +# CFLAGS C compiler flags +# LDFLAGS linker flags, e.g. -L if you have libraries in a +# nonstandard directory +# LIBS libraries to pass to the linker, e.g. -l +# CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if +# you have headers in a nonstandard directory +# FC Fortran compiler command +# FCFLAGS Fortran compiler flags +# F77 Fortran 77 compiler command +# FFLAGS Fortran 77 compiler flags +# CPP C preprocessor +# +# Use these variables to override the choices made by `configure' or to help +# it to find libraries and programs with nonstandard names/locations. +# +# Report bugs to . +# +# from spack import * diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 41a0d2b6f92..0b112a59ce9 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -13,7 +13,6 @@ class Netcdf(Package): version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') variant('mpi', default=True, description='Enables MPI parallelism') - variant('fortran', default=False, description="Download and install NetCDF-Fortran") variant('hdf4', default=False, description="Enable HDF4 support") # Dependencies: @@ -66,11 +65,7 @@ def install(self, spec, prefix): # Fortran support # In version 4.2+, NetCDF-C and NetCDF-Fortran have split. - # They can be installed separately, but this bootstrap procedure - # should be able to install both at the same time. - # Note: this is a new experimental feature. - if '+fortran' in spec: - config_args.append("--enable-remote-fortran-bootstrap") + # Use the netcdf-fortran package to install Fortran support. config_args.append('CPPFLAGS=%s' % ' '.join(CPPFLAGS)) config_args.append('LDFLAGS=%s' % ' '.join(LDFLAGS)) @@ -79,8 +74,3 @@ def install(self, spec, prefix): configure(*config_args) make() make("install") - - # After installing NetCDF-C, install NetCDF-Fortran - if '+fortran' in spec: - make("build-netcdf-fortran") - make("install-netcdf-fortran") From 5db967390450659f794d202bfb5c4b4f11fb964b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 4 Mar 2016 10:51:41 -0600 Subject: [PATCH 54/82] Typo fix in compiler docstring --- lib/spack/spack/compiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 12c02e0ea2b..d38c0b00b12 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -256,12 +256,12 @@ def find(cls, *path): def __repr__(self): - """Return a string represntation of the compiler toolchain.""" + """Return a string representation of the compiler toolchain.""" return self.__str__() def __str__(self): - """Return a string represntation of the compiler toolchain.""" + """Return a string representation of the compiler toolchain.""" return "%s(%s)" % ( self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) From b043c4a5b83f36ad4837fc7302f0919a3d759940 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Fri, 4 Mar 2016 12:04:41 -0500 Subject: [PATCH 55/82] Update package.py Removed comments on configure options. --- .../packages/netcdf-fortran/package.py | 153 ------------------ 1 file changed, 153 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 4e0b14d0121..954e7dc3e81 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -16,156 +16,3 @@ def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() make("install") - - - - -# netcdf-fortran configure parameters are below -# --------------------------------------------- -# -# `configure' configures netCDF-Fortran 4.4.3 to adapt to many kinds of systems. -# -# Usage: ./configure [OPTION]... [VAR=VALUE]... -# -# To assign environment variables (e.g., CC, CFLAGS...), specify them as -# VAR=VALUE. See below for descriptions of some of the useful variables. -# -# Defaults for the options are specified in brackets. -# -# Configuration: -# -h, --help display this help and exit -# --help=short display options specific to this package -# --help=recursive display the short help of all the included packages -# -V, --version display version information and exit -# -q, --quiet, --silent do not print `checking ...' messages -# --cache-file=FILE cache test results in FILE [disabled] -# -C, --config-cache alias for `--cache-file=config.cache' -# -n, --no-create do not create output files -# --srcdir=DIR find the sources in DIR [configure dir or `..'] -# -# Installation directories: -# --prefix=PREFIX install architecture-independent files in PREFIX -# [/usr/local] -# --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX -# [PREFIX] -# -# By default, `make install' will install all the files in -# `/usr/local/bin', `/usr/local/lib' etc. You can specify -# an installation prefix other than `/usr/local' using `--prefix', -# for instance `--prefix=$HOME'. -# -# For better control, use the options below. -# -# Fine tuning of the installation directories: -# --bindir=DIR user executables [EPREFIX/bin] -# --sbindir=DIR system admin executables [EPREFIX/sbin] -# --libexecdir=DIR program executables [EPREFIX/libexec] -# --sysconfdir=DIR read-only single-machine data [PREFIX/etc] -# --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] -# --localstatedir=DIR modifiable single-machine data [PREFIX/var] -# --libdir=DIR object code libraries [EPREFIX/lib] -# --includedir=DIR C header files [PREFIX/include] -# --oldincludedir=DIR C header files for non-gcc [/usr/include] -# --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] -# --datadir=DIR read-only architecture-independent data [DATAROOTDIR] -# --infodir=DIR info documentation [DATAROOTDIR/info] -# --localedir=DIR locale-dependent data [DATAROOTDIR/locale] -# --mandir=DIR man documentation [DATAROOTDIR/man] -# --docdir=DIR documentation root [DATAROOTDIR/doc/netcdf-fortran] -# --htmldir=DIR html documentation [DOCDIR] -# --dvidir=DIR dvi documentation [DOCDIR] -# --pdfdir=DIR pdf documentation [DOCDIR] -# --psdir=DIR ps documentation [DOCDIR] -# -# Program names: -# --program-prefix=PREFIX prepend PREFIX to installed program names -# --program-suffix=SUFFIX append SUFFIX to installed program names -# --program-transform-name=PROGRAM run sed PROGRAM on installed program names -# -# System types: -# --build=BUILD configure for building on BUILD [guessed] -# --host=HOST cross-compile to build programs to run on HOST [BUILD] -# --target=TARGET configure for building compilers for TARGET [HOST] -# -# Optional Features: -# --disable-option-checking ignore unrecognized --enable/--with options -# --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) -# --enable-FEATURE[=ARG] include FEATURE [ARG=yes] -# --enable-silent-rules less verbose build output (undo: "make V=1") -# --disable-silent-rules verbose build output (undo: "make V=0") -# --enable-maintainer-mode -# enable make rules and dependencies not useful (and -# sometimes confusing) to the casual installer -# --enable-valgrind-tests build with valgrind-tests (valgrind is required, -# static builds only) -# --enable-parallel-tests Run extra parallel IO tests. Ignored if netCDF-4 is -# not enabled, or built on a system without parallel -# I/O support. -# --enable-extra-tests run some extra tests that may not pass because of -# known issues -# --enable-doxygen Enable generation of documentation with doxygen. -# --enable-dot Use dot (provided by graphviz) to generate charts -# and graphs in the doxygen-based documentation. -# --enable-internal-docs Include documentation of library internals. This is -# of interest only to those developing the netCDF -# library. -# --enable-dependency-tracking -# do not reject slow dependency extractors -# --disable-dependency-tracking -# speeds up one-time build -# --disable-f03-compiler-check -# disable check of ISO_C_BINDING support in Fortran -# compiler -# --disable-f03 suppress netCDF Fortran 2003 native code -# --disable-fortran-type-check -# cause the Fortran type sizes checks to be skipped -# --enable-large-file-tests -# Run tests which create very large data files (~13 GB -# disk space required, but it will be recovered when -# tests are complete). See option --with-temp-large to -# specify temporary directory -# --enable-benchmarks Run benchmarks. This is an experimental feature. -# --enable-shared[=PKGS] build shared libraries [default=yes] -# --enable-static[=PKGS] build static libraries [default=yes] -# --enable-fast-install[=PKGS] -# optimize for fast installation [default=yes] -# --disable-libtool-lock avoid locking (might break parallel builds) -# --disable-largefile omit support for large files -# --enable-extra-example-tests -# Run extra example tests; requires GNU sed. Ignored -# if netCDF-4 is not enabled. -# --enable-dll build a win32 DLL (only works on mingw) -# -# Optional Packages: -# --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] -# --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) -# --with-temp-large= -# specify directory where large files (i.e. >2 GB) -# will be written, if large files tests are run with -# --enable-large-file-tests -# --with-pic[=PKGS] try to use only PIC/non-PIC objects [default=use -# both] -# --with-gnu-ld assume the C compiler uses GNU ld [default=no] -# --with-sysroot=DIR Search for dependent libraries within DIR -# (or the compiler's sysroot if not specified). -# -# Some influential environment variables: -# CC C compiler command -# CFLAGS C compiler flags -# LDFLAGS linker flags, e.g. -L if you have libraries in a -# nonstandard directory -# LIBS libraries to pass to the linker, e.g. -l -# CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if -# you have headers in a nonstandard directory -# FC Fortran compiler command -# FCFLAGS Fortran compiler flags -# F77 Fortran 77 compiler command -# FFLAGS Fortran 77 compiler flags -# CPP C preprocessor -# -# Use these variables to override the choices made by `configure' or to help -# it to find libraries and programs with nonstandard names/locations. -# -# Report bugs to . -# -# from spack import * From 648d08eb305e04d95ee702b819923f31fd829955 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Fri, 4 Mar 2016 12:05:43 -0500 Subject: [PATCH 56/82] Update package.py Updated homepage URL to general NetCDF homepage. --- var/spack/repos/builtin/packages/netcdf-cxx4/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index fb4c2886cd8..ab717ac6ffb 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -2,7 +2,7 @@ class NetcdfCxx4(Package): """C++ interface for NetCDF4""" - homepage = "http://www.unidata.ucar.edu/downloads/netcdf/netcdf-cxx/index.jsp" + homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz" version('4.2', 'd019853802092cf686254aaba165fc81') From a7b918837eab02ebff5af783d06e56ec7f868d9e Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 4 Mar 2016 15:22:28 -0600 Subject: [PATCH 57/82] GNU m4 depends on libsigsegv --- .../repos/builtin/packages/libsigsegv/package.py | 15 +++++++++++++++ var/spack/repos/builtin/packages/m4/package.py | 2 ++ 2 files changed, 17 insertions(+) create mode 100644 var/spack/repos/builtin/packages/libsigsegv/package.py diff --git a/var/spack/repos/builtin/packages/libsigsegv/package.py b/var/spack/repos/builtin/packages/libsigsegv/package.py new file mode 100644 index 00000000000..4b486198ec6 --- /dev/null +++ b/var/spack/repos/builtin/packages/libsigsegv/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Libsigsegv(Package): + """GNU libsigsegv is a library for handling page faults in user mode.""" + homepage = "https://www.gnu.org/software/libsigsegv/" + url = "ftp://ftp.gnu.org/gnu/libsigsegv/libsigsegv-2.10.tar.gz" + + version('2.10', '7f96fb1f65b3b8cbc1582fb7be774f0f') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--enable-shared') + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index 5d76d8866bc..3890663ad12 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -7,6 +7,8 @@ class M4(Package): version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') + depends_on('libsigsegv') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() From f90eaa5f46bfd64b9b0adb4ab5a58d169efe3c69 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Fri, 4 Mar 2016 14:53:08 -0800 Subject: [PATCH 58/82] Fixed unmatched function signature for do_fetch in jdk package --- var/spack/repos/builtin/packages/jdk/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/jdk/package.py b/var/spack/repos/builtin/packages/jdk/package.py index f8f5fc21bdd..cbcc53ac0a7 100644 --- a/var/spack/repos/builtin/packages/jdk/package.py +++ b/var/spack/repos/builtin/packages/jdk/package.py @@ -28,7 +28,7 @@ class Jdk(Package): '-H', # specify required License Agreement cookie 'Cookie: oraclelicense=accept-securebackup-cookie'] - def do_fetch(self): + def do_fetch(self, mirror_only=False): # Add our custom curl commandline options tty.msg( "[Jdk] Adding required commandline options to curl " + @@ -39,7 +39,7 @@ def do_fetch(self): spack.curl.add_default_arg(option) # Now perform the actual fetch - super(Jdk, self).do_fetch() + super(Jdk, self).do_fetch(mirror_only) def install(self, spec, prefix): From f663d37da75e4ef0f266c63707e77d29a651e07d Mon Sep 17 00:00:00 2001 From: Mark Miller Date: Fri, 4 Mar 2016 17:08:11 -0800 Subject: [PATCH 59/82] Adding zfp package. --- .../repos/builtin/packages/zfp/package.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 var/spack/repos/builtin/packages/zfp/package.py diff --git a/var/spack/repos/builtin/packages/zfp/package.py b/var/spack/repos/builtin/packages/zfp/package.py new file mode 100644 index 00000000000..620fe9d4565 --- /dev/null +++ b/var/spack/repos/builtin/packages/zfp/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Zfp(Package): + """zfp is an open source C library for compressed floating-point arrays that supports + very high throughput read and write random acces, target error bounds or bit rates. + Although bit-for-bit lossless compression is not always possible, zfp is usually + accurate to within machine epsilon in near-lossless mode, and is often orders of + magnitude more accurate than other lossy compressors. + """ + + homepage = "http://computation.llnl.gov/projects/floating-point-compression" + url = "http://computation.llnl.gov/projects/floating-point-compression/download/zfp-0.5.0.tar.gz" + + version('0.5.0', '2ab29a852e65ad85aae38925c5003654') + + def install(self, spec, prefix): + make("shared") + + # No install provided + mkdirp(prefix.lib) + mkdirp(prefix.include) + install('lib/libzfp.so', prefix.lib) + install('inc/zfp.h', prefix.include) + install('inc/types.h', prefix.include) + install('inc/bitstream.h', prefix.include) + install('inc/system.h', prefix.include) From fc6aa7374a243928af38fd1e583c6c67146951cd Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 04:18:48 -0800 Subject: [PATCH 60/82] Fix #104, #54: issues with overlong shebang in deep directories. This does several things: - Add `sbang`: a script to run scripts with long shebang lines. - Documentation for `sbang` is in `bin/sbang`. - Add an `sbang` hook that filters the `bin` directory after install and modifies any scripts wtih shebangs that are too long to use `sbang` instead. - `sbang` is at the top level, so it should be runnable (not much we can do if spack itself is too deep for shebang) - `sbang`, when used as the interpreter, runs the *second* shebang line it finds in a script. - shoud fix issues with too long shebang paths. --- bin/sbang | 84 +++++++++++++++++++++++++++++++ lib/spack/llnl/util/filesystem.py | 11 +++- lib/spack/spack/hooks/sbang.py | 74 +++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 1 deletion(-) create mode 100755 bin/sbang create mode 100644 lib/spack/spack/hooks/sbang.py diff --git a/bin/sbang b/bin/sbang new file mode 100755 index 00000000000..ebfbe2e7a16 --- /dev/null +++ b/bin/sbang @@ -0,0 +1,84 @@ +#!/bin/bash +# +# `sbang`: Run scripts with long shebang lines. +# +# Many operating systems limit the length of shebang lines, making it +# hard to use interpreters that are deep in the directory hierarchy. +# `sbang` can run such scripts, either as a shebang interpreter, or +# directly on the command line. +# +# Usage +# ----------------------------- +# Suppose you have a script, long-shebang.sh, like this: +# +# 1 #!/very/long/path/to/some/interpreter +# 2 +# 3 echo "success!" +# +# Invoking this script will result in an error on some OS's. On +# Linux, you get this: +# +# $ ./long-shebang.sh +# -bash: ./long: /very/long/path/to/some/interp: bad interpreter: +# No such file or directory +# +# On Mac OS X, the system simply assumes the interpreter is the shell +# and tries to run with it, which is likely not what you want. +# +# +# `sbang` on the command line +# ----------------------------- +# You can use `sbang` in two ways. The first is to use it directly, +# from the command line, like this: +# +# $ sbang ./long-shebang.sh +# success! +# +# +# `sbang` as the interpreter +# ----------------------------- +# You can also use `sbang` *as* the interpreter for your script. Put +# `#!/bin/bash /path/to/sbang` on line 1, and move the original +# shebang to line 2 of the script: +# +# 1 #!/bin/bash /path/to/sbang +# 2 #!/long/path/to/real/interpreter with arguments +# 3 +# 4 echo "success!" +# +# $ ./long-shebang.sh +# success! +# +# On Linux, you could shorten line 1 to `#!/path/to/sbang`, but other +# operating systems like Mac OS X require the interpreter to be a +# binary, so it's best to use `sbang` as a `bash` argument. +# Obviously, for this to work, `sbang` needs to have a short enough +# path that *it* will run without hitting OS limits. +# +# +# How it works +# ----------------------------- +# `sbang` is a very simple bash script. It looks at the first two +# lines of a script argument and runs the last line starting with +# `#!`, with the script as an argument. It also forwards arguments. +# + +# First argument is the script we want to actually run. +script="$1" + +# Search the first two lines of script for interpreters. +lines=0 +while read line && ((lines < 2)) ; do + if [[ "$line" = '#!'* ]]; then + interpreter="${line#\#!}" + fi + lines=$((lines+1)) +done < "$script" + +# Invoke any interpreter found, or raise an error if none was found. +if [ -n "$interpreter" ]; then + exec $interpreter "$@" +else + echo "error: sbang found no interpreter in $script" + exit 1 +fi diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index da3cf96050f..10d25bdce8d 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,8 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', + 'copy_mode', 'unset_executable_mode'] import os import sys @@ -158,6 +159,14 @@ def copy_mode(src, dest): os.chmod(dest, dest_mode) +def unset_executable_mode(path): + mode = os.stat(path).st_mode + mode &= ~stat.S_IXUSR + mode &= ~stat.S_IXGRP + mode &= ~stat.S_IXOTH + os.chmod(path, mode) + + def install(src, dest): """Manually install a file to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py new file mode 100644 index 00000000000..6117c4809dc --- /dev/null +++ b/lib/spack/spack/hooks/sbang.py @@ -0,0 +1,74 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os + +from llnl.util.filesystem import * +import llnl.util.tty as tty + +import spack +import spack.modules + +# Character limit for shebang line. Using Linux's 127 characters +# here, as it is the shortest I could find on a modern OS. +shebang_limit = 127 + +def shebang_too_long(path): + """Detects whether an file has a shebang line that is too long.""" + with open(path, 'r') as script: + bytes = script.read(2) + if bytes != '#!': + return False + + line = bytes + script.readline() + return len(line) > shebang_limit + + +def filter_shebang(path): + """Adds a second shebang line, using sbang, at the beginning of a file.""" + backup = path + ".shebang.bak" + os.rename(path, backup) + + with open(backup, 'r') as bak_file: + original = bak_file.read() + + with open(path, 'w') as new_file: + new_file.write('#!/bin/bash %s/bin/sbang\n' % spack.spack_root) + new_file.write(original) + + copy_mode(backup, path) + unset_executable_mode(backup) + + tty.warn("Patched overly long shebang in %s" % path) + + +def post_install(pkg): + """This hook edits scripts so that they call /bin/bash + $spack_prefix/bin/sbang instead of something longer than the + shebang limit.""" + for file in os.listdir(pkg.prefix.bin): + path = os.path.join(pkg.prefix.bin, file) + if shebang_too_long(path): + filter_shebang(path) + From bfce2c7508ada675ebe4398395b30ede82b67093 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 04:42:52 -0800 Subject: [PATCH 61/82] Fix bug in hook: ensure bin directory exists before listing. --- lib/spack/spack/hooks/sbang.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 6117c4809dc..3390ecea295 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -67,6 +67,9 @@ def post_install(pkg): """This hook edits scripts so that they call /bin/bash $spack_prefix/bin/sbang instead of something longer than the shebang limit.""" + if not os.path.isdir(pkg.prefix.bin): + return + for file in os.listdir(pkg.prefix.bin): path = os.path.join(pkg.prefix.bin, file) if shebang_too_long(path): From 3dd630d0a567964acd11bee23ebbc52c7d1c61c5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 14:33:23 -0800 Subject: [PATCH 62/82] Make openssl a variant in libevent. --- var/spack/repos/builtin/packages/libevent/package.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/libevent/package.py b/var/spack/repos/builtin/packages/libevent/package.py index 2a44c49325d..714a155dc00 100644 --- a/var/spack/repos/builtin/packages/libevent/package.py +++ b/var/spack/repos/builtin/packages/libevent/package.py @@ -22,12 +22,16 @@ class Libevent(Package): version('2.0.13', 'af786b4b3f790c9d3279792edf7867fc') version('2.0.12', '42986228baf95e325778ed328a93e070') - - depends_on('openssl') - + variant('openssl', default=True, description="Build with encryption enabled at the libevent level.") + depends_on('openssl', when='+openssl') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + configure_args = [] + if '+openssl' in spec: + configure_args.append('--enable-openssl') + else: + configure_args.append('--enable-openssl') + configure("--prefix=%s" % prefix, *configure_args) make() make("install") From 220b6a9fee0f307d4de1e48b29093812f7dd10ec Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Sat, 5 Mar 2016 16:46:32 -0600 Subject: [PATCH 63/82] Make libsigsegv an optional dependency --- var/spack/repos/builtin/packages/m4/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index 3890663ad12..9d522dfccf3 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -7,7 +7,9 @@ class M4(Package): version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') - depends_on('libsigsegv') + variant('sigsegv', default=True, description="Build the libsigsegv dependency") + + depends_on('libsigsegv', when='+sigsegv') def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 7d847f4dc4f8895d57e492c11cfd7c3fbb794945 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 15:14:21 -0800 Subject: [PATCH 64/82] Minor tweaks to m4 -- be sure to add sigsegv args explicitly. --- var/spack/repos/builtin/packages/m4/package.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index 9d522dfccf3..d6829dbcd4b 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -12,6 +12,12 @@ class M4(Package): depends_on('libsigsegv', when='+sigsegv') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + configure_args = [] + if 'libsigsegv' in spec: + configure_args.append('--with-libsigsegv-prefix=%s' % spec['libsigsegv'].prefix) + else: + configure_args.append('--without-libsigsegv-prefix') + + configure("--prefix=%s" % prefix, *configure_args) make() make("install") From 14d48eba461596b3c5b03cfe269318946406e02c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 18:40:28 -0800 Subject: [PATCH 65/82] Revert c5d9ee8924 for bug fixes. - This reverts commit c5d9ee89246b3d2aeddb756a04588424051d3295. - merged too soon before - reverting and fixing bugs now. Conflicts: lib/spack/spack/mirror.py lib/spack/spack/package.py --- lib/spack/llnl/util/filesystem.py | 35 +++- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/mirror.py | 55 +++--- lib/spack/spack/package.py | 175 +++++++++---------- lib/spack/spack/stage.py | 186 ++++++++++----------- lib/spack/spack/test/concretize.py | 2 - lib/spack/spack/test/config.py | 6 +- lib/spack/spack/test/configure_guess.py | 21 +-- lib/spack/spack/test/database.py | 11 +- lib/spack/spack/test/directory_layout.py | 13 +- lib/spack/spack/test/git_fetch.py | 46 ++--- lib/spack/spack/test/hg_fetch.py | 44 +++-- lib/spack/spack/test/install.py | 9 +- lib/spack/spack/test/link_tree.py | 7 +- lib/spack/spack/test/lock.py | 8 +- lib/spack/spack/test/make_executable.py | 6 +- lib/spack/spack/test/mirror.py | 69 ++++---- lib/spack/spack/test/mock_packages_test.py | 8 +- lib/spack/spack/test/mock_repo.py | 4 - lib/spack/spack/test/multimethod.py | 5 +- lib/spack/spack/test/namespace_trie.py | 1 + lib/spack/spack/test/optional_deps.py | 4 +- lib/spack/spack/test/packages.py | 6 +- lib/spack/spack/test/python_version.py | 3 +- lib/spack/spack/test/spec_dag.py | 2 - lib/spack/spack/test/spec_semantics.py | 1 - lib/spack/spack/test/spec_syntax.py | 3 +- lib/spack/spack/test/stage.py | 130 ++++++-------- lib/spack/spack/test/svn_fetch.py | 49 +++--- lib/spack/spack/test/tally_plugin.py | 4 +- lib/spack/spack/test/unit_install.py | 3 +- lib/spack/spack/test/url_extrapolate.py | 3 - lib/spack/spack/test/url_parse.py | 2 +- lib/spack/spack/test/url_substitution.py | 1 - lib/spack/spack/test/versions.py | 1 + lib/spack/spack/test/yaml.py | 1 + 36 files changed, 422 insertions(+), 504 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index da3cf96050f..366237ef8f3 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree'] import os import sys @@ -240,7 +240,7 @@ def touchp(path): def force_symlink(src, dest): try: os.symlink(src, dest) - except OSError, e: + except OSError as e: os.remove(dest) os.symlink(src, dest) @@ -344,3 +344,34 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) + +def remove_dead_links(root): + """ + Removes any dead link that is present in root + + Args: + root: path where to search for dead links + + """ + for file in os.listdir(root): + path = join_path(root, file) + if os.path.islink(path): + real_path = os.path.realpath(path) + if not os.path.exists(real_path): + os.unlink(path) + +def remove_linked_tree(path): + """ + Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real + directory before removing the link. + + Args: + path: directory to be removed + + """ + if os.path.exists(path): + if os.path.islink(path): + shutil.rmtree(os.path.realpath(path), True) + os.unlink(path) + else: + shutil.rmtree(path, True) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 6e7179122c1..0c8bd1d5281 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -43,4 +43,4 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.do_clean() + package.stage.destroy() diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 58e31c2c7b5..fdc4e7967f5 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -110,7 +110,6 @@ def suggest_archive_basename(resource): return basename - def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -158,17 +157,29 @@ def create(path, specs, **kwargs): "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. - present = [] - mirrored = [] - error = [] + categories = { + 'present': [], + 'mirrored': [], + 'error': [] + } # Iterate through packages and download all the safe tarballs for each of them - everything_already_exists = True for spec in version_specs: - pkg = spec.package - tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) - try: - for ii, stage in enumerate(pkg.stage): + add_single_spec(spec, mirror_root, categories, **kwargs) + + return categories['present'], categories['mirrored'], categories['error'] + + +def add_single_spec(spec, mirror_root, categories, **kwargs): + tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) + spec_exists_in_mirror = True + try: + with spec.package.stage: + # fetcher = stage.fetcher + # fetcher.fetch() + # ... + # fetcher.archive(archive_path) + for ii, stage in enumerate(spec.package.stage): fetcher = stage.fetcher if ii == 0: # create a subdirectory for the current package@version @@ -184,7 +195,7 @@ def create(path, specs, **kwargs): if os.path.exists(archive_path): tty.msg("{name} : already added".format(name=name)) else: - everything_already_exists = False + spec_exists_in_mirror = False fetcher.fetch() if not kwargs.get('no_checksum', False): fetcher.check() @@ -195,20 +206,16 @@ def create(path, specs, **kwargs): fetcher.archive(archive_path) tty.msg("{name} : added".format(name=name)) - if everything_already_exists: - present.append(spec) - else: - mirrored.append(spec) - except Exception, e: - if spack.debug: - sys.excepthook(*sys.exc_info()) - else: - tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message) - error.append(spec) - finally: - pkg.stage.destroy() - - return (present, mirrored, error) + if spec_exists_in_mirror: + categories['present'].append(spec) + else: + categories['mirrored'].append(spec) + except Exception as e: + if spack.debug: + sys.excepthook(*sys.exc_info()) + else: + tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message) + categories['error'].append(spec) class MirrorError(spack.error.SpackError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fb96f61de9e..be45415b759 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -293,7 +293,6 @@ class SomePackage(Package): .. code-block:: python - p.do_clean() # removes the stage directory entirely p.do_restage() # removes the build directory and # re-expands the archive. @@ -503,7 +502,6 @@ def fetcher(self): self._fetcher = self._make_fetcher() return self._fetcher - @fetcher.setter def fetcher(self, f): self._fetcher = f @@ -735,7 +733,7 @@ def do_patch(self): # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. if os.path.isfile(bad_file): - tty.msg("Patching failed last time. Restaging.") + tty.msg("Patching failed last time. Restaging.") self.stage.restage() self.stage.chdir_to_source() @@ -850,102 +848,102 @@ def do_install(self, make_jobs=make_jobs) start_time = time.time() - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() - - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) - - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) - - - def real_work(): - try: - tty.msg("Building %s" % self.name) - - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) - - # Set up process's build environment before running install. - if fake: - self.do_fake_install() + with self.stage: + if not fake: + if not skip_patch: + self.do_patch() else: - # Do the real install in the source directory. - self.stage.chdir_to_source() + self.do_stage() - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) - # This redirects I/O to a build log (and optionally to the terminal) - log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): - dump_environment(env_path) - self.install(self.spec, self.prefix) + def cleanup(): + if not keep_prefix: + # If anything goes wrong, remove the install prefix + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed." + + "Manually remove this directory to fix:", + self.prefix, wrap=True) - # Ensure that something was actually installed. - self._sanity_check_install() + def real_work(): + try: + tty.msg("Building %s" % self.name) - # Move build log into install directory on success - if not fake: - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - install(log_path, log_install_path) - install(env_path, env_install_path) + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) - packages_dir = spack.install_layout.build_packages_path(self.spec) - dump_packages(self.spec, packages_dir) + # Set up process's build environment before running install. + if fake: + self.do_fake_install() + else: + # Do the real install in the source directory. + self.stage.chdir_to_source() - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + # This redirects I/O to a build log (and optionally to the terminal) + log_path = join_path(os.getcwd(), 'spack-build.out') + log_file = open(log_path, 'w') + with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) + self.install(self.spec, self.prefix) - tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s" - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + # Ensure that something was actually installed. + self._sanity_check_install() - except ProcessError, e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e + # Move build log into install directory on success + if not fake: + log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) + install(log_path, log_install_path) + install(env_path, env_install_path) - except: - # other exceptions just clean up and raise. - cleanup() - raise + packages_dir = spack.install_layout.build_packages_path(self.spec) + dump_packages(self.spec, packages_dir) - # Set parallelism before starting build. - self.make_jobs = make_jobs + # On successful install, remove the stage. + if not keep_stage: + self.stage.destroy() - # Do the build. - spack.build_environment.fork(self, real_work) + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) + except ProcessError as e: + # Annotate with location of build log. + e.build_log = log_path + cleanup() + raise e + + except: + # other exceptions just clean up and raise. + cleanup() + raise + + # Set parallelism before starting build. + self.make_jobs = make_jobs + + # Do the build. + spack.build_environment.fork(self, real_work) + + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) + + # Once everything else is done, run post install hooks + spack.hooks.post_install(self) def _sanity_check_install(self): @@ -1149,13 +1147,6 @@ def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() - - def do_clean(self): - """Removes the package's build stage and source tarball.""" - if os.path.exists(self.stage.path): - self.stage.destroy() - - def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) @@ -1192,7 +1183,7 @@ def fetch_remote_versions(self): try: return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) - except spack.error.NoNetworkConnectionError, e: + except spack.error.NoNetworkConnectionError as e: tty.die("Package.fetch_versions couldn't connect to:", e.url, e.message) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 5591cb9ba5a..a22982a6d43 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -42,33 +42,26 @@ class Stage(object): - """A Stage object manages a directory where some source code is - downloaded and built before being installed. It handles - fetching the source code, either as an archive to be expanded - or by checking it out of a repository. A stage's lifecycle - looks like this: + """ + A Stage object is a context manager that handles a directory where some source code is downloaded and built + before being installed. It handles fetching the source code, either as an archive to be expanded or by checking + it out of a repository. A stage's lifecycle looks like this: - Stage() - Constructor creates the stage directory. - fetch() - Fetch a source archive into the stage. - expand_archive() - Expand the source archive. - - Build and install the archive. This is handled by the Package class. - destroy() - Remove the stage once the package has been installed. + ``` + with Stage() as stage: # Context manager creates and destroys the stage directory + fetch() # Fetch a source archive into the stage. + expand_archive() # Expand the source archive. + # Build and install the archive. This is handled by the Package class. + ``` - If spack.use_tmp_stage is True, spack will attempt to create stages - in a tmp directory. Otherwise, stages are created directly in - spack.stage_path. + If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. + Otherwise, stages are created directly in spack.stage_path. - There are two kinds of stages: named and unnamed. Named stages can - persist between runs of spack, e.g. if you fetched a tarball but - didn't finish building it, you won't have to fetch it again. + There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you + fetched a tarball but didn't finish building it, you won't have to fetch it again. - Unnamed stages are created using standard mkdtemp mechanisms or - similar, and are intended to persist for only one run of spack. + Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for + only one run of spack. """ def __init__(self, url_or_fetch_strategy, **kwargs): @@ -96,21 +89,46 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. - self.name = kwargs.get('name') + # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name + # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root + self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names()) self.mirror_path = kwargs.get('mirror_path') self.tmp_root = find_tmp_root() - self.path = None - self._setup() + # Try to construct here a temporary name for the stage directory + # If this is a named stage, then construct a named path. + self.path = join_path(spack.stage_path, self.name) + # Flag to decide whether to delete the stage folder on exit or not + self.delete_on_exit = True - def _cleanup_dead_links(self): - """Remove any dead links in the stage directory.""" - for file in os.listdir(spack.stage_path): - path = join_path(spack.stage_path, file) - if os.path.islink(path): - real_path = os.path.realpath(path) - if not os.path.exists(path): - os.unlink(path) + def __enter__(self): + """ + Entering a stage context will create the stage directory + + Returns: + self + """ + self.create() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Exiting from a stage context will delete the stage directory unless: + - it was explicitly requested not to do so + - an exception has been raised + + Args: + exc_type: exception type + exc_val: exception value + exc_tb: exception traceback + + Returns: + Boolean + """ + self.delete_on_exit = False if exc_type is not None else self.delete_on_exit + + if self.delete_on_exit: + self.destroy() def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we @@ -148,54 +166,6 @@ def _need_to_create_path(self): return False - def _setup(self): - """Creates the stage directory. - If spack.use_tmp_stage is False, the stage directory is created - directly under spack.stage_path. - - If spack.use_tmp_stage is True, this will attempt to create a - stage in a temporary directory and link it into spack.stage_path. - Spack will use the first writable location in spack.tmp_dirs to - create a stage. If there is no valid location in tmp_dirs, fall - back to making the stage inside spack.stage_path. - """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - self._cleanup_dead_links() - - # If this is a named stage, then construct a named path. - if self.name is not None: - self.path = join_path(spack.stage_path, self.name) - - # If this is a temporary stage, them make the temp directory - tmp_dir = None - if self.tmp_root: - if self.name is None: - # Unnamed tmp root. Link the path in - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - self.name = os.path.basename(tmp_dir) - self.path = join_path(spack.stage_path, self.name) - if self._need_to_create_path(): - os.symlink(tmp_dir, self.path) - - else: - if self._need_to_create_path(): - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - - # if we're not using a tmp dir, create the stage directly in the - # stage dir, rather than linking to it. - else: - if self.name is None: - self.path = tempfile.mkdtemp('', STAGE_PREFIX, spack.stage_path) - self.name = os.path.basename(self.path) - else: - if self._need_to_create_path(): - mkdirp(self.path) - - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -276,7 +246,7 @@ def fetch(self, mirror_only=False): self.fetcher = fetcher self.fetcher.fetch() break - except spack.error.SpackError, e: + except spack.error.SpackError as e: tty.msg("Fetching from %s failed." % fetcher) tty.debug(e) continue @@ -328,8 +298,34 @@ def restage(self): """ self.fetcher.reset() + def create(self): + """ + Creates the stage directory + + If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise + this will attempt to create a stage in a temporary directory and link it into spack.stage_path. + + Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location + in tmp_dirs, fall back to making the stage inside spack.stage_path. + """ + # Create the top-level stage directory + mkdirp(spack.stage_path) + remove_dead_links(spack.stage_path) + # If a tmp_root exists then create a directory there and then link it in the stage area, + # otherwise create the stage directory in self.path + if self._need_to_create_path(): + if self.tmp_root: + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + else: + mkdirp(self.path) + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) + def destroy(self): - """Remove this stage directory.""" + """ + Removes this stage directory + """ remove_linked_tree(self.path) # Make sure we don't end up in a removed directory @@ -389,6 +385,15 @@ def source_path(self): def path(self): return self[0].path + def __enter__(self): + for item in self: + item.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for item in reversed(self): + item.__exit__(exc_type, exc_val, exc_tb) + def chdir_to_source(self): return self[0].chdir_to_source() @@ -439,19 +444,6 @@ def ensure_access(file=spack.stage_path): tty.die("Insufficient permissions for %s" % file) -def remove_linked_tree(path): - """Removes a directory and its contents. If the directory is a symlink, - follows the link and reamoves the real directory before removing the - link. - """ - if os.path.exists(path): - if os.path.islink(path): - shutil.rmtree(os.path.realpath(path), True) - os.unlink(path) - else: - shutil.rmtree(path, True) - - def purge(): """Remove all build directories in the top-level stage path.""" if os.path.isdir(spack.stage_path): diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 7f2938aec5a..794344fb6ae 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -22,8 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - import spack from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index d8be5a855b8..0562d2d6209 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -22,13 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import shutil import os +import shutil from tempfile import mkdtemp -from ordereddict_backport import OrderedDict + import spack import spack.config +from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * # Some sample compiler config data diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index a4e8565b62b..2440d120e5d 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -23,20 +23,15 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest import shutil import tempfile +import unittest from llnl.util.filesystem import * - from spack.cmd.create import ConfigureGuesser from spack.stage import Stage - -from spack.fetch_strategy import URLFetchStrategy -from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which from spack.test.mock_packages_test import * -from spack.test.mock_repo import MockArchive +from spack.util.executable import which class InstallTest(unittest.TestCase): @@ -52,8 +47,6 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) - if self.stage: - self.stage.destroy() os.chdir(self.orig_dir) @@ -64,12 +57,12 @@ def check_archive(self, filename, system): url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz') print url - self.stage = Stage(url) - self.stage.fetch() + with Stage(url) as stage: + stage.fetch() - guesser = ConfigureGuesser() - guesser(self.stage) - self.assertEqual(system, guesser.build_system) + guesser = ConfigureGuesser() + guesser(stage) + self.assertEqual(system, guesser.build_system) def test_python(self): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 0205f4b8ce2..9a57e1f03ea 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -26,19 +26,18 @@ These tests check the database is functioning properly, both in memory and in its file """ -import tempfile -import shutil import multiprocessing - -from llnl.util.lock import * -from llnl.util.filesystem import join_path +import shutil +import tempfile import spack +from llnl.util.filesystem import join_path +from llnl.util.lock import * +from llnl.util.tty.colify import colify from spack.database import Database from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * -from llnl.util.tty.colify import colify def _print_ref_counts(): """Print out all ref counts for the graph used here, for debugging""" diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 925cb648edd..d814572d4a0 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -25,20 +25,17 @@ """\ This test verifies that the Spack directory layout works properly. """ -import unittest -import tempfile -import shutil import os - -from llnl.util.filesystem import * +import shutil +import tempfile import spack -from spack.spec import Spec -from spack.repository import RepoPath +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout +from spack.repository import RepoPath +from spack.spec import Spec from spack.test.mock_packages_test import * - # number of packages to test (to reduce test time) max_packages = 10 diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index d84433176a8..35780441166 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -23,19 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which - +from llnl.util.filesystem import * from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo +from spack.version import ver class GitFetchTest(MockPackagesTest): @@ -52,19 +45,15 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def assert_rev(self, rev): """Check that the current git revision is equal to the supplied rev.""" self.assertEqual(self.repo.rev_hash('HEAD'), self.repo.rev_hash(rev)) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -76,26 +65,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('git')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked_file = 'foobarbaz' - touch(untracked_file) - self.assertTrue(os.path.isfile(untracked_file)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked_file)) + untracked_file = 'foobarbaz' + touch(untracked_file) + self.assertTrue(os.path.isfile(untracked_file)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked_file)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_master(self): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index bbcb64e4c1a..b8a0c1ec460 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -23,16 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest - -from llnl.util.filesystem import * - import spack + from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * from spack.test.mock_repo import MockHgRepo +from llnl.util.filesystem import * +from spack.test.mock_packages_test import * class HgFetchTest(MockPackagesTest): @@ -49,13 +45,10 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def try_fetch(self, rev, test_file, args): """Tries to: @@ -68,26 +61,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('hg')] = args - self.pkg.do_stage() - self.assertEqual(self.repo.get_rev(), rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assertEqual(self.repo.get_rev(), rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assertEqual(self.repo.get_rev(), rev) + self.assertEqual(self.repo.get_rev(), rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 8863d13c426..8297893f012 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -22,18 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import unittest import shutil import tempfile -from llnl.util.filesystem import * - import spack -from spack.stage import Stage -from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which +from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite from spack.test.mock_packages_test import * from spack.test.mock_repo import MockArchive diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 886b7ef4c5d..ee37e765c7b 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -24,8 +24,6 @@ ############################################################################## import os import unittest -import shutil -import tempfile from llnl.util.filesystem import * from llnl.util.link_tree import LinkTree @@ -38,6 +36,7 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') + self.stage.create() with working_dir(self.stage.path): touchp('source/1') @@ -51,10 +50,8 @@ def setUp(self): source_path = os.path.join(self.stage.path, 'source') self.link_tree = LinkTree(source_path) - def tearDown(self): - if self.stage: - self.stage.destroy() + self.stage.destroy() def check_file_link(self, filename): diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index bc68df01dbe..3b11d18da49 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -25,15 +25,13 @@ """ These tests ensure that our lock works correctly. """ -import unittest -import os -import tempfile import shutil +import tempfile +import unittest from multiprocessing import Process -from llnl.util.lock import * from llnl.util.filesystem import join_path, touch - +from llnl.util.lock import * from spack.util.multiproc import Barrier # This is the longest a failed test will take, as the barriers will diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index d568a28d443..a2606acf19b 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -28,13 +28,13 @@ This just tests whether the right args are getting passed to make. """ import os -import unittest -import tempfile import shutil +import tempfile +import unittest from llnl.util.filesystem import * -from spack.util.environment import path_put_first from spack.build_environment import MakeExecutable +from spack.util.environment import path_put_first class MakeExecutableTest(unittest.TestCase): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index f83cc8090c9..e707adfe9d5 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -23,11 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from filecmp import dircmp - import spack import spack.mirror -from spack.util.compression import decompressor_for + +from filecmp import dircmp from spack.test.mock_packages_test import * from spack.test.mock_repo import * @@ -74,14 +73,14 @@ def set_up_package(self, name, MockRepoClass, url_attr): def check_mirror(self): - stage = Stage('spack-mirror-test') - mirror_root = join_path(stage.path, 'test-mirror') + with Stage('spack-mirror-test') as stage: + mirror_root = join_path(stage.path, 'test-mirror') + + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) - # register mirror with spack config - mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } - spack.config.update_config('mirrors', mirrors) - try: os.chdir(stage.path) spack.mirror.create( mirror_root, self.repos, no_checksum=True) @@ -97,38 +96,28 @@ def check_mirror(self): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Now try to fetch each package. - for name, mock_repo in self.repos.items(): - spec = Spec(name).concretized() - pkg = spec.package + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package - pkg._stage = None - saved_checksum_setting = spack.do_checksum - try: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) - - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path(mock_repo.path, 'checked_out') - svn('checkout', mock_repo.url, original_path) - - dcmp = dircmp(original_path, pkg.stage.source_path) - - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) - - # and that all original files are present. - self.assertTrue(all(l in exclude for l in dcmp.left_only)) - - finally: - spack.do_checksum = saved_checksum_setting - pkg.do_clean() - finally: - stage.destroy() + saved_checksum_setting = spack.do_checksum + with pkg.stage: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) + dcmp = dircmp(original_path, pkg.stage.source_path) + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) + spack.do_checksum = saved_checksum_setting def test_git_mirror(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index e9f1f95df55..0b8867b61e2 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,17 +22,15 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys import os import shutil -import unittest import tempfile -from ordereddict_backport import OrderedDict - -from llnl.util.filesystem import mkdirp +import unittest import spack import spack.config +from llnl.util.filesystem import mkdirp +from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index ed94023b0eb..a8bdfb55710 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -26,13 +26,9 @@ import shutil from llnl.util.filesystem import * - -import spack -from spack.version import ver from spack.stage import Stage from spack.util.executable import which - # # VCS Systems used by mock repo code. # diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 7bf4ff0a0a5..2d4b8cd584a 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -25,14 +25,11 @@ """ Test for multi_method dispatch. """ -import unittest import spack from spack.multimethod import * -from spack.version import * -from spack.spec import Spec -from spack.multimethod import when from spack.test.mock_packages_test import * +from spack.version import * class MultiMethodTest(MockPackagesTest): diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index 647976df210..2023ba6d96f 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + from spack.util.naming import NamespaceTrie diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index ebd72819992..55f35ea4c96 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -22,10 +22,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import spack -from spack.spec import Spec, CompilerSpec +from spack.spec import Spec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 83984dc5f65..f0b5e05f3b5 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -22,14 +22,12 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - -from llnl.util.filesystem import join_path import spack +from llnl.util.filesystem import join_path from spack.repository import Repo -from spack.util.naming import mod_to_class from spack.test.mock_packages_test import * +from spack.util.naming import mod_to_class class PackagesTest(MockPackagesTest): diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index d74d3b9b7d7..42949753044 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -28,12 +28,11 @@ Spack was originally 2.7, but enough systems in 2014 are still using 2.6 on their frontend nodes that we need 2.6 to get adopted. """ -import unittest import os import re +import unittest import llnl.util.tty as tty - import pyqver2 import spack diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 632f777cdeb..5e6162b6e66 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -31,8 +31,6 @@ import spack import spack.package -from llnl.util.lang import list_modules - from spack.spec import Spec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 44a09cbd7fb..8c33d1ff6e7 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -22,7 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest from spack.spec import * from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 1daaa4be8fc..6e08e30e135 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -23,9 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + import spack.spec -from spack.spec import * from spack.parse import Token +from spack.spec import * # Sample output for a complex lexing. complex_lex = [Token(ID, 'mvapich_foo'), diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index c1b2a2a5730..dbcf89d8645 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -25,15 +25,13 @@ """\ Test that the Stage class works correctly. """ -import unittest -import shutil import os -import getpass +import shutil +import unittest from contextlib import * -from llnl.util.filesystem import * - import spack +from llnl.util.filesystem import * from spack.stage import Stage from spack.util.executable import which @@ -192,116 +190,90 @@ def check_destroy(self, stage, stage_name): def test_setup_and_destroy_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_no_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_setup_and_destroy_no_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_chdir(self): - stage = Stage(archive_url, name=stage_name) - - stage.chdir() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.chdir() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) self.check_destroy(stage, stage_name) def test_fetch(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - self.check_expand_archive(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + self.check_expand_archive(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) self.check_destroy(stage, stage_name) def test_restage(self): - stage = Stage(archive_url, name=stage_name) + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) - stage.fetch() - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + # Try to make a file in the old archive dir + with open('foobar', 'w') as file: + file.write("this file is to be destroyed.") - # Try to make a file in the old archive dir - with open('foobar', 'w') as file: - file.write("this file is to be destroyed.") + self.assertTrue('foobar' in os.listdir(stage.source_path)) - self.assertTrue('foobar' in os.listdir(stage.source_path)) - - # Make sure the file is not there after restage. - stage.restage() - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.chdir_to_source() - self.check_chdir_to_source(stage, stage_name) - self.assertFalse('foobar' in os.listdir(stage.source_path)) - - stage.destroy() + # Make sure the file is not there after restage. + stage.restage() + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.chdir_to_source() + self.check_chdir_to_source(stage, stage_name) + self.assertFalse('foobar' in os.listdir(stage.source_path)) self.check_destroy(stage, stage_name) diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 454a7f1d1ff..1ee4ee700e7 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -24,18 +24,12 @@ ############################################################################## import os import re -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * - import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * + from spack.test.mock_repo import svn, MockSvnRepo +from spack.version import ver +from spack.test.mock_packages_test import * +from llnl.util.filesystem import * class SvnFetchTest(MockPackagesTest): @@ -51,13 +45,10 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" @@ -70,7 +61,6 @@ def get_rev(): return match.group(1) self.assertEqual(get_rev(), rev) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -82,26 +72,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('svn')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index e0b9618e0cb..4163ab95dd6 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -22,10 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from nose.plugins import Plugin - import os +from nose.plugins import Plugin + class Tally(Plugin): name = 'tally' diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index ccc409dd602..18615b7efe8 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -22,10 +22,11 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest import itertools +import unittest import spack + test_install = __import__("spack.cmd.test-install", fromlist=["BuildId", "create_test_output", "TestResult"]) diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 87adf894017..068a335b491 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -25,10 +25,7 @@ """\ Tests ability of spack to extrapolate URL versions from existing versions. """ -import spack import spack.url as url -from spack.spec import Spec -from spack.version import ver from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index efde7c0c73b..561d4658a10 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -27,8 +27,8 @@ detection in Homebrew. """ import unittest + import spack.url as url -from pprint import pprint class UrlParseTest(unittest.TestCase): diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index aec8baf4eaa..2be38af0d34 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -27,7 +27,6 @@ """ import unittest -import spack import spack.url as url diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 108450e098e..2732006eb3d 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -28,6 +28,7 @@ where it makes sense. """ import unittest + from spack.version import * diff --git a/lib/spack/spack/test/yaml.py b/lib/spack/spack/test/yaml.py index 5a357b8e696..b930c022f26 100644 --- a/lib/spack/spack/test/yaml.py +++ b/lib/spack/spack/test/yaml.py @@ -26,6 +26,7 @@ Test Spack's custom YAML format. """ import unittest + import spack.util.spack_yaml as syaml test_file = """\ From ad103dcafa652a839590f5fce28b2e2ce3b5a56d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 19:55:07 -0800 Subject: [PATCH 66/82] Small refactor: add keep parameter to stage, get rid of stage.destroy call. - package.py uses context manager more effectively. - Stage.__init__ has easier to understand method signature now. - keep can be used to override the default behavior either to keep the stage ALL the time or to delete the stage ALL the time. --- lib/spack/llnl/util/filesystem.py | 5 +- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/package.py | 108 +++++++++++++++------------- lib/spack/spack/stage.py | 116 ++++++++++++++++++++---------- 4 files changed, 141 insertions(+), 90 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 366237ef8f3..a92cb0706d2 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -362,8 +362,9 @@ def remove_dead_links(root): def remove_linked_tree(path): """ - Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real - directory before removing the link. + Removes a directory and its contents. If the directory is a + symlink, follows the link and removes the real directory before + removing the link. Args: path: directory to be removed diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 0c8bd1d5281..6e7179122c1 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -43,4 +43,4 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.stage.destroy() + package.do_clean() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index be45415b759..47d259968aa 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -293,6 +293,7 @@ class SomePackage(Package): .. code-block:: python + p.do_clean() # removes the stage directory entirely p.do_restage() # removes the build directory and # re-expands the archive. @@ -455,7 +456,7 @@ def _make_stage(self): # Construct a composite stage on top of the composite FetchStrategy composite_fetcher = self.fetcher composite_stage = StageComposite() - resources = self._get_resources() + resources = self._get_needed_resources() for ii, fetcher in enumerate(composite_fetcher): if ii == 0: # Construct root stage first @@ -484,12 +485,14 @@ def stage(self, stage): def _make_fetcher(self): - # Construct a composite fetcher that always contains at least one element (the root package). In case there - # are resources associated with the package, append their fetcher to the composite. + # Construct a composite fetcher that always contains at least + # one element (the root package). In case there are resources + # associated with the package, append their fetcher to the + # composite. root_fetcher = fs.for_package_version(self, self.version) fetcher = fs.FetchStrategyComposite() # Composite fetcher fetcher.append(root_fetcher) # Root fetcher is always present - resources = self._get_resources() + resources = self._get_needed_resources() for resource in resources: fetcher.append(resource.fetcher) return fetcher @@ -706,6 +709,7 @@ def do_stage(self, mirror_only=False): self.stage.expand_archive() self.stage.chdir_to_source() + def do_patch(self): """Calls do_stage(), then applied patches to the expanded tarball if they haven't been applied already.""" @@ -798,7 +802,7 @@ def do_fake_install(self): mkdirp(self.prefix.man1) - def _get_resources(self): + def _get_needed_resources(self): resources = [] # Select the resources that are needed for this build for when_spec, resource_list in self.resources.items(): @@ -816,7 +820,7 @@ def _resource_stage(self, resource): def do_install(self, - keep_prefix=False, keep_stage=False, ignore_deps=False, + keep_prefix=False, keep_stage=None, ignore_deps=False, skip_patch=False, verbose=False, make_jobs=None, fake=False): """Called by commands to install a package and its dependencies. @@ -825,7 +829,8 @@ def do_install(self, Args: keep_prefix -- Keep install prefix on failure. By default, destroys it. - keep_stage -- Keep stage on successful build. By default, destroys it. + keep_stage -- Set to True or false to always keep or always delete stage. + By default, stage is destroyed only if there are no exceptions. ignore_deps -- Do not install dependencies before installing this package. fake -- Don't really build -- install fake stub files instead. skip_patch -- Skip patch stage of build if True. @@ -848,32 +853,33 @@ def do_install(self, make_jobs=make_jobs) start_time = time.time() - with self.stage: - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() + if not fake: + if not skip_patch: + self.do_patch() + else: + self.do_stage() - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) + def cleanup(): + if not keep_prefix: + # If anything goes wrong, remove the install prefix + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed." + + "Manually remove this directory to fix:", + self.prefix, wrap=True) - def real_work(): - try: - tty.msg("Building %s" % self.name) + def real_work(): + try: + tty.msg("Building %s" % self.name) + self.stage.keep = keep_stage + with self.stage: # Run the pre-install hook in the child process after # the directory is created. spack.hooks.pre_install(self) @@ -888,7 +894,7 @@ def real_work(): # Save the build environment in a file before building. env_path = join_path(os.getcwd(), 'spack-build.env') - # This redirects I/O to a build log (and optionally to the terminal) + # Redirect I/O to a build log (and optionally to the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') with log_output(log_file, verbose, sys.stdout.isatty(), True): @@ -908,29 +914,25 @@ def real_work(): packages_dir = spack.install_layout.build_packages_path(self.spec) dump_packages(self.spec, packages_dir) - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + except ProcessError as e: + # Annotate with location of build log. + e.build_log = log_path + cleanup() + raise e - except ProcessError as e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e - - except: - # other exceptions just clean up and raise. - cleanup() - raise + except: + # other exceptions just clean up and raise. + cleanup() + raise # Set parallelism before starting build. self.make_jobs = make_jobs @@ -1147,6 +1149,12 @@ def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() + + def do_clean(self): + """Removes the package's build stage and source tarball.""" + self.stage.destroy() + + def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index a22982a6d43..b117c76aa17 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -42,29 +42,53 @@ class Stage(object): - """ - A Stage object is a context manager that handles a directory where some source code is downloaded and built - before being installed. It handles fetching the source code, either as an archive to be expanded or by checking - it out of a repository. A stage's lifecycle looks like this: + """Manages a temporary stage directory for building. + + A Stage object is a context manager that handles a directory where + some source code is downloaded and built before being installed. + It handles fetching the source code, either as an archive to be + expanded or by checking it out of a repository. A stage's + lifecycle looks like this: ``` - with Stage() as stage: # Context manager creates and destroys the stage directory - fetch() # Fetch a source archive into the stage. - expand_archive() # Expand the source archive. - # Build and install the archive. This is handled by the Package class. + with Stage() as stage: # Context manager creates and destroys the stage directory + stage.fetch() # Fetch a source archive into the stage. + stage.expand_archive() # Expand the source archive. + # Build and install the archive. (handled by user of Stage) ``` - If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. - Otherwise, stages are created directly in spack.stage_path. + When used as a context manager, the stage is automatically + destroyed if no exception is raised by the context. If an + excpetion is raised, the stage is left in the filesystem and NOT + destroyed, for potential reuse later. - There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you - fetched a tarball but didn't finish building it, you won't have to fetch it again. + You can also use the stage's create/destroy functions manually, + like this: - Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for - only one run of spack. + ``` + stage = Stage() + try: + stage.create() # Explicitly create the stage directory. + stage.fetch() # Fetch a source archive into the stage. + stage.expand_archive() # Expand the source archive. + # Build and install the archive. (handled by user of Stage) + finally: + stage.destroy() # Explicitly destroy the stage directory. + ``` + + If spack.use_tmp_stage is True, spack will attempt to create + stages in a tmp directory. Otherwise, stages are created directly + in spack.stage_path. + + There are two kinds of stages: named and unnamed. Named stages + can persist between runs of spack, e.g. if you fetched a tarball + but didn't finish building it, you won't have to fetch it again. + + Unnamed stages are created using standard mkdtemp mechanisms or + similar, and are intended to persist for only one run of spack. """ - def __init__(self, url_or_fetch_strategy, **kwargs): + def __init__(self, url_or_fetch_strategy, name=None, mirror_path=None, keep=None): """Create a stage object. Parameters: url_or_fetch_strategy @@ -76,6 +100,18 @@ def __init__(self, url_or_fetch_strategy, **kwargs): and will persist between runs (or if you construct another stage object later). If name is not provided, then this stage will be given a unique name automatically. + + mirror_path + If provided, Stage will search Spack's mirrors for + this archive at the mirror_path, before using the + default fetch strategy. + + keep + By default, when used as a context manager, the Stage + is cleaned up when everything goes well, and it is + kept intact when an exception is raised. You can + override this behavior by setting keep to True + (always keep) or False (always delete). """ # TODO: fetch/stage coupling needs to be reworked -- the logic # TODO: here is convoluted and not modular enough. @@ -91,15 +127,19 @@ def __init__(self, url_or_fetch_strategy, **kwargs): # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root - self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names()) - self.mirror_path = kwargs.get('mirror_path') + self.name = name + if name is None: + self.name = STAGE_PREFIX + next(tempfile._get_candidate_names()) + self.mirror_path = mirror_path self.tmp_root = find_tmp_root() # Try to construct here a temporary name for the stage directory # If this is a named stage, then construct a named path. self.path = join_path(spack.stage_path, self.name) + # Flag to decide whether to delete the stage folder on exit or not - self.delete_on_exit = True + self.keep = keep + def __enter__(self): """ @@ -111,6 +151,7 @@ def __enter__(self): self.create() return self + def __exit__(self, exc_type, exc_val, exc_tb): """ Exiting from a stage context will delete the stage directory unless: @@ -125,11 +166,15 @@ def __exit__(self, exc_type, exc_val, exc_tb): Returns: Boolean """ - self.delete_on_exit = False if exc_type is not None else self.delete_on_exit + if self.keep is None: + # Default: delete when there are no exceptions. + if exc_type is None: self.destroy() - if self.delete_on_exit: + elif not self.keep: + # Overridden. Either always keep or always delete. self.destroy() + def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. @@ -201,7 +246,7 @@ def chdir(self): if os.path.isdir(self.path): os.chdir(self.path) else: - tty.die("Setup failed: no such directory: " + self.path) + raise ChdirError("Setup failed: no such directory: " + self.path) def fetch(self, mirror_only=False): """Downloads an archive or checks out code from a repository.""" @@ -302,11 +347,14 @@ def create(self): """ Creates the stage directory - If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise - this will attempt to create a stage in a temporary directory and link it into spack.stage_path. + If self.tmp_root evaluates to False, the stage directory is + created directly under spack.stage_path, otherwise this will + attempt to create a stage in a temporary directory and link it + into spack.stage_path. - Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location - in tmp_dirs, fall back to making the stage inside spack.stage_path. + Spack will use the first writable location in spack.tmp_dirs + to create a stage. If there is no valid location in tmp_dirs, + fall back to making the stage inside spack.stage_path. """ # Create the top-level stage directory mkdirp(spack.stage_path) @@ -323,9 +371,7 @@ def create(self): ensure_access(self.path) def destroy(self): - """ - Removes this stage directory - """ + """Removes this stage directory.""" remove_linked_tree(self.path) # Make sure we don't end up in a removed directory @@ -370,7 +416,7 @@ def expand_archive(self): shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'check', 'expand_archive', 'restage', 'destroy']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) class StageComposite: """ Composite for Stage type objects. The first item in this composite is considered to be the root package, and @@ -410,7 +456,7 @@ def chdir(self): if os.path.isdir(self.path): os.chdir(self.path) else: - tty.die("Setup failed: no such directory: " + self.path) + raise ChdirError("Setup failed: no such directory: " + self.path) def chdir_to_source(self): self.chdir() @@ -472,19 +518,15 @@ def find_tmp_root(): class StageError(spack.error.SpackError): - def __init__(self, message, long_message=None): - super(self, StageError).__init__(message, long_message) + """"Superclass for all errors encountered during staging.""" class RestageError(StageError): - def __init__(self, message, long_msg=None): - super(RestageError, self).__init__(message, long_msg) + """"Error encountered during restaging.""" class ChdirError(StageError): - def __init__(self, message, long_msg=None): - super(ChdirError, self).__init__(message, long_msg) - + """Raised when Spack can't change directories.""" # Keep this in namespace for convenience FailedDownloadError = fs.FailedDownloadError From ae87948a232c17982a7987e5423ac77acdc59194 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sat, 5 Mar 2016 23:05:45 -0500 Subject: [PATCH 67/82] Fixed URLs Removed vestigal mpi variant in netcdf-fortran --- var/spack/repos/builtin/packages/netcdf-cxx4/package.py | 1 - var/spack/repos/builtin/packages/netcdf-fortran/package.py | 4 +--- var/spack/repos/builtin/packages/netcdf/package.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index ab717ac6ffb..b83e964b00a 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -11,6 +11,5 @@ class NetcdfCxx4(Package): def install(self, spec, prefix): configure('--prefix=%s' % prefix) - make() make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 954e7dc3e81..e4e33445e54 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -3,13 +3,11 @@ class NetcdfFortran(Package): """Fortran interface for NetCDF4""" - homepage = "http://www.unidata.ucar.edu/downloads/netcdf/netcdf-cxx/index.jsp" + homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-fortran-4.4.3.tar.gz" version('4.4.3', 'bfd4ae23a34635b273d3eb0d91cbde9e') - variant('mpi', default=True, description='Enables MPI parallelism') - depends_on('netcdf') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 0b112a59ce9..227362399a9 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -6,7 +6,7 @@ class Netcdf(Package): data formats that support the creation, access, and sharing of array-oriented scientific data.""" - homepage = "http://www.unidata.ucar.edu/software/netcdf/" + homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.3.tar.gz" version('4.4.0', 'cffda0cbd97fdb3a06e9274f7aef438e') From e515042a36e7aa21e52943dab1e8b5594f3f0e94 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 6 Mar 2016 01:41:48 -0800 Subject: [PATCH 68/82] Fix stage creation bug, simplify do_install code. --- lib/spack/spack/package.py | 73 +++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 47d259968aa..ce8cce27e23 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -467,6 +467,11 @@ def _make_stage(self): stage = self._make_resource_stage(composite_stage[0], fetcher, resource) # Append the item to the composite composite_stage.append(stage) + + # Create stage on first access. Needed because fetch, stage, + # patch, and install can be called independently of each + # other, so `with self.stage:` in do_install isn't sufficient. + composite_stage.create() return composite_stage @property @@ -846,35 +851,38 @@ def do_install(self, tty.msg("Installing %s" % self.name) + # First, install dependencies recursively. if not ignore_deps: self.do_install_dependencies( keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps, - fake=fake, skip_patch=skip_patch, verbose=verbose, - make_jobs=make_jobs) - - start_time = time.time() - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() - - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) + fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs) def cleanup(): + """Handles removing install prefix on error.""" if not keep_prefix: - # If anything goes wrong, remove the install prefix self.remove_prefix() else: tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + + "Spack will think this package is installed. " + "Manually remove this directory to fix:", self.prefix, wrap=True) + # Then install the package itself. def real_work(): + """Forked for each build. Has its own process and python + module space set up by build_environment.fork().""" + start_time = time.time() + if not fake: + if not skip_patch: + self.do_patch() + else: + self.do_stage() + + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) + try: tty.msg("Building %s" % self.name) @@ -884,9 +892,9 @@ def real_work(): # the directory is created. spack.hooks.pre_install(self) - # Set up process's build environment before running install. if fake: self.do_fake_install() + else: # Do the real install in the source directory. self.stage.chdir_to_source() @@ -901,18 +909,17 @@ def real_work(): dump_environment(env_path) self.install(self.spec, self.prefix) - # Ensure that something was actually installed. - self._sanity_check_install() + # Ensure that something was actually installed. + self._sanity_check_install() - # Move build log into install directory on success - if not fake: + # Copy provenance into the install directory on success log_install_path = spack.install_layout.build_log_path(self.spec) env_install_path = spack.install_layout.build_env_path(self.spec) + packages_dir = spack.install_layout.build_packages_path(self.spec) + install(log_path, log_install_path) install(env_path, env_install_path) - - packages_dir = spack.install_layout.build_packages_path(self.spec) - dump_packages(self.spec, packages_dir) + dump_packages(self.spec, packages_dir) # Stop timer. self._total_time = time.time() - start_time @@ -934,18 +941,18 @@ def real_work(): cleanup() raise - # Set parallelism before starting build. - self.make_jobs = make_jobs + # Set parallelism before starting build. + self.make_jobs = make_jobs - # Do the build. - spack.build_environment.fork(self, real_work) + # Do the build. + spack.build_environment.fork(self, real_work) - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) + # Once everything else is done, run post install hooks + spack.hooks.post_install(self) def _sanity_check_install(self): From 240ada5775c7857932279d86e4305ef001d33717 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 6 Mar 2016 16:51:09 -0800 Subject: [PATCH 69/82] Add `expand=False` option for URL downloads. - Allows skipping the expand step for downloads. - Fixed stage so that it knows expansion didn't fail when there is a no-expand URLFetchStrategy. - Updated docs to reflect new option, and provided an example. --- lib/spack/docs/packaging_guide.rst | 29 ++++++++++++++++++ lib/spack/llnl/util/filesystem.py | 9 +++++- lib/spack/spack/fetch_strategy.py | 7 ++++- lib/spack/spack/mirror.py | 13 ++++++-- lib/spack/spack/stage.py | 48 ++++++++++++++++++++---------- 5 files changed, 86 insertions(+), 20 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 59ba63fa355..bae8c34d522 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -401,6 +401,35 @@ construct the new one for ``8.2.1``. When you supply a custom URL for a version, Spack uses that URL *verbatim* and does not perform extrapolation. +Skipping the expand step +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spack normally expands archives automatically after downloading +them. If you want to skip this step (e.g., for self-extracting +executables and other custom archive types), you can add +``expand=False`` to a ``version`` directive. + +.. code-block:: python + + version('8.2.1', '4136d7b4c04df68b686570afa26988ac', + url='http://example.com/foo-8.2.1-special-version.tar.gz', 'expand=False') + +When ``expand`` is set to ``False``, Spack sets the current working +directory to the directory containing the downloaded archive before it +calls your ``install`` method. Within ``install``, the path to the +downloaded archive is available as ``self.stage.archive_file``. + +Here is an example snippet for packages distribuetd as self-extracting +archives. The example sets permissions on the downloaded file to make +it executable, then runs it with some arguments. + +.. code-block:: python + + def install(self, spec, prefix): + set_executable(self.stage.archive_file) + installer = Executable(self.stage.archive_file) + installer('--prefix=%s' % prefix, 'arg1', 'arg2', 'etc.') + Checksums ~~~~~~~~~~~~~~~~~ diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index a92cb0706d2..f218b7c424b 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,8 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', + 'set_executable', 'remove_dead_links', 'remove_linked_tree'] import os import sys @@ -345,6 +346,12 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) + +def set_executable(path): + st = os.stat(path) + os.chmod(path, st.st_mode | stat.S_IEXEC) + + def remove_dead_links(root): """ Removes any dead link that is present in root diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index ec17cb97f14..0d0a7db8a91 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -82,7 +82,6 @@ class FetchStrategy(object): class __metaclass__(type): """This metaclass registers all fetch strategies in a list.""" - def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) if cls.enabled: all_strategies.append(cls) @@ -145,6 +144,8 @@ def __init__(self, url=None, digest=None, **kwargs): self.digest = kwargs.get('md5', None) if not self.digest: self.digest = digest + self.expand_archive = kwargs.get('expand', True) + if not self.url: raise ValueError("URLFetchStrategy requires a url for fetching.") @@ -218,6 +219,10 @@ def archive_file(self): @_needs_stage def expand(self): + if not self.expand_archive: + tty.msg("Skipping expand step for %s" % self.archive_file) + return + tty.msg("Staging archive: %s" % self.archive_file) self.stage.chdir() diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index fdc4e7967f5..6981f69ac0e 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -51,13 +51,20 @@ def mirror_archive_filename(spec, fetcher): raise ValueError("mirror.path requires spec with concrete version.") if isinstance(fetcher, fs.URLFetchStrategy): - # If we fetch this version with a URLFetchStrategy, use URL's archive type - ext = url.downloaded_file_extension(fetcher.url) + if fetcher.expand_archive: + # If we fetch this version with a URLFetchStrategy, use URL's archive type + ext = url.downloaded_file_extension(fetcher.url) + else: + # If the archive shouldn't be expanded, don't check for its extension. + ext = None else: # Otherwise we'll make a .tar.gz ourselves ext = 'tar.gz' - return "%s-%s.%s" % (spec.package.name, spec.version, ext) + filename = "%s-%s" % (spec.package.name, spec.version) + if ext: + filename += ".%s" % ext + return filename def mirror_archive_path(spec, fetcher): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index b117c76aa17..b405915a752 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -229,13 +229,22 @@ def archive_file(self): @property def source_path(self): - """Returns the path to the expanded/checked out source code - within this fetch strategy's path. + """Returns the path to the expanded/checked out source code. - This assumes nothing else is going ot be put in the - FetchStrategy's path. It searches for the first - subdirectory of the path it can find, then returns that. + To find the source code, this method searches for the first + subdirectory of the stage that it can find, and returns it. + This assumes nothing besides the archive file will be in the + stage path, but it has the advantage that we don't need to + know the name of the archive or its contents. + + If the fetch strategy is not supposed to expand the downloaded + file, it will just return the stage path. If the archive needs + to be expanded, it will return None when no archive is found. """ + if isinstance(self.fetcher, fs.URLFetchStrategy): + if not self.fetcher.expand_archive: + return self.path + for p in [os.path.join(self.path, f) for f in os.listdir(self.path)]: if os.path.isdir(p): return p @@ -416,21 +425,15 @@ def expand_archive(self): shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) class StageComposite: """ Composite for Stage type objects. The first item in this composite is considered to be the root package, and operations that return a value are forwarded to it. """ - - @property - def source_path(self): - return self[0].source_path - - @property - def path(self): - return self[0].path - + # + # __enter__ and __exit__ delegate to all stages in the composite. + # def __enter__(self): for item in self: item.__enter__() @@ -440,9 +443,24 @@ def __exit__(self, exc_type, exc_val, exc_tb): for item in reversed(self): item.__exit__(exc_type, exc_val, exc_tb) + # + # Below functions act only on the *first* stage in the composite. + # + @property + def source_path(self): + return self[0].source_path + + @property + def path(self): + return self[0].path + def chdir_to_source(self): return self[0].chdir_to_source() + @property + def archive_file(self): + return self[0].archive_file + class DIYStage(object): """Simple class that allows any directory to be a spack stage.""" From 1ee90a6fe792bce1730bfd3fddb4fa12b4597361 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 6 Mar 2016 16:52:34 -0800 Subject: [PATCH 70/82] Make spack_cc, spack_cxx, spack_f77, spack_fc available in build env. - Add for convenience for packages that need to refer to these. - Added an example in the documentation. --- lib/spack/docs/packaging_guide.rst | 9 +++++++++ lib/spack/spack/build_environment.py | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index bae8c34d522..983adb28b01 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -2137,6 +2137,15 @@ Filtering functions Examples: + #. Filtering a Makefile to force it to use Spack's compiler wrappers: + + .. code-block:: python + + filter_file(r'^CC\s*=.*', spack_cc, 'Makefile') + filter_file(r'^CXX\s*=.*', spack_cxx, 'Makefile') + filter_file(r'^F77\s*=.*', spack_f77, 'Makefile') + filter_file(r'^FC\s*=.*', spack_fc, 'Makefile') + #. Replacing ``#!/usr/bin/perl`` with ``#!/usr/bin/env perl`` in ``bib2xhtml``: .. code-block:: python diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 1b877780801..e22597a789b 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -214,6 +214,13 @@ def set_module_variables_for_package(pkg, m): m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE') m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg))) + # Put spack compiler paths in module scope. + link_dir = spack.build_env_path + m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc']) + m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx']) + m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77']) + m.spack_f90 = join_path(link_dir, pkg.compiler.link_paths['fc']) + # Emulate some shell commands for convenience m.pwd = os.getcwd m.cd = os.chdir From 108277fb5dc648c0e93fbd22e6ea6613329acc9d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 6 Mar 2016 19:32:00 -0800 Subject: [PATCH 71/82] Fix docs that have gone stale since repos were added. --- lib/spack/docs/developer_guide.rst | 15 ++++++++++++++- lib/spack/docs/features.rst | 2 +- lib/spack/docs/packaging_guide.rst | 20 ++++++++++---------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index db47de80f59..0b618aa6830 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -73,19 +73,32 @@ with a high level view of Spack's directory structure:: spack/ <- installation root bin/ spack <- main spack executable + + etc/ + spack/ <- Spack config files. + Can be overridden by files in ~/.spack. + var/ spack/ <- build & stage directories + repos/ <- contains package repositories + builtin/ <- pkg repository that comes with Spack + repo.yaml <- descriptor for the builtin repository + packages/ <- directories under here contain packages + opt/ spack/ <- packages are installed here + lib/ spack/ docs/ <- source for this documentation env/ <- compiler wrappers for build environment + external/ <- external libs included in Spack distro + llnl/ <- some general-use libraries + spack/ <- spack module; contains Python code cmd/ <- each file in here is a spack subcommand compilers/ <- compiler description files - packages/ <- each file in here is a spack package test/ <- unit test modules util/ <- common code diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index fcb810086da..0998ba8da4d 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -103,7 +103,7 @@ creates a simple python file: It doesn't take much python coding to get from there to a working package: -.. literalinclude:: ../../../var/spack/packages/libelf/package.py +.. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py :lines: 25- Spack also provides wrapper functions around common commands like diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 59ba63fa355..a3e1f4de592 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -84,7 +84,7 @@ always choose to download just one tarball initially, and run If it fails entirely, you can get minimal boilerplate by using :ref:`spack-edit-f`, or you can manually create a directory and - ``package.py`` file for the package in ``var/spack/packages``. + ``package.py`` file for the package in ``var/spack/repos/builtin/packages``. .. note:: @@ -203,7 +203,7 @@ edit`` command: So, if you used ``spack create`` to create a package, then saved and closed the resulting file, you can get back to it with ``spack edit``. The ``cmake`` package actually lives in -``$SPACK_ROOT/var/spack/packages/cmake/package.py``, but this provides +``$SPACK_ROOT/var/spack/repos/builtin/packages/cmake/package.py``, but this provides a much simpler shortcut and saves you the trouble of typing the full path. @@ -269,18 +269,18 @@ live in Spack's directory structure. In general, `spack-create`_ and `spack-edit`_ handle creating package files for you, so you can skip most of the details here. -``var/spack/packages`` +``var/spack/repos/builtin/packages`` ~~~~~~~~~~~~~~~~~~~~~~~ A Spack installation directory is structured like a standard UNIX install prefix (``bin``, ``lib``, ``include``, ``var``, ``opt``, etc.). Most of the code for Spack lives in ``$SPACK_ROOT/lib/spack``. -Packages themselves live in ``$SPACK_ROOT/var/spack/packages``. +Packages themselves live in ``$SPACK_ROOT/var/spack/repos/builtin/packages``. If you ``cd`` to that directory, you will see directories for each package: -.. command-output:: cd $SPACK_ROOT/var/spack/packages; ls -CF +.. command-output:: cd $SPACK_ROOT/var/spack/repos/builtin/packages; ls -CF :shell: :ellipsis: 10 @@ -288,7 +288,7 @@ Each directory contains a file called ``package.py``, which is where all the python code for the package goes. For example, the ``libelf`` package lives in:: - $SPACK_ROOT/var/spack/packages/libelf/package.py + $SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py Alongside the ``package.py`` file, a package may contain extra directories or files (like patches) that it needs to build. @@ -301,7 +301,7 @@ Packages are named after the directory containing ``package.py``. So, ``libelf``'s ``package.py`` lives in a directory called ``libelf``. The ``package.py`` file defines a class called ``Libelf``, which extends Spack's ``Package`` class. for example, here is -``$SPACK_ROOT/var/spack/packages/libelf/package.py``: +``$SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py``: .. code-block:: python :linenos: @@ -328,7 +328,7 @@ these: $ spack install libelf@0.8.13 Spack sees the package name in the spec and looks for -``libelf/package.py`` in ``var/spack/packages``. Likewise, if you say +``libelf/package.py`` in ``var/spack/repos/builtin/packages``. Likewise, if you say ``spack install py-numpy``, then Spack looks for ``py-numpy/package.py``. @@ -703,7 +703,7 @@ supply is a filename, then the patch needs to live within the spack source tree. For example, the patch above lives in a directory structure like this:: - $SPACK_ROOT/var/spack/packages/ + $SPACK_ROOT/var/spack/repos/builtin/packages/ mvapich2/ package.py ad_lustre_rwcontig_open_source.patch @@ -1533,7 +1533,7 @@ The last element of a package is its ``install()`` method. This is where the real work of installation happens, and it's the main part of the package you'll need to customize for each piece of software. -.. literalinclude:: ../../../var/spack/packages/libelf/package.py +.. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py :start-after: 0.8.12 :linenos: From 265ef337a8cdb7397aa01858077787ca4c2669fb Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 3 Mar 2016 10:40:06 -0500 Subject: [PATCH 72/82] Don't quote -rpath literal --- lib/spack/env/cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index c3d11357224..fb0df79d334 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -176,17 +176,17 @@ while [ -n "$1" ]; do -Wl,*) arg="${1#-Wl,}" # TODO: Handle multiple -Wl, continuations of -Wl,-rpath - if [[ $arg == '-rpath='* ]]; then + if [[ $arg == -rpath=* ]]; then arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg == '-rpath,'* ]]; then + elif [[ $arg == -rpath,* ]]; then arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg == '-rpath' ]]; then + elif [[ $arg == -rpath ]]; then shift; arg="$1" if [[ $arg != '-Wl,'* ]]; then die "-Wl,-rpath was not followed by -Wl,*" From a399451e1e32c9a294f5b735237cef93ddbd4131 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 6 Mar 2016 23:34:48 -0500 Subject: [PATCH 73/82] Update OpenSSL to 1.0.2g --- var/spack/repos/builtin/packages/openssl/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index c73102f05d3..70afaf4038a 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -17,6 +17,7 @@ class Openssl(Package): version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') + version('1.0.2g', 'f3c710c045cdee5fd114feb69feba7aa') depends_on("zlib") parallel = False From 547933e7e4997a175bf03363b5c0cf7e687f2000 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 7 Mar 2016 01:08:08 -0800 Subject: [PATCH 74/82] Fix bug in -Xlinker argument handling - also update test to match. --- lib/spack/env/cc | 15 ++++++++------- lib/spack/spack/test/cc.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 5e6ed939852..a19346ce971 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -199,19 +199,20 @@ while [ -n "$1" ]; do other_args+=("-Wl,$arg") fi ;; - -Xlinker,*) - arg="${1#-Xlinker,}" - if [ -z "$arg" ]; then shift; arg="$1"; fi + -Xlinker) + shift; arg="$1"; if [[ $arg = -rpath=* ]]; then rpaths+=("${arg#-rpath=}") elif [[ $arg = -rpath ]]; then shift; arg="$1" - if [[ $arg != -Xlinker,* ]]; then - die "-Xlinker,-rpath was not followed by -Xlinker,*" + if [[ $arg != -Xlinker ]]; then + die "-Xlinker -rpath was not followed by -Xlinker " fi - rpaths+=("${arg#-Xlinker,}") + shift; arg="$1" + rpaths+=("$arg") else - other_args+=("-Xlinker,$arg") + other_args+=("-Xlinker") + other_args+=("$arg") fi ;; *) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 11420ec44a1..f3f6d4a22e6 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -43,7 +43,7 @@ '-llib1', '-llib2', 'arg4', '-Wl,--end-group', - '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath', '-Xlinker,/fourth/rpath', + '-Xlinker', '-rpath', '-Xlinker', '/third/rpath', '-Xlinker', '-rpath', '-Xlinker', '/fourth/rpath', '-llib3', '-llib4', 'arg5', 'arg6'] From 12ddf241a53f85d871c981307e5f682d2e979c75 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 7 Mar 2016 02:39:14 -0800 Subject: [PATCH 75/82] Fix code block in docs. --- lib/spack/spack/multimethod.py | 36 ++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index 51c6a8e89d9..3cd17e796a3 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -138,7 +138,7 @@ class when(object): methods like install() that depend on the package's spec. For example: - .. code-block:: + .. code-block:: python class SomePackage(Package): ... @@ -163,26 +163,28 @@ def install(self, prefix): if you only have part of the install that is platform specific, you could do this: - class SomePackage(Package): - ... - # virtual dependence on MPI. - # could resolve to mpich, mpich2, OpenMPI - depends_on('mpi') + .. code-block:: python - def setup(self): - # do nothing in the default case - pass + class SomePackage(Package): + ... + # virtual dependence on MPI. + # could resolve to mpich, mpich2, OpenMPI + depends_on('mpi') - @when('^openmpi') - def setup(self): - # do something special when this is built with OpenMPI for - # its MPI implementations. + def setup(self): + # do nothing in the default case + pass + + @when('^openmpi') + def setup(self): + # do something special when this is built with OpenMPI for + # its MPI implementations. - def install(self, prefix): - # Do common install stuff - self.setup() - # Do more common install stuff + def install(self, prefix): + # Do common install stuff + self.setup() + # Do more common install stuff There must be one (and only one) @when clause that matches the package's spec. If there is more than one, or if none match, From 77ec27c73013eb14821a4249f6a08a8321e10eef Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 8 Mar 2016 11:09:41 +0100 Subject: [PATCH 76/82] fixed bug : similar issues in checksum and md5 as were solved in ad103dcafa652a839590f5fce28b2e2ce3b5a56d --- lib/spack/spack/cmd/checksum.py | 39 +++++++++--------------- lib/spack/spack/cmd/md5.py | 54 ++++++++++++++++----------------- lib/spack/spack/cmd/stage.py | 4 ++- lib/spack/spack/stage.py | 1 + 4 files changed, 45 insertions(+), 53 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 966ff9a5e96..5504673c9e4 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -22,23 +22,18 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import re import argparse import hashlib -from pprint import pprint -from subprocess import CalledProcessError import llnl.util.tty as tty -from llnl.util.tty.colify import colify - import spack import spack.cmd import spack.util.crypto from spack.stage import Stage, FailedDownloadError from spack.version import * -description ="Checksum available versions of a package." +description = "Checksum available versions of a package." + def setup_parser(subparser): subparser.add_argument( @@ -60,30 +55,24 @@ def get_checksums(versions, urls, **kwargs): hashes = [] i = 0 for url, version in zip(urls, versions): - stage = Stage(url) try: - stage.fetch() - if i == 0 and first_stage_function: - first_stage_function(stage) + with Stage(url) as stage: + stage.delete_on_exit = not keep_stage + stage.fetch() + if i == 0 and first_stage_function: + first_stage_function(stage) - hashes.append((version, - spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) - except FailedDownloadError, e: + hashes.append((version, + spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) + i += 1 + except FailedDownloadError as e: tty.msg("Failed to fetch %s" % url) - continue - except Exception, e: + except Exception as e: tty.msg('Something failed on %s, skipping.\n (%s)' % (url, e)) - continue - - finally: - if not keep_stage: - stage.destroy() - i += 1 return hashes - def checksum(parser, args): # get the package we're going to generate checksums for pkg = spack.repo.get(args.package) @@ -106,8 +95,8 @@ def checksum(parser, args): tty.msg("Found %s versions of %s" % (len(versions), pkg.name), *spack.cmd.elide_list( - ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) - print + ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) + print() archives_to_fetch = tty.get_number( "How many would you like to checksum?", default=5, abort='q') diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index 879ef9f7b74..20508abf99c 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -22,51 +22,51 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os import argparse import hashlib - -from contextlib import contextmanager +import os import llnl.util.tty as tty -from llnl.util.filesystem import * - import spack.util.crypto from spack.stage import Stage, FailedDownloadError description = "Calculate md5 checksums for files/urls." -@contextmanager -def stager(url): - _cwd = os.getcwd() - _stager = Stage(url) - try: - _stager.fetch() - yield _stager - except FailedDownloadError: - tty.msg("Failed to fetch %s" % url) - finally: - _stager.destroy() - os.chdir(_cwd) # the Stage class changes the current working dir so it has to be restored def setup_parser(subparser): setup_parser.parser = subparser subparser.add_argument('files', nargs=argparse.REMAINDER, help="Files to checksum.") + +def compute_md5_checksum(url): + if not os.path.isfile(url): + with Stage(url) as stage: + stage.fetch() + value = spack.util.crypto.checksum(hashlib.md5, stage.archive_file) + else: + value = spack.util.crypto.checksum(hashlib.md5, url) + return value + + def md5(parser, args): if not args.files: setup_parser.parser.print_help() return 1 - for f in args.files: - if not os.path.isfile(f): - with stager(f) as stage: - checksum = spack.util.crypto.checksum(hashlib.md5, stage.archive_file) - print "%s %s" % (checksum, f) - else: - if not can_access(f): - tty.die("Cannot read file: %s" % f) + results = [] + for url in args.files: + try: + checksum = compute_md5_checksum(url) + results.append((checksum, url)) + except FailedDownloadError as e: + tty.warn("Failed to fetch %s" % url) + tty.warn("%s" % e) + except IOError as e: + tty.warn("Error when reading %s" % url) + tty.warn("%s" % e) - checksum = spack.util.crypto.checksum(hashlib.md5, f) - print "%s %s" % (checksum, f) + # Dump the MD5s at last without interleaving them with downloads + tty.msg("Number of MD5 check-sums computed: %s " % len(results)) + for checksum, url in results: + tty.msg("%s %s" % (checksum, url)) diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index 5786780efb4..b575f6c4566 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -50,4 +50,6 @@ def stage(parser, args): specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.do_stage() + with package.stage as stage: + stage.delete_on_exit = False + package.do_stage() diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index b117c76aa17..9404e12e849 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -438,6 +438,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): for item in reversed(self): + item.delete_on_exit = getattr(self, 'delete_on_exit', True) item.__exit__(exc_type, exc_val, exc_tb) def chdir_to_source(self): From 18ce5ccf8fb31dae71848f541f3e139bd582953c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 02:49:11 -0800 Subject: [PATCH 77/82] Avoid race in pango's `make install`, set parallel=False --- var/spack/repos/builtin/packages/pango/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/pango/package.py b/var/spack/repos/builtin/packages/pango/package.py index df43625bf5c..79dad3a3d2f 100644 --- a/var/spack/repos/builtin/packages/pango/package.py +++ b/var/spack/repos/builtin/packages/pango/package.py @@ -16,4 +16,4 @@ class Pango(Package): def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() - make("install") + make("install", parallel=False) From 5aadb6df19ea6d7575ee5f5e5f18d92934245d98 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 02:50:26 -0800 Subject: [PATCH 78/82] Fixes #502. Create install prefix before build, clean up do_install. - Fix bug introduced during merge of stage refactor. - install prefix was not created before build_environment.fork() - build_environment.fork() calls setup_dependent_environment - python's setup_dependent_environment can inadvertently create the install prefix before directory_layout expects it. - Clean up Package.do_install: - simplify control flow: parent process now entirely responsible for creating/destroying the install prefix. cleanup is now in one place. - Hoisting cleanup out of the child improves nesting of try/catch in `real_work`. - `real_work` renamed to `build_process` --- lib/spack/spack/directory_layout.py | 56 +++++++++----- lib/spack/spack/package.py | 115 +++++++++++++--------------- 2 files changed, 91 insertions(+), 80 deletions(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 08c23627f46..242eb1afa04 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -85,6 +85,16 @@ def create_install_directory(self, spec): raise NotImplementedError() + def check_installed(self, spec): + """Checks whether a spec is installed. + + Return the spec's prefix, if it is installed, None otherwise. + + Raise an exception if the install is inconsistent or corrupt. + """ + raise NotImplementedError() + + def extension_map(self, spec): """Get a dict of currently installed extension packages for a spec. @@ -246,26 +256,36 @@ def build_packages_path(self, spec): def create_install_directory(self, spec): _check_concrete(spec) + prefix = self.check_installed(spec) + if prefix: + raise InstallDirectoryAlreadyExistsError(prefix) + + mkdirp(self.metadata_path(spec)) + self.write_spec(spec, self.spec_file_path(spec)) + + + def check_installed(self, spec): + _check_concrete(spec) path = self.path_for_spec(spec) spec_file_path = self.spec_file_path(spec) - if os.path.isdir(path): - if not os.path.isfile(spec_file_path): - raise InconsistentInstallDirectoryError( - 'No spec file found at path %s' % spec_file_path) + if not os.path.isdir(path): + return None - installed_spec = self.read_spec(spec_file_path) - if installed_spec == self.spec: - raise InstallDirectoryAlreadyExistsError(path) + if not os.path.isfile(spec_file_path): + raise InconsistentInstallDirectoryError( + 'Inconsistent state: install prefix exists but contains no spec.yaml:', + " " + path) - if spec.dag_hash() == installed_spec.dag_hash(): - raise SpecHashCollisionError(installed_hash, spec_hash) - else: - raise InconsistentInstallDirectoryError( - 'Spec file in %s does not match hash!' % spec_file_path) + installed_spec = self.read_spec(spec_file_path) + if installed_spec == spec: + return path - mkdirp(self.metadata_path(spec)) - self.write_spec(spec, spec_file_path) + if spec.dag_hash() == installed_spec.dag_hash(): + raise SpecHashCollisionError(installed_hash, spec_hash) + else: + raise InconsistentInstallDirectoryError( + 'Spec file in %s does not match hash!' % spec_file_path) def all_specs(self): @@ -399,8 +419,8 @@ def remove_extension(self, spec, ext_spec): class DirectoryLayoutError(SpackError): """Superclass for directory layout errors.""" - def __init__(self, message): - super(DirectoryLayoutError, self).__init__(message) + def __init__(self, message, long_msg=None): + super(DirectoryLayoutError, self).__init__(message, long_msg) class SpecHashCollisionError(DirectoryLayoutError): @@ -422,8 +442,8 @@ def __init__(self, installed_spec, prefix, error): class InconsistentInstallDirectoryError(DirectoryLayoutError): """Raised when a package seems to be installed to the wrong place.""" - def __init__(self, message): - super(InconsistentInstallDirectoryError, self).__init__(message) + def __init__(self, message, long_msg=None): + super(InconsistentInstallDirectoryError, self).__init__(message, long_msg) class InstallDirectoryAlreadyExistsError(DirectoryLayoutError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index d4acbf50243..972a0410b9a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -845,7 +845,8 @@ def do_install(self, if not self.spec.concrete: raise ValueError("Can only install concrete packages.") - if os.path.exists(self.prefix): + # Ensure package is not already installed + if spack.install_layout.check_installed(self.spec): tty.msg("%s is already installed in %s" % (self.name, self.prefix)) return @@ -857,18 +858,11 @@ def do_install(self, keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps, fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs) - def cleanup(): - """Handles removing install prefix on error.""" - if not keep_prefix: - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed. " + - "Manually remove this directory to fix:", - self.prefix, wrap=True) + # Set parallelism before starting build. + self.make_jobs = make_jobs # Then install the package itself. - def real_work(): + def build_process(): """Forked for each build. Has its own process and python module space set up by build_environment.fork().""" start_time = time.time() @@ -878,30 +872,24 @@ def real_work(): else: self.do_stage() - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) + tty.msg("Building %s" % self.name) - try: - tty.msg("Building %s" % self.name) + self.stage.keep = keep_stage + with self.stage: + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) - self.stage.keep = keep_stage - with self.stage: - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) + if fake: + self.do_fake_install() + else: + # Do the real install in the source directory. + self.stage.chdir_to_source() - if fake: - self.do_fake_install() - - else: - # Do the real install in the source directory. - self.stage.chdir_to_source() - - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') + try: # Redirect I/O to a build log (and optionally to the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') @@ -909,43 +897,46 @@ def real_work(): dump_environment(env_path) self.install(self.spec, self.prefix) - # Ensure that something was actually installed. - self._sanity_check_install() + except ProcessError as e: + # Annotate ProcessErrors with the location of the build log. + e.build_log = log_path + raise e - # Copy provenance into the install directory on success - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - packages_dir = spack.install_layout.build_packages_path(self.spec) + # Ensure that something was actually installed. + self._sanity_check_install() - install(log_path, log_install_path) - install(env_path, env_install_path) - dump_packages(self.spec, packages_dir) + # Copy provenance into the install directory on success + log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) + packages_dir = spack.install_layout.build_packages_path(self.spec) - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + install(log_path, log_install_path) + install(env_path, env_install_path) + dump_packages(self.spec, packages_dir) - tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - except ProcessError as e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - except: - # other exceptions just clean up and raise. - cleanup() - raise - - # Set parallelism before starting build. - self.make_jobs = make_jobs - - # Do the build. - spack.build_environment.fork(self, real_work) + try: + # Create the install prefix and fork the build process. + spack.install_layout.create_install_directory(self.spec) + spack.build_environment.fork(self, build_process) + except: + # remove the install prefix if anything went wrong during install. + if not keep_prefix: + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed. " + + "Manually remove this directory to fix:", + self.prefix, wrap=True) + raise # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. From a4861a30729e32aef493b3418d03874837eecc08 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 10:39:46 -0800 Subject: [PATCH 79/82] Minor changes/bugfixes on md5/checksum PR --- lib/spack/spack/cmd/checksum.py | 5 ++--- lib/spack/spack/cmd/md5.py | 4 ++-- lib/spack/spack/cmd/stage.py | 4 +--- lib/spack/spack/stage.py | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 5504673c9e4..518d2703dcd 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -56,8 +56,7 @@ def get_checksums(versions, urls, **kwargs): i = 0 for url, version in zip(urls, versions): try: - with Stage(url) as stage: - stage.delete_on_exit = not keep_stage + with Stage(url, keep=keep_stage) as stage: stage.fetch() if i == 0 and first_stage_function: first_stage_function(stage) @@ -96,7 +95,7 @@ def checksum(parser, args): tty.msg("Found %s versions of %s" % (len(versions), pkg.name), *spack.cmd.elide_list( ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) - print() + print archives_to_fetch = tty.get_number( "How many would you like to checksum?", default=5, abort='q') diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index 20508abf99c..f99fc0f8c2b 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -67,6 +67,6 @@ def md5(parser, args): tty.warn("%s" % e) # Dump the MD5s at last without interleaving them with downloads - tty.msg("Number of MD5 check-sums computed: %s " % len(results)) + tty.msg("%d MD5 checksums:" % len(results)) for checksum, url in results: - tty.msg("%s %s" % (checksum, url)) + print "%s %s" % (checksum, url) diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index b575f6c4566..5786780efb4 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -50,6 +50,4 @@ def stage(parser, args): specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: package = spack.repo.get(spec) - with package.stage as stage: - stage.delete_on_exit = False - package.do_stage() + package.do_stage() diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 0d35511c347..d2ed03c271e 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -441,7 +441,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): for item in reversed(self): - item.delete_on_exit = getattr(self, 'delete_on_exit', True) + item.keep = getattr(self, 'keep', None) item.__exit__(exc_type, exc_val, exc_tb) # From a1be45d0e7f1921176b40b2aa497309029f1f7ad Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 13:56:44 -0800 Subject: [PATCH 80/82] Fix bug with setting module-scope vars in derived package classes. --- lib/spack/spack/build_environment.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index e22597a789b..87fc310b5a3 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -177,8 +177,6 @@ def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ - m = pkg.module - # number of jobs spack will to build with. jobs = multiprocessing.cpu_count() if not pkg.parallel: From df84677d1640139673c26a4c407698486905fbb3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 13:57:13 -0800 Subject: [PATCH 81/82] Make diy generate verbose build output by default. - added -q option to shut it up --- lib/spack/spack/cmd/diy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 9df53312f80..2c3a8761ab6 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -45,6 +45,9 @@ def setup_parser(subparser): subparser.add_argument( '--skip-patch', action='store_true', help="Skip patching for the DIY build.") + subparser.add_argument( + '-q', '--quiet', action='store_true', dest='quiet', + help="Do not display verbose build output while installing.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="specs to use for install. Must contain package AND verison.") @@ -92,4 +95,5 @@ def diy(self, args): package.do_install( keep_prefix=args.keep_prefix, ignore_deps=args.ignore_deps, + verbose=not args.quiet, keep_stage=True) # don't remove source dir for DIY. From 9c6184373752746e1219263c1afd9d1955911891 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 13:58:41 -0800 Subject: [PATCH 82/82] Fix bugs in DIYStage: fetch & context handling. - DIYStage needs to be a context handler - DIYStage.fetch needs to take 2 args. --- lib/spack/spack/stage.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index d2ed03c271e..5354135e6a3 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -477,10 +477,14 @@ def chdir(self): else: raise ChdirError("Setup failed: no such directory: " + self.path) + # DIY stages do nothing as context managers. + def __enter__(self): pass + def __exit__(self, exc_type, exc_val, exc_tb): pass + def chdir_to_source(self): self.chdir() - def fetch(self): + def fetch(self, mirror_only): tty.msg("No need to fetch for DIY.") def check(self):