From 0890ac4507e5568d1d5a5da9bbd3c729238a9b3a Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 23 May 2016 10:54:48 +0200 Subject: [PATCH 01/36] modules : explicit module file load from configuration files --- lib/spack/spack/config.py | 1 + lib/spack/spack/modules.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index ec37bd290c9..c3133697f42 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -286,6 +286,7 @@ 'autoload': {'$ref': '#/definitions/dependency_selection'}, 'prerequisites': {'$ref': '#/definitions/dependency_selection'}, 'conflict': {'$ref': '#/definitions/array_of_strings'}, + 'load': {'$ref': '#/definitions/array_of_strings'}, 'environment': { 'type': 'object', 'default': {}, diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index d2b819e80ac..9fa92a593f6 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -381,6 +381,8 @@ def write(self): for x in filter_blacklisted( module_configuration.pop('autoload', []), self.name): module_file_content += self.autoload(x) + for x in module_configuration.pop('load', []): + module_file_content += self.autoload(x) for x in filter_blacklisted( module_configuration.pop('prerequisites', []), self.name): module_file_content += self.prerequisite(x) @@ -402,8 +404,12 @@ def module_specific_content(self, configuration): return tuple() def autoload(self, spec): - m = type(self)(spec) - return self.autoload_format.format(module_file=m.use_name) + if not isinstance(spec, str): + m = type(self)(spec) + module_file = m.use_name + else: + module_file = spec + return self.autoload_format.format(module_file=module_file) def prerequisite(self, spec): m = type(self)(spec) From 84707ed92631bbc4e1c0a7d5c4f1ba19e8e60192 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 24 May 2016 15:23:52 +0200 Subject: [PATCH 02/36] modules : it's possible to add suffixes to module files based on constraint --- lib/spack/spack/config.py | 1 + lib/spack/spack/modules.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index c3133697f42..9037c5d796f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -287,6 +287,7 @@ 'prerequisites': {'$ref': '#/definitions/dependency_selection'}, 'conflict': {'$ref': '#/definitions/array_of_strings'}, 'load': {'$ref': '#/definitions/array_of_strings'}, + 'suffixes': {'$ref': '#/definitions/dictionary_of_strings'}, 'environment': { 'type': 'object', 'default': {}, diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 9fa92a593f6..fb58be6ce01 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -285,11 +285,18 @@ def use_name(self): naming_tokens = self.tokens naming_scheme = self.naming_scheme name = naming_scheme.format(**naming_tokens) - name += '-' + self.spec.dag_hash( - ) # Always append the hash to make the module file unique # Not everybody is working on linux... parts = name.split('/') name = join_path(*parts) + # Add optional suffixes based on constraints + configuration, _ = parse_config_options(self) + suffixes = [name] + for constraint, suffix in configuration.get('suffixes', {}).items(): + if constraint in self.spec: + suffixes.append(suffix) + # Always append the hash to make the module file unique + suffixes.append(self.spec.dag_hash()) + name = '-'.join(suffixes) return name @property From c16d7c481c700a55a592daed05424250656adb34 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sat, 28 May 2016 23:30:56 -0500 Subject: [PATCH 03/36] Get glib to compile with gcc-6.1 This commit adds a patch to fix the case where gcc-6.1 emits an error for string format literal that was a warning in previous versions of gcc. The patch reference is https://bugzilla.gnome.org/show_bug.cgi?id=761550 Also ran the package file through autopep8. --- .../packages/glib/g_date_strftime.patch | 34 +++++++++++++++++++ .../repos/builtin/packages/glib/package.py | 7 ++-- 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 var/spack/repos/builtin/packages/glib/g_date_strftime.patch diff --git a/var/spack/repos/builtin/packages/glib/g_date_strftime.patch b/var/spack/repos/builtin/packages/glib/g_date_strftime.patch new file mode 100644 index 00000000000..532c7432774 --- /dev/null +++ b/var/spack/repos/builtin/packages/glib/g_date_strftime.patch @@ -0,0 +1,34 @@ +From 00148329967adb196138372771052a3f606a6ea3 Mon Sep 17 00:00:00 2001 +From: coypu +Date: Wed, 2 Mar 2016 19:43:10 +0200 +Subject: [PATCH 2/2] gdate: Suppress string format literal warning + +Newer versions of GCC emit an error here, but we know it's safe. +https://bugzilla.gnome.org/761550 +--- + glib/gdate.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/glib/gdate.c b/glib/gdate.c +index 4aece02..92c34d2 100644 +--- a/glib/gdate.c ++++ b/glib/gdate.c +@@ -2439,6 +2439,9 @@ win32_strftime_helper (const GDate *d, + * + * Returns: number of characters written to the buffer, or 0 the buffer was too small + */ ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wformat-nonliteral" ++ + gsize + g_date_strftime (gchar *s, + gsize slen, +@@ -2549,3 +2552,5 @@ g_date_strftime (gchar *s, + return retval; + #endif + } ++ ++#pragma GCC diagnostic pop +-- +2.7.1 + diff --git a/var/spack/repos/builtin/packages/glib/package.py b/var/spack/repos/builtin/packages/glib/package.py index b146ea63d85..e8b92020bdc 100644 --- a/var/spack/repos/builtin/packages/glib/package.py +++ b/var/spack/repos/builtin/packages/glib/package.py @@ -25,20 +25,23 @@ from spack import * import sys + class Glib(Package): """The GLib package contains a low-level libraries useful for providing data structure handling for C, portability wrappers and interfaces for such runtime functionality as an event loop, threads, dynamic loading and an object system.""" homepage = "https://developer.gnome.org/glib/" - url = "http://ftp.gnome.org/pub/gnome/sources/glib/2.42/glib-2.42.1.tar.xz" + url = "http://ftp.gnome.org/pub/gnome/sources/glib/2.42/glib-2.42.1.tar.xz" version('2.42.1', '89c4119e50e767d3532158605ee9121a') depends_on("libffi") depends_on("zlib") depends_on("pkg-config") - depends_on('gettext', sys.platform=='darwin') + depends_on('gettext', sys.platform == 'darwin') + + patch('g_date_strftime.patch') def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 66aaaa17512dbb2a72333e62f0166557785d4dbf Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Sun, 29 May 2016 15:30:09 -0500 Subject: [PATCH 04/36] Add a comment regarding the reason for the patch. --- var/spack/repos/builtin/packages/glib/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/glib/package.py b/var/spack/repos/builtin/packages/glib/package.py index e8b92020bdc..342a61a6a1b 100644 --- a/var/spack/repos/builtin/packages/glib/package.py +++ b/var/spack/repos/builtin/packages/glib/package.py @@ -41,6 +41,7 @@ class Glib(Package): depends_on("pkg-config") depends_on('gettext', sys.platform == 'darwin') + # The following patch is needed for gcc-6.1 patch('g_date_strftime.patch') def install(self, spec, prefix): From 0df0677d5bb4c5d5dce6bb9371c356af4b299170 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 1 Jun 2016 03:00:03 +0200 Subject: [PATCH 05/36] Adding a variant to link against the spack libraries instead of the system ones --- .../repos/builtin/packages/emacs/package.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/emacs/package.py b/var/spack/repos/builtin/packages/emacs/package.py index 4b05864a1e3..b1177d63393 100644 --- a/var/spack/repos/builtin/packages/emacs/package.py +++ b/var/spack/repos/builtin/packages/emacs/package.py @@ -31,15 +31,27 @@ class Emacs(Package): version('24.5', 'd74b597503a68105e61b5b9f6d065b44') + variant('use-system-deps', default=True, description='Uses the library on the system to compile') + depends_on('ncurses') # Emacs also depends on: # GTK or other widget library # libtiff, png, etc. # For now, we assume the system provides all that stuff. # For Ubuntu 14.04 LTS: - # sudo apt-get install libgtk-3-dev libxpm-dev libtiff5-dev libjpeg8-dev libgif-dev libpng12-dev + # sudo apt-get install libgtk-3-dev libxpm-dev libtiff5-dev libjpeg8-dev libgif-dev libpng12-dev + depends_on('libtiff', when='~use-system-deps') + depends_on('libpng', when='~use-system-deps') def install(self, spec, prefix): - configure('--prefix=%s' % prefix) + if '~use-system-deps' in self.spec: + args = [ + '--with-xpm=no', + '--with-gif=no', + ] + else: + args = [] + + configure('--prefix=%s' % prefix, *args) make() make("install") From 5d4a0e4050ad0fa02ffd482a61374bde946c62b7 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 1 Jun 2016 03:28:42 +0200 Subject: [PATCH 06/36] Corrections for flake8 --- var/spack/repos/builtin/packages/emacs/package.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/emacs/package.py b/var/spack/repos/builtin/packages/emacs/package.py index b1177d63393..b6d638e3c32 100644 --- a/var/spack/repos/builtin/packages/emacs/package.py +++ b/var/spack/repos/builtin/packages/emacs/package.py @@ -24,22 +24,25 @@ ############################################################################## from spack import * + class Emacs(Package): """The Emacs programmable text editor.""" + homepage = "https://www.gnu.org/software/emacs" url = "http://ftp.gnu.org/gnu/emacs/emacs-24.5.tar.gz" version('24.5', 'd74b597503a68105e61b5b9f6d065b44') variant('use-system-deps', default=True, description='Uses the library on the system to compile') - + depends_on('ncurses') # Emacs also depends on: # GTK or other widget library # libtiff, png, etc. # For now, we assume the system provides all that stuff. # For Ubuntu 14.04 LTS: - # sudo apt-get install libgtk-3-dev libxpm-dev libtiff5-dev libjpeg8-dev libgif-dev libpng12-dev + # sudo apt-get install libgtk-3-dev libxpm-dev + # libtiff5-dev libjpeg8-dev libgif-dev libpng12-dev depends_on('libtiff', when='~use-system-deps') depends_on('libpng', when='~use-system-deps') @@ -48,10 +51,10 @@ def install(self, spec, prefix): args = [ '--with-xpm=no', '--with-gif=no', - ] + ] else: args = [] - + configure('--prefix=%s' % prefix, *args) make() make("install") From 8b6035ab8268de726933b657837d5c1bd8d13d4e Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Fri, 3 Jun 2016 00:21:41 +0200 Subject: [PATCH 07/36] Making emacs dependent on internal packages --- .../repos/builtin/packages/emacs/package.py | 36 ++++++++------- .../repos/builtin/packages/giflib/package.py | 41 +++++++++++++++++ .../repos/builtin/packages/libxpm/package.py | 44 +++++++++++++++++++ 3 files changed, 106 insertions(+), 15 deletions(-) create mode 100644 var/spack/repos/builtin/packages/giflib/package.py create mode 100644 var/spack/repos/builtin/packages/libxpm/package.py diff --git a/var/spack/repos/builtin/packages/emacs/package.py b/var/spack/repos/builtin/packages/emacs/package.py index b6d638e3c32..a9ebd6d42ff 100644 --- a/var/spack/repos/builtin/packages/emacs/package.py +++ b/var/spack/repos/builtin/packages/emacs/package.py @@ -23,6 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * +import llnl.util.tty as tty class Emacs(Package): @@ -33,28 +34,33 @@ class Emacs(Package): version('24.5', 'd74b597503a68105e61b5b9f6d065b44') - variant('use-system-deps', default=True, description='Uses the library on the system to compile') + variant('X', default=True, description="Enable a X toolkit (GTK+)") + variant('gtkplus', default=False, description="Enable a GTK+ as X toolkit (this variant is ignored if ~X)") depends_on('ncurses') - # Emacs also depends on: - # GTK or other widget library - # libtiff, png, etc. - # For now, we assume the system provides all that stuff. - # For Ubuntu 14.04 LTS: - # sudo apt-get install libgtk-3-dev libxpm-dev - # libtiff5-dev libjpeg8-dev libgif-dev libpng12-dev - depends_on('libtiff', when='~use-system-deps') - depends_on('libpng', when='~use-system-deps') + depends_on('libtiff', when='+X') + depends_on('libpng', when='+X') + depends_on('libxpm', when='+X') + depends_on('giflib', when='+X') + depends_on('gtkplus', when='+X+gtkplus') def install(self, spec, prefix): - if '~use-system-deps' in self.spec: + args = [] + if '+X' in spec: + if '+gtkplus' in spec: + toolkit = 'gtk{0}'.format(spec['gtkplus'].version.up_to(1)) + else: + toolkit = 'no' args = [ - '--with-xpm=no', - '--with-gif=no', + '--with-x', + '--with-x-toolkit={0}'.format(toolkit) ] else: - args = [] + args = ['--without-x'] + if '+gtkplus' in spec: + tty.warn('The variant +gtkplus is ignored if ~X is selected.') + + configure('--prefix={0}'.format(prefix), *args) - configure('--prefix=%s' % prefix, *args) make() make("install") diff --git a/var/spack/repos/builtin/packages/giflib/package.py b/var/spack/repos/builtin/packages/giflib/package.py new file mode 100644 index 00000000000..7082384b9b8 --- /dev/null +++ b/var/spack/repos/builtin/packages/giflib/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Giflib(Package): + """The GIFLIB project maintains the giflib service library, which has + been pulling images out of GIFs since 1989.""" + + homepage = "http://giflib.sourceforge.net/" + url = "https://downloads.sourceforge.net/project/giflib/giflib-5.1.4.tar.bz2" + + version('5.1.4', '2c171ced93c0e83bb09e6ccad8e3ba2b') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libxpm/package.py b/var/spack/repos/builtin/packages/libxpm/package.py new file mode 100644 index 00000000000..10ca8158c2c --- /dev/null +++ b/var/spack/repos/builtin/packages/libxpm/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Libxpm(Package): + """Xpm file format library""" + + homepage = "https://www.x.org/" + url = "https://www.x.org/archive//individual/lib/libXpm-3.5.11.tar.gz" + + version('3.5.11', '7c67c878ee048206b070bc0b24154f04') + version('3.5.10', 'a70507638d74541bf30a771f1e5938bb') + version('3.5.9', 'd6d4b0f76248a6b346eb42dfcdaa72a6') + version('3.5.8', '2d81d6633e67ac5562e2fbee126b2897') + version('3.5.7', '7bbc8f112f7143ed6961a58ce4e14558') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") From 0e71b5dde8fb8c235122b16368cb075a35be9bfb Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 10:07:13 +0200 Subject: [PATCH 08/36] unit test : update dictionary extending lists --- lib/spack/spack/test/modules.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index c73badf8f24..0465c3e67fb 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -27,6 +27,7 @@ import StringIO import spack.modules +import unittest from spack.test.mock_packages_test import MockPackagesTest FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) @@ -100,6 +101,33 @@ def mock_open(filename, mode): } } +class HelperFunctionsTests(unittest.TestCase): + def test_update_dictionary_extending_list(self): + target = { + 'foo': { + 'a': 1, + 'b': 2, + 'd': 4 + }, + 'bar': [1, 2, 4], + 'baz': 'foobar' + } + update = { + 'foo': { + 'c': 3, + }, + 'bar': [3], + 'baz': 'foobaz', + 'newkey': { + 'd': 4 + } + } + spack.modules.update_dictionary_extending_lists(target, update) + self.assertTrue(len(target) == 4) + self.assertTrue(len(target['foo']) == 4) + self.assertTrue(len(target['bar']) == 4) + self.assertEqual(target['baz'], 'foobaz') + class TclTests(MockPackagesTest): def setUp(self): From 2207ab8f23d099f535e95eec6b604862a86e7275 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 10:44:37 +0200 Subject: [PATCH 09/36] unit test : inspect_path --- lib/spack/spack/test/modules.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 0465c3e67fb..94d508853c7 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -101,6 +101,7 @@ def mock_open(filename, mode): } } + class HelperFunctionsTests(unittest.TestCase): def test_update_dictionary_extending_list(self): target = { @@ -128,6 +129,14 @@ def test_update_dictionary_extending_list(self): self.assertTrue(len(target['bar']) == 4) self.assertEqual(target['baz'], 'foobaz') + def test_inspect_path(self): + env = spack.modules.inspect_path('/usr') + names = [item.name for item in env] + self.assertTrue('PATH' in names) + self.assertTrue('LIBRARY_PATH' in names) + self.assertTrue('LD_LIBRARY_PATH' in names) + self.assertTrue('CPATH' in names) + class TclTests(MockPackagesTest): def setUp(self): From 97e28348165d88040fbdc1772c69ae3d06c7378f Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 11:20:35 +0200 Subject: [PATCH 10/36] unit test : wrong tag for dependencies --- lib/spack/spack/test/modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 94d508853c7..9320f6673fa 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -163,6 +163,7 @@ def test_simple_case(self): spec = spack.spec.Spec('mpich@3.0.4 arch=x86-linux') content = self.get_modulefile_content(spec) self.assertTrue('module-whatis "mpich @3.0.4"' in content) + self.assertRaises(TypeError, spack.modules.dependencies, spec, 'non-existing-tag') def test_autoload(self): spack.modules.CONFIGURATION = configuration_autoload_direct From a3f39c23157cd094d50b3b661be6574cd1814305 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 11:23:23 +0200 Subject: [PATCH 11/36] autopep8 : fixed style issues --- lib/spack/spack/test/modules.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 9320f6673fa..d76e0cd054c 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -103,6 +103,7 @@ def mock_open(filename, mode): class HelperFunctionsTests(unittest.TestCase): + def test_update_dictionary_extending_list(self): target = { 'foo': { @@ -139,6 +140,7 @@ def test_inspect_path(self): class TclTests(MockPackagesTest): + def setUp(self): super(TclTests, self).setUp() self.configuration_obj = spack.modules.CONFIGURATION @@ -163,7 +165,8 @@ def test_simple_case(self): spec = spack.spec.Spec('mpich@3.0.4 arch=x86-linux') content = self.get_modulefile_content(spec) self.assertTrue('module-whatis "mpich @3.0.4"' in content) - self.assertRaises(TypeError, spack.modules.dependencies, spec, 'non-existing-tag') + self.assertRaises(TypeError, spack.modules.dependencies, + spec, 'non-existing-tag') def test_autoload(self): spack.modules.CONFIGURATION = configuration_autoload_direct From 408aa10210384490348d997069dce02fe61d336d Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 16:18:15 +0200 Subject: [PATCH 12/36] unit test : whitelist --- lib/spack/spack/test/modules.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index d76e0cd054c..098777c48b2 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -84,7 +84,8 @@ def mock_open(filename, mode): configuration_blacklist = { 'enable': ['tcl'], 'tcl': { - 'blacklist': ['callpath'], + 'whitelist': ['zmpi'], + 'blacklist': ['callpath', 'mpi'], 'all': { 'autoload': 'direct' } @@ -207,6 +208,7 @@ def test_blacklist(self): spack.modules.CONFIGURATION = configuration_blacklist spec = spack.spec.Spec('mpileaks arch=x86-linux') content = self.get_modulefile_content(spec) + print('\n'.join(content)) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) self.assertEqual(len([x for x in content if 'module load ' in x]), 1) From 09b1daa7b971eaac221cbd1c043df8f46a38fa33 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 16:23:41 +0200 Subject: [PATCH 13/36] unit test : write a blacklisted module --- lib/spack/spack/test/modules.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 098777c48b2..46d6824acd2 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -211,6 +211,9 @@ def test_blacklist(self): print('\n'.join(content)) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) self.assertEqual(len([x for x in content if 'module load ' in x]), 1) + spec = spack.spec.Spec('callpath arch=x86-linux') + # Returns a StringIO instead of a string as no module file was written + self.assertRaises(AttributeError, self.get_modulefile_content, spec) def test_conflicts(self): spack.modules.CONFIGURATION = configuration_conflicts From 422d3d4db60f7e00fdbda452c94bfb5d155f304e Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 16:46:15 +0200 Subject: [PATCH 14/36] unit test : write a whitelisted module --- lib/spack/spack/test/modules.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 46d6824acd2..7fb1c6a9216 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -214,6 +214,11 @@ def test_blacklist(self): spec = spack.spec.Spec('callpath arch=x86-linux') # Returns a StringIO instead of a string as no module file was written self.assertRaises(AttributeError, self.get_modulefile_content, spec) + spec = spack.spec.Spec('zmpi arch=x86-linux') + content = self.get_modulefile_content(spec) + print('\n'.join(content)) + self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) + self.assertEqual(len([x for x in content if 'module load ' in x]), 1) def test_conflicts(self): spack.modules.CONFIGURATION = configuration_conflicts From ffa6fed3a3205d15d6c9ff8ca8c92cbecdbe2a0c Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 17:20:06 +0200 Subject: [PATCH 15/36] unit test : custom module file load --- lib/spack/spack/test/modules.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 7fb1c6a9216..bf5555d9d48 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -75,8 +75,13 @@ def mock_open(filename, mode): 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} }, 'arch=x86-linux': { - 'environment': {'set': {'FOO': 'foo'}, - 'unset': ['BAR']} + 'environment': { + 'set': {'FOO': 'foo'}, + 'unset': ['BAR'] + } + }, + 'arch=x64-linux': { + 'load': ['foo/bar'] } } } @@ -203,6 +208,8 @@ def test_alter_environment(self): self.assertEqual( len([x for x in content if 'setenv FOO "foo"' in x]), 0) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0) + self.assertEqual(len([x for x in content if 'is-loaded foo/bar' in x]), 1) + self.assertEqual(len([x for x in content if 'module load foo/bar' in x]), 1) def test_blacklist(self): spack.modules.CONFIGURATION = configuration_blacklist From 2d6db06fca54aa840593a1dbb4e99e8f0e58bbce Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 17:24:27 +0200 Subject: [PATCH 16/36] unit test : prerequisites --- lib/spack/spack/test/modules.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index bf5555d9d48..99cc280d27c 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -68,6 +68,24 @@ def mock_open(filename, mode): } } +configuration_prerequisites_direct = { + 'enable': ['tcl'], + 'tcl': { + 'all': { + 'prerequisites': 'direct' + } + } +} + +configuration_prerequisites_all = { + 'enable': ['tcl'], + 'tcl': { + 'all': { + 'prerequisites': 'all' + } + } +} + configuration_alter_environment = { 'enable': ['tcl'], 'tcl': { @@ -187,6 +205,17 @@ def test_autoload(self): self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5) self.assertEqual(len([x for x in content if 'module load ' in x]), 5) + def test_prerequisites(self): + spack.modules.CONFIGURATION = configuration_prerequisites_direct + spec = spack.spec.Spec('mpileaks arch=x86-linux') + content = self.get_modulefile_content(spec) + self.assertEqual(len([x for x in content if 'prereq' in x]), 2) + + spack.modules.CONFIGURATION = configuration_prerequisites_all + spec = spack.spec.Spec('mpileaks arch=x86-linux') + content = self.get_modulefile_content(spec) + self.assertEqual(len([x for x in content if 'prereq' in x]), 5) + def test_alter_environment(self): spack.modules.CONFIGURATION = configuration_alter_environment spec = spack.spec.Spec('mpileaks arch=x86-linux') From d43cfefa43b47942388003b30ab7fd52b3035fde Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 17:24:52 +0200 Subject: [PATCH 17/36] autopep8 --- lib/spack/spack/test/modules.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 99cc280d27c..05a33494809 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -237,8 +237,10 @@ def test_alter_environment(self): self.assertEqual( len([x for x in content if 'setenv FOO "foo"' in x]), 0) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0) - self.assertEqual(len([x for x in content if 'is-loaded foo/bar' in x]), 1) - self.assertEqual(len([x for x in content if 'module load foo/bar' in x]), 1) + self.assertEqual( + len([x for x in content if 'is-loaded foo/bar' in x]), 1) + self.assertEqual( + len([x for x in content if 'module load foo/bar' in x]), 1) def test_blacklist(self): spack.modules.CONFIGURATION = configuration_blacklist From 669caefccb431a79dcdb1d3743afbac86c84630a Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 18:00:18 +0200 Subject: [PATCH 18/36] unit test : wrong naming scheme in conflict --- lib/spack/spack/test/modules.py | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 05a33494809..f695bf75f19 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -125,6 +125,16 @@ def mock_open(filename, mode): } } +configuration_wrong_conflicts = { + 'enable': ['tcl'], + 'tcl': { + 'naming_scheme': '{name}/{version}-{compiler.name}', + 'all': { + 'conflict': ['{name}/{compiler.name}'] + } + } +} + class HelperFunctionsTests(unittest.TestCase): @@ -268,3 +278,45 @@ def test_conflicts(self): len([x for x in content if x == 'conflict mpileaks']), 1) self.assertEqual( len([x for x in content if x == 'conflict intel/14.0.1']), 1) + + spack.modules.CONFIGURATION = configuration_wrong_conflicts + self.assertRaises(SystemExit, self.get_modulefile_content, spec) + +configuration_dotkit = { + 'enable': ['dotkit'], + 'dotkit': { + 'all': { + 'prerequisites': 'direct' + } + } +} + + +class DotkitTests(MockPackagesTest): + + def setUp(self): + super(DotkitTests, self).setUp() + self.configuration_obj = spack.modules.CONFIGURATION + spack.modules.open = mock_open + # Make sure that a non-mocked configuration will trigger an error + spack.modules.CONFIGURATION = None + + def tearDown(self): + del spack.modules.open + spack.modules.CONFIGURATION = self.configuration_obj + super(DotkitTests, self).tearDown() + + def get_modulefile_content(self, spec): + spec.concretize() + generator = spack.modules.Dotkit(spec) + generator.write() + content = FILE_REGISTRY[generator.file_name].split('\n') + return content + + def test_dotkit(self): + spack.modules.CONFIGURATION = configuration_dotkit + spec = spack.spec.Spec('mpileaks arch=x86-linux') + content = self.get_modulefile_content(spec) + print('\n'.join(content)) + self.assertTrue('#c spack' in content) + self.assertTrue('#d mpileaks @2.3' in content) From 02cadb882b60b33a86b2ffc4d3e2320c16e66c7b Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 18:09:06 +0200 Subject: [PATCH 19/36] unit test : suffixes --- lib/spack/spack/test/modules.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index f695bf75f19..7f4643409e5 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -135,6 +135,17 @@ def mock_open(filename, mode): } } +configuration_suffix = { + 'enable': ['tcl'], + 'tcl': { + 'mpileaks': { + 'suffixes': { + '+debug': 'foo', + '~debug': 'bar' + } + } + } +} class HelperFunctionsTests(unittest.TestCase): @@ -282,6 +293,20 @@ def test_conflicts(self): spack.modules.CONFIGURATION = configuration_wrong_conflicts self.assertRaises(SystemExit, self.get_modulefile_content, spec) + + def test_suffixes(self): + spack.modules.CONFIGURATION = configuration_suffix + spec = spack.spec.Spec('mpileaks+debug arch=x86-linux') + spec.concretize() + generator = spack.modules.TclModule(spec) + self.assertTrue('foo' in generator.use_name) + + spec = spack.spec.Spec('mpileaks~debug arch=x86-linux') + spec.concretize() + generator = spack.modules.TclModule(spec) + self.assertTrue('bar' in generator.use_name) + + configuration_dotkit = { 'enable': ['dotkit'], 'dotkit': { From bdd874eb7fad2e193cb1186a499b9cd742632f62 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 4 Jun 2016 18:09:28 +0200 Subject: [PATCH 20/36] autopep8 --- lib/spack/spack/test/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 7f4643409e5..bb83ea9059b 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -147,6 +147,7 @@ def mock_open(filename, mode): } } + class HelperFunctionsTests(unittest.TestCase): def test_update_dictionary_extending_list(self): @@ -293,7 +294,6 @@ def test_conflicts(self): spack.modules.CONFIGURATION = configuration_wrong_conflicts self.assertRaises(SystemExit, self.get_modulefile_content, spec) - def test_suffixes(self): spack.modules.CONFIGURATION = configuration_suffix spec = spack.spec.Spec('mpileaks+debug arch=x86-linux') From 7ce08c4216189d747352c1319f921e823785daa8 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 8 Jun 2016 13:51:31 -0600 Subject: [PATCH 21/36] Remove last two remaining from external imports --- lib/spack/spack/cmd/repo.py | 2 +- lib/spack/spack/repository.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 399237b1699..cbd8f4784e9 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -26,7 +26,7 @@ import re import shutil -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.filesystem import join_path, mkdirp diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 70134964ad8..2c160a5f45d 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -30,7 +30,7 @@ import re import traceback from bisect import bisect_left -from external import yaml +import yaml import llnl.util.tty as tty from llnl.util.filesystem import * From 4952c4c7def96caef2b47160f8b9e08392ae348d Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 8 Jun 2016 14:22:26 -0600 Subject: [PATCH 22/36] Also remove from external from bin/spack --- bin/spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/spack b/bin/spack index 3544feb10ac..dac48437a6f 100755 --- a/bin/spack +++ b/bin/spack @@ -77,7 +77,7 @@ import llnl.util.tty as tty from llnl.util.tty.color import * import spack from spack.error import SpackError -from external import argparse +import argparse # Command parsing parser = argparse.ArgumentParser( From a33077b77fc83bf2cb7e78f0724922fc61137898 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 18 Jun 2016 13:07:14 +0200 Subject: [PATCH 23/36] unit tests : fixed test_alter_environment --- lib/spack/spack/test/modules.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 59ee6078262..889c09c730f 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -93,17 +93,14 @@ def mock_open(filename, mode): 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} }, 'platform=test target=x86_64': { - 'environment': {'set': {'FOO': 'foo'}, - 'unset': ['BAR']} - 'arch=x86-linux': { 'environment': { 'set': {'FOO': 'foo'}, 'unset': ['BAR'] } }, - 'arch=x64-linux': { + 'platform=test target=x86_32': { 'load': ['foo/bar'] - } + } } } @@ -246,7 +243,6 @@ def test_alter_environment(self): spack.modules.CONFIGURATION = configuration_alter_environment spec = spack.spec.Spec('mpileaks platform=test target=x86_64') content = self.get_modulefile_content(spec) - print content self.assertEqual( len([x for x in content @@ -257,7 +253,6 @@ def test_alter_environment(self): spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32') content = self.get_modulefile_content(spec) - print content self.assertEqual( len([x for x in content From 670669ef808a9a737e760ced8a9fd1d000f360b9 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 18 Jun 2016 13:10:35 +0200 Subject: [PATCH 24/36] qa : fixed flake8 issues --- lib/spack/spack/test/modules.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 889c09c730f..443e4d13a69 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -100,7 +100,7 @@ def mock_open(filename, mode): }, 'platform=test target=x86_32': { 'load': ['foo/bar'] - } + } } } @@ -201,7 +201,6 @@ def tearDown(self): def get_modulefile_content(self, spec): spec.concretize() - print spec, '&&&&&' generator = spack.modules.TclModule(spec) generator.write() content = FILE_REGISTRY[generator.file_name].split('\n') @@ -269,7 +268,6 @@ def test_blacklist(self): spack.modules.CONFIGURATION = configuration_blacklist spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) - print('\n'.join(content)) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) self.assertEqual(len([x for x in content if 'module load ' in x]), 1) spec = spack.spec.Spec('callpath arch=x86-linux') @@ -277,7 +275,6 @@ def test_blacklist(self): self.assertRaises(AttributeError, self.get_modulefile_content, spec) spec = spack.spec.Spec('zmpi arch=x86-linux') content = self.get_modulefile_content(spec) - print('\n'.join(content)) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) self.assertEqual(len([x for x in content if 'module load ' in x]), 1) @@ -343,6 +340,5 @@ def test_dotkit(self): spack.modules.CONFIGURATION = configuration_dotkit spec = spack.spec.Spec('mpileaks arch=x86-linux') content = self.get_modulefile_content(spec) - print('\n'.join(content)) self.assertTrue('#c spack' in content) self.assertTrue('#d mpileaks @2.3' in content) From 060fb596f7897b0da14a0bcb601a644e489ac140 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 22 Jun 2016 09:52:17 -0500 Subject: [PATCH 25/36] Add latest version of netcdf-fortran --- var/spack/repos/builtin/packages/netcdf-fortran/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 3d1951ceeee..e52ff1af874 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -24,12 +24,14 @@ ############################################################################## from spack import * + class NetcdfFortran(Package): """Fortran interface for NetCDF4""" homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-fortran-4.4.3.tar.gz" + version('4.4.4', 'e855c789cd72e1b8bc1354366bf6ac72') version('4.4.3', 'bfd4ae23a34635b273d3eb0d91cbde9e') depends_on('netcdf') From e398c0dc5bdec54ee06ffa7694502b9a73af4011 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 22 Jun 2016 11:00:00 -0400 Subject: [PATCH 26/36] Update Julia to 0.4.6 --- var/spack/repos/builtin/packages/julia/package.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/julia/package.py b/var/spack/repos/builtin/packages/julia/package.py index 69b52130e9c..c2700a81493 100644 --- a/var/spack/repos/builtin/packages/julia/package.py +++ b/var/spack/repos/builtin/packages/julia/package.py @@ -32,10 +32,14 @@ class Julia(Package): version('master', git='https://github.com/JuliaLang/julia.git', branch='master') + version('release-0.4', + git='https://github.com/JuliaLang/julia.git', branch='release-0.4') + version('0.4.6', 'd88db18c579049c23ab8ef427ccedf5d', preferred=True) version('0.4.5', '69141ff5aa6cee7c0ec8c85a34aa49a6') version('0.4.3', '8a4a59fd335b05090dd1ebefbbe5aaac') - patch('gc.patch') + patch('gc.patch', when='@0.4:0.4.5') + patch('gc.patch', when='@release-0.4') patch('openblas.patch', when='@0.4:0.4.5') # Build-time dependencies: @@ -92,11 +96,6 @@ class Julia(Package): depends_on("mpi") def install(self, spec, prefix): - if '@master' in spec: - # Julia needs to know the offset from a specific commit - git = which('git') - git('fetch', '--unshallow') - # Explicitly setting CC, CXX, or FC breaks building libuv, one # of Julia's dependencies. This might be a Darwin-specific # problem. Given how Spack sets up compilers, Julia should From b66f5b6231107606aad757cfe2bd65fb7ff6d886 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 22 Jun 2016 10:05:20 -0500 Subject: [PATCH 27/36] Add latest version of cURL --- var/spack/repos/builtin/packages/curl/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py index e3de6ee5a15..79570205faf 100644 --- a/var/spack/repos/builtin/packages/curl/package.py +++ b/var/spack/repos/builtin/packages/curl/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class Curl(Package): """cURL is an open source command line tool and library for transferring data with URL syntax""" @@ -31,6 +32,7 @@ class Curl(Package): homepage = "http://curl.haxx.se" url = "http://curl.haxx.se/download/curl-7.46.0.tar.bz2" + version('7.49.1', '6bb1f7af5b58b30e4e6414b8c1abccab') version('7.47.1', '9ea3123449439bbd960cd25cf98796fb') version('7.46.0', '9979f989a2a9930d10f1b3deeabc2148') version('7.45.0', '62c1a352b28558f25ba6209214beadc8') From f29b55957ac00b440a7c8b0f609ee5380c32d5db Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 22 Jun 2016 17:08:02 +0200 Subject: [PATCH 28/36] documentation : removed references to never implemented features --- lib/spack/docs/configuration.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index c613071c65b..a6f876b2aad 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -140,7 +140,6 @@ Here's an example packages.yaml file that sets preferred packages: packages: dyninst: compiler: [gcc@4.9] - variants: +debug gperftools: version: [2.2, 2.4, 2.3] all: @@ -150,8 +149,8 @@ Here's an example packages.yaml file that sets preferred packages: At a high level, this example is specifying how packages should be -concretized. The dyninst package should prefer using gcc 4.9 and -be built with debug options. The gperftools package should prefer version +concretized. The dyninst package should prefer using gcc 4.9. +The gperftools package should prefer version 2.2 over 2.4. Every package on the system should prefer mvapich for its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). These options are used to fill in implicit defaults. Any of them can be overwritten @@ -160,7 +159,7 @@ on the command line if explicitly requested. Each packages.yaml file begins with the string ``packages:`` and package names are specified on the next level. The special string ``all`` applies settings to each package. Underneath each package name is -one or more components: ``compiler``, ``variants``, ``version``, +one or more components: ``compiler``, ``version``, or ``providers``. Each component has an ordered list of spec ``constraints``, with earlier entries in the list being preferred over later entries. From c904a42c5b7e905bd793e90d1deb4b32023404f2 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 22 Jun 2016 11:09:45 -0400 Subject: [PATCH 29/36] Address flake errors --- .../repos/builtin/packages/julia/package.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/julia/package.py b/var/spack/repos/builtin/packages/julia/package.py index c2700a81493..99d138dcff9 100644 --- a/var/spack/repos/builtin/packages/julia/package.py +++ b/var/spack/repos/builtin/packages/julia/package.py @@ -22,8 +22,9 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## + from spack import * -import os + class Julia(Package): """The Julia Language: A fresh approach to technical computing""" @@ -101,15 +102,16 @@ def install(self, spec, prefix): # problem. Given how Spack sets up compilers, Julia should # still use Spack's compilers, even if we don't specify them # explicitly. - options = [#"CC=cc", - #"CXX=c++", - #"FC=fc", - #"USE_SYSTEM_ARPACK=1", - #"USE_SYSTEM_FFTW=1", - #"USE_SYSTEM_GMP=1", - #"USE_SYSTEM_MPFR=1", - #TODO "USE_SYSTEM_PCRE=1", - "prefix=%s" % prefix] + options = [ + # "CC=cc", + # "CXX=c++", + # "FC=fc", + # "USE_SYSTEM_ARPACK=1", + # "USE_SYSTEM_FFTW=1", + # "USE_SYSTEM_GMP=1", + # "USE_SYSTEM_MPFR=1", + # "USE_SYSTEM_PCRE=1", + "prefix=%s" % prefix] with open('Make.user', 'w') as f: f.write('\n'.join(options) + '\n') make() From ffe9e77e36ca2c64ff8d2ebdc14c6b04d1184d15 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Wed, 22 Jun 2016 12:08:28 -0400 Subject: [PATCH 30/36] Improved Spack database errors to help users fix them. --- lib/spack/spack/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index f941346bb13..9b0e88b9ad7 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -631,11 +631,11 @@ def _exit(self): class CorruptDatabaseError(SpackError): def __init__(self, path, msg=''): super(CorruptDatabaseError, self).__init__( - "Spack database is corrupt: %s. %s" % (path, msg)) + "Spack database is corrupt: %s. %s. Try running `spack reindex` to fix." % (path, msg)) class InvalidDatabaseVersionError(SpackError): def __init__(self, expected, found): super(InvalidDatabaseVersionError, self).__init__( - "Expected database version %s but found version %s" % + "Expected database version %s but found version %s. Try running `spack reindex` to fix." % (expected, found)) From 10810482bc308e5a4f6d201c201a56b91dc9bd64 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Wed, 22 Jun 2016 14:03:32 -0400 Subject: [PATCH 31/36] Update database.py PEP8 --- lib/spack/spack/database.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 9b0e88b9ad7..a4bbff3d5a0 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -631,11 +631,13 @@ def _exit(self): class CorruptDatabaseError(SpackError): def __init__(self, path, msg=''): super(CorruptDatabaseError, self).__init__( - "Spack database is corrupt: %s. %s. Try running `spack reindex` to fix." % (path, msg)) + "Spack database is corrupt: %s. %s." + \ + "Try running `spack reindex` to fix." % (path, msg)) class InvalidDatabaseVersionError(SpackError): def __init__(self, expected, found): super(InvalidDatabaseVersionError, self).__init__( - "Expected database version %s but found version %s. Try running `spack reindex` to fix." % + "Expected database version %s but found version %s." + \ + "Try running `spack reindex` to fix." % (expected, found)) From 8921bf455215ebdf180136586a4fb25db501c8ef Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 23 Jun 2016 10:08:40 -0500 Subject: [PATCH 32/36] Add MPI environment variables to packages --- var/spack/repos/builtin/packages/mpich/package.py | 5 +++++ var/spack/repos/builtin/packages/mvapich2/package.py | 5 +++++ var/spack/repos/builtin/packages/openmpi/package.py | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index ea1844aea09..90670ee517e 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -50,6 +50,11 @@ class Mpich(Package): provides('mpi@:1.3', when='@1:') def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + spack_env.set('MPICC', join_path(self.prefix.bin, 'mpicc')) + spack_env.set('MPICXX', join_path(self.prefix.bin, 'mpic++')) + spack_env.set('MPIF77', join_path(self.prefix.bin, 'mpif77')) + spack_env.set('MPIF90', join_path(self.prefix.bin, 'mpif90')) + spack_env.set('MPICH_CC', spack_cc) spack_env.set('MPICH_CXX', spack_cxx) spack_env.set('MPICH_F77', spack_f77) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index 34bace10d66..da386445bbd 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -191,6 +191,11 @@ def setup_environment(self, spack_env, run_env): run_env.set('SLURM_MPI_TYPE', 'pmi2') def setup_dependent_environment(self, spack_env, run_env, extension_spec): + spack_env.set('MPICC', join_path(self.prefix.bin, 'mpicc')) + spack_env.set('MPICXX', join_path(self.prefix.bin, 'mpicxx')) + spack_env.set('MPIF77', join_path(self.prefix.bin, 'mpif77')) + spack_env.set('MPIF90', join_path(self.prefix.bin, 'mpif90')) + spack_env.set('MPICH_CC', spack_cc) spack_env.set('MPICH_CXX', spack_cxx) spack_env.set('MPICH_F77', spack_f77) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 588d862d968..15c8eefe964 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -109,6 +109,11 @@ def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) # NOQA: ignore=E501 def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + spack_env.set('MPICC', join_path(self.prefix.bin, 'mpicc')) + spack_env.set('MPICXX', join_path(self.prefix.bin, 'mpic++')) + spack_env.set('MPIF77', join_path(self.prefix.bin, 'mpif77')) + spack_env.set('MPIF90', join_path(self.prefix.bin, 'mpif90')) + spack_env.set('OMPI_CC', spack_cc) spack_env.set('OMPI_CXX', spack_cxx) spack_env.set('OMPI_FC', spack_fc) From 71a297f12108fa0628ffe205ee40811d8b5a51a6 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 23 Jun 2016 16:44:41 -0500 Subject: [PATCH 33/36] Fix backup logic in filter_file --- lib/spack/llnl/util/filesystem.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index d72e8bae924..e800c6717ad 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -48,7 +48,7 @@ def filter_file(regex, repl, *filenames, **kwargs): """Like sed, but uses python regular expressions. - Filters every line of file through regex and replaces the file + Filters every line of each file through regex and replaces the file with a filtered version. Preserves mode of filtered files. As with re.sub, ``repl`` can be either a string or a callable. @@ -59,7 +59,7 @@ def filter_file(regex, repl, *filenames, **kwargs): Keyword Options: string[=False] If True, treat regex as a plain string. - backup[=True] Make a backup files suffixed with ~ + backup[=True] Make backup file(s) suffixed with ~ ignore_absent[=False] Ignore any files that don't exist. """ string = kwargs.get('string', False) @@ -80,26 +80,26 @@ def groupid_to_group(x): regex = re.escape(regex) for filename in filenames: - backup = filename + "~" + backup_filename = filename + "~" if ignore_absent and not os.path.exists(filename): continue - shutil.copy(filename, backup) + shutil.copy(filename, backup_filename) try: - with closing(open(backup)) as infile: + with closing(open(backup_filename)) as infile: with closing(open(filename, 'w')) as outfile: for line in infile: foo = re.sub(regex, repl, line) outfile.write(foo) except: # clean up the original file on failure. - shutil.move(backup, filename) + shutil.move(backup_filename, filename) raise finally: if not backup: - shutil.rmtree(backup, ignore_errors=True) + os.remove(backup_filename) class FileFilter(object): @@ -114,7 +114,7 @@ def filter(self, regex, repl, **kwargs): def change_sed_delimiter(old_delim, new_delim, *filenames): """Find all sed search/replace commands and change the delimiter. e.g., if the file contains seds that look like 's///', you can - call change_sed_delimeter('/', '@', file) to change the + call change_sed_delimiter('/', '@', file) to change the delimiter to '@'. NOTE that this routine will fail if the delimiter is ' or ". @@ -179,7 +179,7 @@ def install(src, dest): """Manually install a file to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) - # Expand dsst to its eventual full path if it is a directory. + # Expand dest to its eventual full path if it is a directory. if os.path.isdir(dest): dest = join_path(dest, os.path.basename(src)) @@ -219,7 +219,7 @@ def mkdirp(*paths): if not os.path.exists(path): os.makedirs(path) elif not os.path.isdir(path): - raise OSError(errno.EEXIST, "File alredy exists", path) + raise OSError(errno.EEXIST, "File already exists", path) def force_remove(*paths): @@ -309,7 +309,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): Optional args: - order=[pre|post] -- Whether to do pre- or post-order traveral. + order=[pre|post] -- Whether to do pre- or post-order traversal. ignore= -- Predicate indicating which files to ignore. @@ -414,7 +414,7 @@ def fix_darwin_install_name(path): currently won't follow subfolders. Args: - path: directory in which .dylib files are alocated + path: directory in which .dylib files are located """ libs = glob.glob(join_path(path, "*.dylib")) @@ -438,7 +438,7 @@ def to_link_flags(library): A string of linking flags. """ dir = os.path.dirname(library) - # Asume libXYZ.suffix + # Assume libXYZ.suffix name = os.path.basename(library)[3:].split(".")[0] res = '-L%s -l%s' % (dir, name) return res From 2c6c21a3bf6acb7345e46f55141ced829912dcfc Mon Sep 17 00:00:00 2001 From: mwilliammyers Date: Thu, 23 Jun 2016 15:54:20 -0600 Subject: [PATCH 34/36] Add new package py-protobuf --- .../builtin/packages/py-protobuf/package.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-protobuf/package.py diff --git a/var/spack/repos/builtin/packages/py-protobuf/package.py b/var/spack/repos/builtin/packages/py-protobuf/package.py new file mode 100644 index 00000000000..da3a65c22ea --- /dev/null +++ b/var/spack/repos/builtin/packages/py-protobuf/package.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyProtobuf(Package): + """Protocol buffers are Google's language-neutral, platform-neutral, + extensible mechanism for serializing structured data - think XML, but + smaller, faster, and simpler. You define how you want your data to be + structured once, then you can use special generated source code to easily + write and read your structured data to and from a variety of data streams + and using a variety of languages.""" + + homepage = 'https://developers.google.com/protocol-buffers/' + url = 'https://pypi.python.org/packages/source/p/protobuf/protobuf-3.0.0b2.tar.gz' + + version('3.0.0b2', 'f0d3bd2394345a9af4a277cd0302ae83') + version('2.6.1', '6bf843912193f70073db7f22e2ea55e2') + version('2.5.0', '338813f3629d59e9579fed9035ecd457') + version('2.4.1', '72f5141d20ab1bcae6b1e00acfb1068a') + version('2.3.0', 'bb020c962f252fe81bfda8fb433bafdd') + + extends('python') + + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) From 11863d2de3e764f56b595e33a1ac98e9539553c6 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sun, 26 Jun 2016 21:56:06 +0200 Subject: [PATCH 35/36] packages : moved adios and mxml into builtin repo --- var/spack/{ => repos/builtin}/packages/adios/package.py | 0 var/spack/{ => repos/builtin}/packages/mxml/package.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename var/spack/{ => repos/builtin}/packages/adios/package.py (100%) rename var/spack/{ => repos/builtin}/packages/mxml/package.py (100%) diff --git a/var/spack/packages/adios/package.py b/var/spack/repos/builtin/packages/adios/package.py similarity index 100% rename from var/spack/packages/adios/package.py rename to var/spack/repos/builtin/packages/adios/package.py diff --git a/var/spack/packages/mxml/package.py b/var/spack/repos/builtin/packages/mxml/package.py similarity index 100% rename from var/spack/packages/mxml/package.py rename to var/spack/repos/builtin/packages/mxml/package.py From 9de05031889bca759f9d8b249d86930fe93ac1d0 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sun, 26 Jun 2016 22:00:10 +0200 Subject: [PATCH 36/36] qa : flake8 issues --- .../repos/builtin/packages/adios/package.py | 23 +++++++++++-------- .../repos/builtin/packages/mxml/package.py | 15 ++++++------ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/var/spack/repos/builtin/packages/adios/package.py b/var/spack/repos/builtin/packages/adios/package.py index 260dcbe851d..9e0452ba6f1 100644 --- a/var/spack/repos/builtin/packages/adios/package.py +++ b/var/spack/repos/builtin/packages/adios/package.py @@ -1,15 +1,18 @@ import os from spack import * + + class Adios(Package): - """The Adaptable IO System (ADIOS) provides a simple, - flexible way for scientists to describe the - data in their code that may need to be written, - read, or processed outside of the running simulation """ - + The Adaptable IO System (ADIOS) provides a simple, + flexible way for scientists to describe the + data in their code that may need to be written, + read, or processed outside of the running simulation + """ + homepage = "http://www.olcf.ornl.gov/center-projects/adios/" - url = "http://users.nccs.gov/~pnorbert/adios-1.9.0.tar.gz" + url = "http://users.nccs.gov/~pnorbert/adios-1.9.0.tar.gz" version('1.9.0', 'dbf5cb10e32add2f04c9b4052b7ffa76') @@ -22,13 +25,13 @@ class Adios(Package): depends_on('mxml') def install(self, spec, prefix): - configure_args = ["--prefix=%s" % prefix, - "--with-mxml=%s" % spec['mxml'].prefix, + configure_args = ["--prefix=%s" % prefix, + "--with-mxml=%s" % spec['mxml'].prefix, "--with-hdf5=%s" % spec['hdf5'].prefix, "--with-netcdf=%s" % os.environ["NETCDF_DIR"], "--with-infiniband=no", - "MPICC=cc","MPICXX=CC","MPIFC=ftn", - "CPPFLAGS=-DMPICH_IGNORE_CXX_SEEK"] + "MPICC=cc", "MPICXX=CC", "MPIFC=ftn", + "CPPFLAGS=-DMPICH_IGNORE_CXX_SEEK"] if spec.satisfies('%gcc'): configure_args.extend(["CC=gcc", "CXX=g++", "FC=gfortran"]) diff --git a/var/spack/repos/builtin/packages/mxml/package.py b/var/spack/repos/builtin/packages/mxml/package.py index f79251d312f..254a5f1595f 100644 --- a/var/spack/repos/builtin/packages/mxml/package.py +++ b/var/spack/repos/builtin/packages/mxml/package.py @@ -1,14 +1,15 @@ -import os from spack import * + class Mxml(Package): - """Mini-XML is a small XML library that you can use to read and write XML - and XML-like data files in your application without requiring large - non-standard libraries + """ + Mini-XML is a small XML library that you can use to read and write XML + and XML-like data files in your application without requiring large + non-standard libraries """ homepage = "http://www.msweet.org" - url = "http://www.msweet.org/files/project3/mxml-2.9.tar.gz" + url = "http://www.msweet.org/files/project3/mxml-2.9.tar.gz" version('2.9', 'e21cad0f7aacd18f942aa0568a8dee19') version('2.8', 'd85ee6d30de053581242c4a86e79a5d2') @@ -16,11 +17,11 @@ class Mxml(Package): version('2.6', '68977789ae64985dddbd1a1a1652642e') version('2.5', 'f706377fba630b39fa02fd63642b17e5') - # module swap PrgEnv-intel PrgEnv-$COMP (Can use whatever compiler you want to use) + # module swap PrgEnv-intel PrgEnv-$COMP + # (Can use whatever compiler you want to use) # Case statement to change CC and CXX flags def install(self, spec, prefix): configure('--prefix=%s' % prefix, "--disable-shared", 'CFLAGS=-static') make() make("install") -